**On the Host servers [nfs01 and nfs02]**
Device Mapper Multipathing Devices (DMMP)
- Scan for virtual disks \\
- Display the multipath device topology \\
- Create a partition on a multipath device node \\
- Add a partition to Device Mapper(DM) \\
- Crate a file systems on a Device Mapper partition \\
- Mount a Device mapper partition \\
1. Install all packages that are needed
[root@nfs02 ~]# yum install device-mapper device-mapper-devel device-mapper-multipath kpartx
[root@nfs02 ~]# yum install sg3_utils sg3_utils-devel
2. Start multipath service
[root@nfs02 ~]# service multipathd start
Starting multipathd daemon: [ OK ]
[root@nfs02 ~]# chkconfig multipathd on
[root@nfs02 ~]#
3. Scan for devices and partitions
[root@nfs02 ~]# rescan-scsi-bus.sh
Scanning SCSI subsystem for new devices
Scanning host 0 for SCSI target IDs 0 1 2 3 4 5 6 7, all LUNs
Scanning host 1 for SCSI target IDs 0 1 2 3 4 5 6 7, all LUNs
Scanning host 2 for SCSI target IDs 0 1 2 3 4 5 6 7, all LUNs
Scanning host 3 for SCSI target IDs 0 1 2 3 4 5 6 7, all LUNs
Scanning host 4 for SCSI target IDs 0 1 2 3 4 5 6 7, all LUNs
Scanning host 5 for SCSI target IDs 0 1 2 3 4 5 6 7, all LUNs
Scanning host 6 for SCSI target IDs 0 1 2 3 4 5 6 7, all LUNs
Scanning host 7 for SCSI target IDs 0 1 2 3 4 5 6 7, all LUNs
Scanning host 8 for SCSI target IDs 0 1 2 3 4 5 6 7, all LUNs
Scanning host 9 for SCSI target IDs 0 1 2 3 4 5 6 7, all LUNs
Scanning for device 9 0 0 0 ...
OLD: Host: scsi9 Channel: 00 Id: 00 Lun: 00
Vendor: PLDS Model: DVD-ROM DU-8D5LH Rev: 3D51
Type: CD-ROM ANSI SCSI revision: 05
Scanning host 10 for SCSI target IDs 0 1 2 3 4 5 6 7, all LUNs
Scanning for device 10 2 0 0 ...
OLD: Host: scsi10 Channel: 02 Id: 00 Lun: 00
Vendor: DELL Model: PERC H330 Mini Rev: 4.27
Type: Direct-Access ANSI SCSI revision: 05
Scanning host 11 for SCSI target IDs 0 1 2 3 4 5 6 7, all LUNs
Scanning for device 11 0 0 0 ...
NEW: Host: scsi11 Channel: 00 Id: 00 Lun: 00
Vendor: DELL Model: MD34xx Rev: 0825
Type: Direct-Access ANSI SCSI revision: 05
Scanning for device 11 0 0 31 ...
OLD: Host: scsi11 Channel: 00 Id: 00 Lun: 31
Vendor: DELL Model: Universal Xport Rev: 0825
Type: Direct-Access ANSI SCSI revision: 05
Scanning for device 11 0 0 1 ...
OLD: Host: scsi11 Channel: 00 Id: 00 Lun: 01
Vendor: DELL Model: MD34xx Rev: 0825
Type: Direct-Access ANSI SCSI revision: 05
Scanning for device 11 0 0 2 ...
NEW: Host: scsi11 Channel: 00 Id: 00 Lun: 02
Vendor: DELL Model: MD34xx Rev: 0825
Type: Direct-Access ANSI SCSI revision: 05
Scanning for device 11 0 0 3 ...
NEW: Host: scsi11 Channel: 00 Id: 00 Lun: 03
Vendor: DELL Model: MD34xx Rev: 0825
Type: Direct-Access ANSI SCSI revision: 05
Scanning for device 11 0 1 0 ...
NEW: Host: scsi11 Channel: 00 Id: 01 Lun: 00
Vendor: DELL Model: MD34xx Rev: 0825
Type: Direct-Access ANSI SCSI revision: 05
Scanning for device 11 0 1 31 ...
OLD: Host: scsi11 Channel: 00 Id: 01 Lun: 31
Vendor: DELL Model: Universal Xport Rev: 0825
Type: Direct-Access ANSI SCSI revision: 05
Scanning for device 11 0 1 1 ...
OLD: Host: scsi11 Channel: 00 Id: 01 Lun: 01
Vendor: DELL Model: MD34xx Rev: 0825
Type: Direct-Access ANSI SCSI revision: 05
Scanning for device 11 0 1 2 ...
NEW: Host: scsi11 Channel: 00 Id: 01 Lun: 02
Vendor: DELL Model: MD34xx Rev: 0825
Type: Direct-Access ANSI SCSI revision: 05
Scanning for device 11 0 1 3 ...
NEW: Host: scsi11 Channel: 00 Id: 01 Lun: 03
Vendor: DELL Model: MD34xx Rev: 0825
Type: Direct-Access ANSI SCSI revision: 05
6 new or changed device(s) found.
[11:0:0:0]
[11:0:0:2]
[11:0:0:3]
[11:0:1:0]
[11:0:1:2]
[11:0:1:3]
0 remapped or resized device(s) found.
0 device(s) removed.
[root@nfs02 ~]#
4. Display device mappings
[root@nfs02 ~]# multipath -ll
mpathe (3600a098000b6278f00000240599ffeea) dm-3 DELL,MD34xx
size=680G features='3 queue_if_no_path pg_init_retries 50' hwhandler='1 rdac' wp=rw
|-+- policy='round-robin 0' prio=14 status=active
| `- 11:0:0:3 sde 8:64 active ready running
`-+- policy='round-robin 0' prio=9 status=enabled
`- 11:0:1:3 sdi 8:128 active ready running
mpathd (3600a098000b5c042000003c1599fff37) dm-2 DELL,MD34xx
size=200G features='3 queue_if_no_path pg_init_retries 50' hwhandler='1 rdac' wp=rw
|-+- policy='round-robin 0' prio=14 status=active
| `- 11:0:1:2 sdh 8:112 active ready running
`-+- policy='round-robin 0' prio=9 status=enabled
`- 11:0:0:2 sdd 8:48 active ready running
mpathc (3600a098000b5c042000003bf599ffed2) dm-0 DELL,MD34xx
size=1.5T features='3 queue_if_no_path pg_init_retries 50' hwhandler='1 rdac' wp=rw
|-+- policy='round-robin 0' prio=14 status=active
| `- 11:0:1:0 sdf 8:80 active ready running
`-+- policy='round-robin 0' prio=9 status=enabled
`- 11:0:0:0 sdb 8:16 active ready running
mpathf (3600a098000b6278f0000023e599ffe83) dm-1 DELL,MD34xx
size=2.5T features='3 queue_if_no_path pg_init_retries 50' hwhandler='1 rdac' wp=rw
|-+- policy='round-robin 0' prio=14 status=active
| `- 11:0:0:1 sdc 8:32 active ready running
`-+- policy='round-robin 0' prio=9 status=enabled
`- 11:0:1:1 sdg 8:96 active ready running
5. Grab the image of nfs02and reboot nfs01 just to check if all is well.
[root@nfs01 ~]# multipath -ll
mpathe (3600a098000b6278f00000240599ffeea) dm-3 DELL,MD34xx
size=680G features='3 queue_if_no_path pg_init_retries 50' hwhandler='1 rdac' wp=rw
|-+- policy='round-robin 0' prio=14 status=active
| `- 11:0:0:3 sde 8:64 active ready running
`-+- policy='round-robin 0' prio=9 status=enabled
`- 11:0:1:3 sdi 8:128 active ready running
mpathd (3600a098000b5c042000003c1599fff37) dm-2 DELL,MD34xx
size=200G features='3 queue_if_no_path pg_init_retries 50' hwhandler='1 rdac' wp=rw
|-+- policy='round-robin 0' prio=14 status=active
| `- 11:0:1:2 sdh 8:112 active ready running
`-+- policy='round-robin 0' prio=9 status=enabled
`- 11:0:0:2 sdd 8:48 active ready running
mpathc (3600a098000b5c042000003bf599ffed2) dm-0 DELL,MD34xx
size=1.5T features='3 queue_if_no_path pg_init_retries 50' hwhandler='1 rdac' wp=rw
|-+- policy='round-robin 0' prio=14 status=active
| `- 11:0:1:0 sdf 8:80 active ready running
`-+- policy='round-robin 0' prio=9 status=enabled
`- 11:0:0:0 sdb 8:16 active ready running
mpathf (3600a098000b6278f0000023e599ffe83) dm-1 DELL,MD34xx
size=2.5T features='3 queue_if_no_path pg_init_retries 50' hwhandler='1 rdac' wp=rw
|-+- policy='round-robin 0' prio=14 status=active
| `- 11:0:0:1 sdc 8:32 active ready running
`-+- policy='round-robin 0' prio=9 status=enabled
`- 11:0:1:1 sdg 8:96 active ready running
[root@nfs01 ~]#
6. Storage Array Virtual Disks - Using Dell Storage Manager
Disk /dev/mapper/mpathc: 1.5 TB APPS
Disk /dev/mapper/mpathd: 200 GB PBS_SHARED
Disk /dev/mapper/mpathe: 680 GB CM_SHARED
Disk /dev/mapper/mpathf: 2.5 TB HOME
7. Install "parted" if not already installed
[root@nfs01 ~]# yum install parted parted-devel
8. Create partitions
[root@nfs01 ~]# parted /dev/mapper/mpathc
GNU Parted 2.1
Using /dev/mapper/mpathc
Welcome to GNU Parted! Type 'help' to view a list of commands.
(parted) mklabel GPT
(parted) mkpart primary xfs 0% 100%
(parted) print
Model: Linux device-mapper (multipath) (dm)
Disk /dev/mapper/mpathc: 1649GB
Sector size (logical/physical): 512B/512B
Partition Table: gpt
Number Start End Size File system Name Flags
1 1049kB 1649GB 1649GB primary
(parted) quit
Information: You may need to update /etc/fstab.
[root@nfs01 ~]#
9. Create a XFS File system from the partition
[root@nfs01 ~]# mkfs.xfs -L APPS /dev/mapper/mpathcp1
meta-data=/dev/mapper/mpathcp1 isize=256 agcount=4, agsize=100663168 blks
= sectsz=512 attr=2, projid32bit=0
data = bsize=4096 blocks=402652672, imaxpct=5
= sunit=0 swidth=0 blks
naming =version 2 bsize=4096 ascii-ci=0
log =internal log bsize=4096 blocks=196607, version=2
= sectsz=512 sunit=0 blks, lazy-count=1
realtime =none extsz=4096 blocks=0, rtextents=0
[root@nfs01 ~]#
10. Crate partitions and file system for all virtual disks [PBS_SHARED,CM_SHARED,HOME]
11. Mount on /etc/fstab using LABELS - this is just for testing
[root@nfs01 ~]# cat /etc/fstab |grep LABEL
LABEL=APPS /apps xfs defaults 0 0
LABEL=PBS_SHARED /pbsshared xfs defaults 0 0
LABEL=CM_SHARED /cmshared xfs defaults 0 0
LABEL=HOME /nfshome xfs defaults 0 0
[root@nfs01 ~]# mkdir /apps /pbsshared /cmshared /nfshome
[root@nfs01 ~]# mount -a
[root@nfs01 ~]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/sda1 20G 4.5G 16G 23% /
none 7.8G 0 7.8G 0% /dev/shm
/dev/sda6 891G 33M 891G 1% /local
/dev/sda3 2.0G 320M 1.7G 16% /tmp
/dev/sda2 6.0G 300M 5.7G 5% /var
master:/cm/shared 342G 68G 275G 20% /cm/shared
master:/home 342G 68G 275G 20% /home
/dev/mapper/mpathcp1 1.5T 33M 1.5T 1% /apps
/dev/mapper/mpathdp1 200G 33M 200G 1% /pbsshared
/dev/mapper/mpathep1 680G 33M 680G 1% /cmshared
/dev/mapper/mpathfp1 2.5T 33M 2.5T 1% /nfshome
[root@nfs01 ~]#
12. Add mount points to fsmounts on device nfs01 using bright
[bright1->device[nfs01]->fsmounts]% add /apps
[bright1->device*[nfs01*]->fsmounts*[/apps*]]% set device /dev/mapper/mpathcp1
[bright1->device*[nfs01*]->fsmounts*[/apps*]]% set filesystem xfs
[bright1->device*[nfs01*]->fsmounts*[/apps*]]% commit
[bright1->device[nfs01]->fsmounts[/apps]]% exit
[bright1->device[nfs01]->fsmounts]% add /pbsshared
[bright1->device*[nfs01*]->fsmounts*[/pbsshared*]]% set device /dev/mapper/mpathdp1
[bright1->device*[nfs01*]->fsmounts*[/pbsshared*]]% set filesystem xfs
[bright1->device*[nfs01*]->fsmounts*[/pbsshared*]]% commit
[bright1->device[nfs01]->fsmounts[/pbsshared]]% exit
[bright1->device[nfs01]->fsmounts]% add /cmshared
[bright1->device*[nfs01*]->fsmounts*[/cmshared*]]% set device /dev/mapper/mpathep1
[bright1->device*[nfs01*]->fsmounts*[/cmshared*]]% set filesystem xfs
[bright1->device*[nfs01*]->fsmounts*[/cmshared*]]% commit
[bright1->device[nfs01]->fsmounts[/cmshared]]% exit
[bright1->device[nfs01]->fsmounts]% add /nfshome
[bright1->device*[nfs01*]->fsmounts*[/nfshome*]]% set device /dev/mapper/mpathfp1
[bright1->device*[nfs01*]->fsmounts*[/nfshome*]]% set filesystem xfs
[bright1->device*[nfs01*]->fsmounts*[/nfshome*]]% commit
[bright1->device[nfs01]->fsmounts[/nfshome]]% exit
[bright1->device[nfs01]->fsmounts]% list
Device Mountpoint (key) Filesystem
--------------------------------- -------------------------------- ----------------
[nfs] none /dev/pts devpts
[nfs] none /proc proc
[nfs] none /sys sysfs
[nfs] none /dev/shm tmpfs
[nfs] $localnfsserver:/cm/shared /cm/shared nfs
[nfs] $localnfsserver:/home /home nfs
/dev/mapper/mpathcp1 /apps xfs
/dev/mapper/mpathdp1 /pbsshared xfs
/dev/mapper/mpathep1 /cmshared xfs
/dev/mapper/mpathfp1 /nfshome xfs
[bright1->device[nfs01]->fsmounts]%
13. The above step does the following (as soon as you commit
[root@nfs01 ~]# df -h|grep path
/dev/mapper/mpathcp1 1.5T 33M 1.5T 1% /apps
/dev/mapper/mpathdp1 200G 33M 200G 1% /pbsshared
/dev/mapper/mpathep1 680G 33M 680G 1% /cmshared
/dev/mapper/mpathfp1 2.5T 33M 2.5T 1% /nfshome
[root@nfs01 ~]# cat /etc/fstab |grep path
/dev/mapper/mpathcp1 /apps xfs defaults 0 0
/dev/mapper/mpathdp1 /pbsshared xfs defaults 0 0
/dev/mapper/mpathep1 /cmshared xfs defaults 0 0
/dev/mapper/mpathfp1 /nfshome xfs defaults 0 0
[root@nfs01 ~]#
14. Set up nfsexports through bright
[bright1->device*[nfs01*]->fsexports*[/apps*]]% set hosts 172.20.0.0/24
[bright1->device*[nfs01*]->fsexports*[/apps*]]% set async yes
[bright1->device*[nfs01*]->fsexports*[/apps*]]% set rootsquash no
[bright1->device*[nfs01*]->fsexports*[/apps*]]% set write yes
[bright1->device*[nfs01*]->fsexports*[/apps*]]% append hosts " 172.18.0.0/24"
[bright1->device*[nfs01*]->fsexports*[/apps*]]% commit
[bright1->device[nfs01]->fsexports[/cmshared]]% exit
[bright1->device[nfs01]->fsexports]% clone /apps /pbsshared
[bright1->device[nfs01]->fsexports[/pbsshared]]% set path /pbsshared
[bright1->device*[nfs01*]->fsexports*[/pbsshared*]]% commit
[bright1->device[nfs01]->fsexports]% clone /apps /cmshared
[bright1->device[nfs01]->fsexports[/cmshared]]% set path /cmshared
[bright1->device*[nfs01*]->fsexports*[/cmshared*]]% commit
[bright1->device[nfs01]->fsexports]% clone /apps /nfshome
[bright1->device[nfs01]->fsexports[/nfshome]]% set path /nfshome
[bright1->device*[nfs01*]->fsexports*[/nfshome*]]% commit
Produces ...
[root@nfs01 ~]# cat /etc/exports
# This section of this file was automatically generated by cmd. Do not edit manually!
# BEGIN AUTOGENERATED SECTION -- DO NOT REMOVE
/apps 172.20.0.0/24(rw,no_root_squash,async) 172.18.0.0/24(rw,no_root_squash,async)
/pbsshared 172.20.0.0/24(rw,no_root_squash,async) 172.18.0.0/24(rw,no_root_squash,async)
/cmshared 172.20.0.0/24(rw,no_root_squash,async) 172.18.0.0/24(rw,no_root_squash,async)
/nfshome 172.20.0.0/24(rw,no_root_squash,async) 172.18.0.0/24(rw,no_root_squash,async)
# END AUTOGENERATED SECTION -- DO NOT REMOVE
[root@nfs01 ~]#
15. Mount the fsmounts point [on all node categories]
[bright1->category]% foreach compute iml login scheduler (fsmounts; add /home; set device nfs01.ib.cluster:/nfshome; set filesystem nfs; commit)
[bright1->category]% foreach compute iml login scheduler (fsmounts; add /cm/shared; set device nfs01.ib.cluster:/cmshared; set filesystem nfs; commit)
[bright1->category]% foreach compute iml login scheduler (fsmounts; add /apps; set device nfs01.ib.cluster:/apps; set filesystem nfs; commit)
[bright1->category]% foreach scheduler (fsmounts; add /pbsshared; set device nfs01.ib.cluster:/pbsshared; set filesystem nfs; commit)
[bright1->category]%