Dell MD3060 JBOD

calendar May 11, 2018 · 6 min read · DELLMD3060JBOD UBUNTU  ·
Share on: linkedin copy

Here is what it looks like to have a Dell MD3060 JBOD Chassis attached to a an Ubuntu box

 1root@sata01:~# lsscsi
 2[0:0:0:0] cd/dvd PLDS DVD+-RW DS-8A3S HD51 /dev/sr0
 3[2:2:0:0] disk DELL PERC H700 2.10 /dev/sda
 4[3:0:0:0] disk iDRAC LCDRIVE 0323 /dev/sdaf
 5[4:0:0:0] cd/dvd iDRAC Virtual CD 0323 /dev/sr1
 6[4:0:0:1] disk iDRAC Virtual Floppy 0323 /dev/sdag
 7[5:0:31:0] enclosu DELL MD3060e 039D -
 8[5:0:32:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdah
 9[5:0:33:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdai
10[5:0:34:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdaj
11[5:0:35:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdak
12[5:0:36:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdal
13[5:0:37:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdam
14[5:0:38:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdan
15[5:0:39:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdao
16[5:0:40:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdap
17[5:0:41:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdb
18[5:0:42:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdc
19[5:0:43:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdd
20[5:0:44:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sde
21[5:0:45:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdf
22[5:0:46:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdg
23[5:0:47:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdaq
24[5:0:48:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdh
25[5:0:49:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdi
26[5:0:50:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdj
27[5:0:51:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdk
28[5:0:52:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdl
29[5:0:53:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdm
30[5:0:54:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdar
31[5:0:55:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdn
32[5:0:56:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdo
33[5:0:57:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdp
34[5:0:58:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdq
35[5:0:59:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdr
36[5:0:60:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sds
37[5:0:61:0] disk TOSHIBA MG04SCA40ENY EG02 /dev/sdt
To list only the SATA disks filter on " t(SSP)"
1root@sata01:~# for i in /dev/bsg/expand* ; do smp_discover $i | grep " t(SSP)" | wc -l; done
212
312
40
50
60
76

 1root@sata01:~# for i in /dev/bsg/expand* ; do smp_discover $i; done
 2 phy 0:D:attached:[500003983832bef2:00 t(SSP)] 6 Gbps
 3 phy 1:D:attached:[50000398383814f2:00 t(SSP)] 6 Gbps
 4 phy 2:D:attached:[50000398382a56fa:00 t(SSP)] 6 Gbps
 5 phy 3:D:attached:[500003983832bf02:00 t(SSP)] 6 Gbps
 6 phy 4:D:attached:[50000398382a573a:00 t(SSP)] 6 Gbps
 7 phy 5:D:attached:[500003983833e472:00 t(SSP)] 6 Gbps
 8 phy 6:D:attached:[500003983832bece:00 t(SSP)] 6 Gbps
 9 phy 7:D:attached:[5000039848013812:00 t(SSP)] 6 Gbps
10 phy 8:D:attached:[5000039848013816:00 t(SSP)] 6 Gbps
11 phy 9:D:attached:[5000039848013916:00 t(SSP)] 6 Gbps
12 phy 10:D:attached:[50000398480137f6:00 t(SSP)] 6 Gbps
13 phy 11:D:attached:[500003984801380a:00 t(SSP)] 6 Gbps
14 phy 12:S:attached:[50080e548c83503f:12 exp t(SMP)] 6 Gbps
15 phy 13:S:attached:[50080e548c83503f:13 exp t(SMP)] 6 Gbps
16 phy 14:S:attached:[50080e548c83503f:14 exp t(SMP)] 6 Gbps
17 phy 15:S:attached:[50080e548c83503f:15 exp t(SMP)] 6 Gbps
18 phy 16:D:attached:[50080e54a55db03d:00 V i(SSP+SMP) t(SSP)] 6 Gbps
19 phy 0:D:attached:[500003984801382a:00 t(SSP)] 6 Gbps
20 phy 1:D:attached:[5000039848013722:00 t(SSP)] 6 Gbps
21 phy 2:D:attached:[500003983833c74a:00 t(SSP)] 6 Gbps
22 phy 3:D:attached:[500003983838a4aa:00 t(SSP)] 6 Gbps
23 phy 4:D:attached:[50000398382a572a:00 t(SSP)] 6 Gbps
24 phy 5:D:attached:[500003983838a4a2:00 t(SSP)] 6 Gbps
25 phy 6:D:attached:[500003984801381e:00 t(SSP)] 6 Gbps
26 phy 7:D:attached:[500003984801381a:00 t(SSP)] 6 Gbps
27 phy 8:D:attached:[500003983830e1ca:00 t(SSP)] 6 Gbps
28 phy 9:D:attached:[50000398480137de:00 t(SSP)] 6 Gbps
29 phy 10:D:attached:[50000398382a56ce:00 t(SSP)] 6 Gbps
30 phy 11:D:attached:[500003983830e1d2:00 t(SSP)] 6 Gbps
31 phy 12:S:attached:[50080e548c83503f:16 exp t(SMP)] 6 Gbps
32 phy 13:S:attached:[50080e548c83503f:17 exp t(SMP)] 6 Gbps
33 phy 14:S:attached:[50080e548c83503f:18 exp t(SMP)] 6 Gbps
34 phy 15:S:attached:[50080e548c83503f:19 exp t(SMP)] 6 Gbps
35 phy 16:D:attached:[50080e54a54ef03d:00 V i(SSP+SMP) t(SSP)] 6 Gbps
36 phy 0:U:attached:[50080e54a567703f:12 exp t(SMP)] 6 Gbps
37 phy 1:U:attached:[50080e54a567703f:13 exp t(SMP)] 6 Gbps
38 phy 2:U:attached:[50080e54a567703f:14 exp t(SMP)] 6 Gbps
39 phy 3:U:attached:[50080e54a567703f:15 exp t(SMP)] 6 Gbps
40 phy 4:U:attached:[50080e54a57b703f:12 exp t(SMP)] 6 Gbps
41 phy 5:U:attached:[50080e54a57b703f:13 exp t(SMP)] 6 Gbps
42 phy 6:U:attached:[50080e54a57b703f:14 exp t(SMP)] 6 Gbps
43 phy 7:U:attached:[50080e54a57b703f:15 exp t(SMP)] 6 Gbps
44 phy 8:U:attached:[50080e54a56b303f:12 exp t(SMP)] 6 Gbps
45 phy 9:U:attached:[50080e54a56b303f:13 exp t(SMP)] 6 Gbps
46 phy 10:U:attached:[50080e54a56b303f:14 exp t(SMP)] 6 Gbps
47 phy 11:U:attached:[50080e54a56b303f:15 exp t(SMP)] 6 Gbps
48 phy 12:U:attached:[50080e54a55db03f:12 exp t(SMP)] 6 Gbps
49 phy 13:U:attached:[50080e54a55db03f:13 exp t(SMP)] 6 Gbps
50 phy 14:U:attached:[50080e54a55db03f:14 exp t(SMP)] 6 Gbps
51 phy 15:U:attached:[50080e54a55db03f:15 exp t(SMP)] 6 Gbps
52 phy 16:U:attached:[50080e54a54ef03f:12 exp t(SMP)] 6 Gbps
53 phy 17:U:attached:[50080e54a54ef03f:13 exp t(SMP)] 6 Gbps
54 phy 18:U:attached:[50080e54a54ef03f:14 exp t(SMP)] 6 Gbps
55 phy 19:U:attached:[50080e54a54ef03f:15 exp t(SMP)] 6 Gbps
56 phy 20:U:attached:[51866da05f99eb00:07 i(SSP+STP+SMP)] 6 Gbps
57 phy 21:U:attached:[51866da05f99eb00:06 i(SSP+STP+SMP)] 6 Gbps
58 phy 22:U:attached:[51866da05f99eb00:05 i(SSP+STP+SMP)] 6 Gbps
59 phy 23:U:attached:[51866da05f99eb00:04 i(SSP+STP+SMP)] 6 Gbps
60 phy 32:D:attached:[50080e548c83503d:00 V i(SSP+SMP) t(SSP)] 6 Gbps
61 phy 12:S:attached:[50080e548c83503f:00 exp t(SMP)] 6 Gbps
62 phy 13:S:attached:[50080e548c83503f:01 exp t(SMP)] 6 Gbps
63 phy 14:S:attached:[50080e548c83503f:02 exp t(SMP)] 6 Gbps
64 phy 15:S:attached:[50080e548c83503f:03 exp t(SMP)] 6 Gbps
65 phy 16:D:attached:[50080e54a567703d:00 V i(SSP+SMP) t(SSP)] 6 Gbps
66 phy 12:S:attached:[50080e548c83503f:04 exp t(SMP)] 6 Gbps
67 phy 13:S:attached:[50080e548c83503f:05 exp t(SMP)] 6 Gbps
68 phy 14:S:attached:[50080e548c83503f:06 exp t(SMP)] 6 Gbps
69 phy 15:S:attached:[50080e548c83503f:07 exp t(SMP)] 6 Gbps
70 phy 16:D:attached:[50080e54a57b703d:00 V i(SSP+SMP) t(SSP)] 6 Gbps
71 phy 0:D:attached:[500003983832be2e:00 t(SSP)] 6 Gbps
72 phy 1:D:attached:[5000039848013936:00 t(SSP)] 6 Gbps
73 phy 2:D:attached:[500003983832be8e:00 t(SSP)] 6 Gbps
74 phy 3:D:attached:[500003983830e1da:00 t(SSP)] 6 Gbps
75 phy 4:D:attached:[500003984801391e:00 t(SSP)] 6 Gbps
76 phy 5:D:attached:[500003983832be76:00 t(SSP)] 6 Gbps
77 phy 12:S:attached:[50080e548c83503f:08 exp t(SMP)] 6 Gbps
78 phy 13:S:attached:[50080e548c83503f:09 exp t(SMP)] 6 Gbps
79 phy 14:S:attached:[50080e548c83503f:10 exp t(SMP)] 6 Gbps
80 phy 15:S:attached:[50080e548c83503f:11 exp t(SMP)] 6 Gbps
81 phy 16:D:attached:[50080e54a56b303d:00 V i(SSP+SMP) t(SSP)] 6 Gbps
What info can we get form Udev
1root@sata01:~# udevadm info -q path -n /dev/sdar
2/devices/pci0000:00/0000:00:07.0/0000:04:00.0/host5/port-5:1/expander-5:6/port-5:6:5/expander-5:11/port-5:11:4/end_device-5:11:4/target5:0:54/5:0:54:0/block/sdar
3root@edpk-sata01:~# udevadm info -a -p /devices/pci0000:00/0000:00:07.0/0000:04:00.0/host5/port-5:1/expander-5:6/port-5:6:5/expander-5:11/port-5:11:4/end_device-5:11:4/target5:0:54/5:0:54:0/block/sdar

Udevadm info starts with the device specified by the devpath and then walks up the chain of parent devices. It prints for every device found, all possible attributes in the udev rules key format. A rule to match, can be composed by the attributes of the device and the attributes from one single parent device.

  1looking at device '/devices/pci0000:00/0000:00:07.0/0000:04:00.0/host5/port-5:1/expander-5:6/port-5:6:5/expander-5:11/port-5:11:4/end_device-5:11:4/target5:0:54/5:0:54:0/block/sdar'`:
  2
  3
  4 KERNEL=="sdar"
  5 SUBSYSTEM=="block"
  6 DRIVER==""
  7 ATTR{alignment_offset}=="0"
  8 ATTR{capability}=="50"
  9 ATTR{discard_alignment}=="0"
 10 ATTR{events}==""
 11 ATTR{events_async}==""
 12 ATTR{events_poll_msecs}=="-1"
 13 ATTR{ext_range}=="256"
 14 ATTR{inflight}==" 0 0"
 15 ATTR{range}=="16"
 16 ATTR{removable}=="0"
 17 ATTR{ro}=="0"
 18 ATTR{size}=="7814037168"
 19 ATTR{stat}==" 32963 1619 7699048 104236 7501 779 3505528 328772 0 240440 433100"
 20
 21looking at parent device '/devices/pci0000:00/0000:00:07.0/0000:04:00.0/host5/port-5:1/expander-5:6/port-5:6:5/expander-5:11/port-5:11:4/end_device-5:11:4/target5:0:54/5:0:54:0':
 22
 23 KERNELS=="5:0:54:0"
 24 SUBSYSTEMS=="scsi"
 25 DRIVERS=="sd"
 26 ATTRS{device_blocked}=="0"
 27 ATTRS{device_busy}=="0"
 28 ATTRS{dh_state}=="detached"
 29 ATTRS{eh_timeout}=="10"
 30 ATTRS{evt_capacity_change_reported}=="0"
 31 ATTRS{evt_inquiry_change_reported}=="0"
 32 ATTRS{evt_lun_change_reported}=="0"
 33 ATTRS{evt_media_change}=="0"
 34 ATTRS{evt_mode_parameter_change_reported}=="0"
 35 ATTRS{evt_soft_threshold_reached}=="0"
 36 ATTRS{inquiry}==""
 37 ATTRS{iocounterbits}=="32"
 38 ATTRS{iodone_cnt}=="0xa12e"
 39 ATTRS{ioerr_cnt}=="0x0"
 40 ATTRS{iorequest_cnt}=="0xa12e"
 41 ATTRS{model}=="MG04SCA40ENY "
 42 ATTRS{queue_depth}=="254"
 43 ATTRS{queue_ramp_up_period}=="120000"
 44 ATTRS{queue_type}=="simple"
 45 ATTRS{rev}=="EG02"
 46 ATTRS{sas_address}=="0x50000398382a572a"
 47 ATTRS{sas_device_handle}=="0x0026"
 48 ATTRS{scsi_level}=="7"
 49 ATTRS{state}=="running"
 50 ATTRS{timeout}=="30"
 51 ATTRS{type}=="0"
 52 ATTRS{vendor}=="TOSHIBA "
 53 ATTRS{vpd_pg80}==""
 54 ATTRS{vpd_pg83}==""
 55
 56
 57looking at parent device '/devices/pci0000:00/0000:00:07.0/0000:04:00.0/host5/port-5:1/expander-5:6/port-5:6:5/expander-5:11/port-5:11:4/end_device-5:11:4/target5:0:54':
 58
 59 KERNELS=="target5:0:54"
 60 SUBSYSTEMS=="scsi"
 61 DRIVERS==""
 62
 63
 64looking at parent device `/devices/pci0000:00/0000:00:07.0/0000:04:00.0/host5/port-5:1/expander-5:6/port-5:6:5/expander-5:11/port-5:11:4/end_device-5:11:4':
 65
 66 KERNELS=="end_device-5:11:4"
 67 SUBSYSTEMS==""
 68 DRIVERS==""
 69
 70looking at parent device '/devices/pci0000:00/0000:00:07.0/0000:04:00.0/host5/port-5:1/expander-5:6/port-5:6:5/expander-5:11/port-5:11:4':
 71
 72 KERNELS=="port-5:11:4"
 73 SUBSYSTEMS==""
 74 DRIVERS==""
 75
 76looking at parent device '/devices/pci0000:00/0000:00:07.0/0000:04:00.0/host5/port-5:1/expander-5:6/port-5:6:5/expander-5:11`:
 77
 78 KERNELS=="expander-5:11"
 79 SUBSYSTEMS==""
 80 DRIVERS==""
 81
 82looking at parent device '/devices/pci0000:00/0000:00:07.0/0000:04:00.0/host5/port-5:1/expander-5:6/port-5:6:5':
 83 KERNELS=="port-5:6:5"
 84 SUBSYSTEMS==""
 85 DRIVERS==""
 86
 87looking at parent device '/devices/pci0000:00/0000:00:07.0/0000:04:00.0/host5/port-5:1/expander-5:6':
 88 KERNELS=="expander-5:6"
 89 SUBSYSTEMS==""
 90 DRIVERS==""
 91
 92looking at parent device '/devices/pci0000:00/0000:00:07.0/0000:04:00.0/host5/port-5:1':
 93 KERNELS=="port-5:1"
 94 SUBSYSTEMS==""
 95 DRIVERS==""
 96
 97looking at parent device '/devices/pci0000:00/0000:00:07.0/0000:04:00.0/host5':
 98 KERNELS=="host5"
 99 SUBSYSTEMS=="scsi"
100 DRIVERS==""
101
102looking at parent device '/devices/pci0000:00/0000:00:07.0/0000:04:00.0':
103 KERNELS=="0000:04:00.0"
104 SUBSYSTEMS=="pci"
105 DRIVERS=="mpt3sas"
106 ATTRS{broken_parity_status}=="0"
107 ATTRS{class}=="0x010700"
108 ATTRS{config}==""
109 ATTRS{consistent_dma_mask_bits}=="64"
110 ATTRS{d3cold_allowed}=="1"
111 ATTRS{device}=="0x0072"
112 ATTRS{dma_mask_bits}=="64"
113 ATTRS{driver_override}=="(null)"
114 ATTRS{enable}=="1"
115 ATTRS{irq}=="35"
116 ATTRS{local_cpulist}=="0-23"
117 ATTRS{local_cpus}=="00ffffff"
118 ATTRS{msi_bus}=="1"
119 ATTRS{numa_node}=="-1"
120 ATTRS{subsystem_device}=="0x1f1c"
121 ATTRS{subsystem_vendor}=="0x1028"
122 ATTRS{vendor}=="0x1000"
123 ATTRS{vpd}==""
124
125looking at parent device '/devices/pci0000:00/0000:00:07.0':
126 KERNELS=="0000:00:07.0"
127 SUBSYSTEMS=="pci"
128 DRIVERS=="pcieport"
129 ATTRS{broken_parity_status}=="0"
130 ATTRS{class}=="0x060400"
131 ATTRS{consistent_dma_mask_bits}=="32"
132 ATTRS{d3cold_allowed}=="0"
133 ATTRS{device}=="0x340e"
134 ATTRS{dma_mask_bits}=="32"
135 ATTRS{driver_override}=="(null)"
136 ATTRS{enable}=="1"
137 ATTRS{irq}=="27"
138 ATTRS{local_cpulist}=="0-23"
139 ATTRS{local_cpus}=="00ffffff"
140 ATTRS{msi_bus}=="1"
141 ATTRS{numa_node}=="-1"
142 ATTRS{subsystem_device}=="0x0236"
143 ATTRS{subsystem_vendor}=="0x1028"
144 ATTRS{vendor}=="0x8086"
145
146looking at parent device '/devices/pci0000:00':
147 KERNELS=="pci0000:00"
148 SUBSYSTEMS==""
149 DRIVERS==""

Seems to crash the machine 10% of the time it's run

1udevadm info -a -p $(udevadm info -q path -n /dev/sdl) | grep sas