ill play as well. More interested in IO, but this is still valuable. Similar reads and writes from Sas3 rust drives and sata SSD drives. IO is where the ssd pool shines.
2 pools.
axle = 10x sas3 6 tb raidz2
root@rotor[/mnt/axle]# dd if=/dev/zero of=/mnt/axle/tmp.dat bs=2048k count=50k
51200+0 records in
51200+0 records out
107374182400 bytes transferred in 34.003642 secs (3157725984 bytes/sec)
3157.73 MB/s write
root@rotor[/mnt/axle]# dd if=/mnt/axle/tmp.dat of=/dev/null bs=2048k count=50k
51200+0 records in
51200+0 records out
107374182400 bytes transferred in 14.426246 secs (7442974789 bytes/sec)
7442.97 MB/s read
axle-ssd = 4 x 1 tb sata ssd raidz1
root@rotor[/mnt/axle]# dd if=/dev/zero of=/mnt/axle-ssd/tmp.dat bs=2048k count=50k
51200+0 records in
51200+0 records out
107374182400 bytes transferred in 32.803882 secs (3273215689 bytes/sec)
3273.22 MB/s Write
root@rotor[/mnt/axle]# dd if=/mnt/axle-ssd/tmp.dat of=/dev/null bs=2048k count=50k
51200+0 records in
51200+0 records out
107374182400 bytes transferred in 14.590442 secs (7359214025 bytes/sec)
7359.21 MB/s Read
Test write IOPS by performing random writes, using an I/O block size of 4 KB and an I/O depth of at least 64:
fio --name=write_iops --directory=/mnt/axle/fio --size=10G \--time_based --runtime=60s --ramp_time=2s --direct=1 \--verify=0 --bs=4K --iodepth=64 --rw=randwrite --group_reporting=1
Axle
root@rotor[/mnt/axle]# fio --name=write_iops --directory=/mnt/axle/fio --size=10G \--time_based --runtime=60s --ramp_time=2s --direct=1 \--verify=0 --bs=4K --iodepth=64 --rw=randwrite --group_reporting=1
write_iops: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=psync, iodepth=64
fio-3.27
Starting 1 process
write_iops: Laying out IO file (1 file / 10240MiB)
Jobs: 1 (f=1): [w(1)][100.0%][w=16.7MiB/s][w=4285 IOPS][eta 00m:00s]
write_iops: (groupid=0, jobs=1): err= 0: pid=6336: Sun Sep 25 22:30:27 2022
write: IOPS=4277, BW=16.7MiB/s (17.5MB/s)(1003MiB/60001msec); 0 zone resets
clat (usec): min=13, max=121178, avg=230.83, stdev=674.52
lat (usec): min=13, max=121178, avg=231.20, stdev=675.02
clat percentiles (usec):
| 1.00th=[ 35], 5.00th=[ 58], 10.00th=[ 68], 20.00th=[ 73],
| 30.00th=[ 93], 40.00th=[ 113], 50.00th=[ 130], 60.00th=[ 153],
| 70.00th=[ 167], 80.00th=[ 188], 90.00th=[ 375], 95.00th=[ 840],
| 99.00th=[ 1991], 99.50th=[ 2737], 99.90th=[ 5276], 99.95th=[ 7046],
| 99.99th=[20055]
bw ( KiB/s): min= 7792, max=39497, per=100.00%, avg=17148.46, stdev=3609.30, samples=119
iops : min= 1948, max= 9874, avg=4286.85, stdev=902.30, samples=119
lat (usec) : 20=0.23%, 50=3.67%, 100=30.16%, 250=52.36%, 500=5.57%
lat (usec) : 750=2.44%, 1000=1.46%
lat (msec) : 2=3.12%, 4=0.80%, 10=0.17%, 20=0.02%, 50=0.01%
lat (msec) : 100=0.01%, 250=0.01%
cpu : usr=0.98%, sys=37.48%, ctx=308730, majf=0, minf=1
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,256653,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
WRITE: bw=16.7MiB/s (17.5MB/s), 16.7MiB/s-16.7MiB/s (17.5MB/s-17.5MB/s), io=1003MiB (1051MB), run=60001-60001msec
Axle-ssd
root@rotor[/mnt/axle-ssd/fio]# fio --name=write_iops --directory=/mnt/axle-ssd/fio --size=10G \--time_based --runtime=60
s --ramp_time=2s --direct=1 \--verify=0 --bs=4K --iodepth=64 --rw=randwrite --group_reporting=1
write_iops: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=psync, iodepth=64
fio-3.27
Starting 1 process
write_iops: Laying out IO file (1 file / 10240MiB)
Jobs: 1 (f=1): [w(1)][100.0%][w=31.9MiB/s][w=8166 IOPS][eta 00m:00s]
write_iops: (groupid=0, jobs=1): err= 0: pid=6708: Sun Sep 25 22:34:38 2022
write: IOPS=8415, BW=32.9MiB/s (34.5MB/s)(1973MiB/60001msec); 0 zone resets
clat (usec): min=11, max=62111, avg=116.42, stdev=344.10
lat (usec): min=11, max=62111, avg=116.70, stdev=344.38
clat percentiles (usec):
| 1.00th=[ 14], 5.00th=[ 17], 10.00th=[ 36], 20.00th=[ 51],
| 30.00th=[ 56], 40.00th=[ 60], 50.00th=[ 62], 60.00th=[ 67],
| 70.00th=[ 83], 80.00th=[ 94], 90.00th=[ 133], 95.00th=[ 347],
| 99.00th=[ 1336], 99.50th=[ 1827], 99.90th=[ 3032], 99.95th=[ 3654],
| 99.99th=[ 6063]
bw ( KiB/s): min=16262, max=48335, per=99.94%, avg=33642.94, stdev=8326.06, samples=119
iops : min= 4065, max=12083, avg=8410.44, stdev=2081.51, samples=119
lat (usec) : 20=5.26%, 50=13.67%, 100=65.45%, 250=9.57%, 500=2.34%
lat (usec) : 750=1.30%, 1000=0.79%
lat (msec) : 2=1.24%, 4=0.36%, 10=0.03%, 20=0.01%, 50=0.01%
lat (msec) : 100=0.01%
cpu : usr=1.05%, sys=54.44%, ctx=196821, majf=0, minf=1
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=0,504964,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
WRITE: bw=32.9MiB/s (34.5MB/s), 32.9MiB/s-32.9MiB/s (34.5MB/s-34.5MB/s), io=1973MiB (2068MB), run=60001-60001msec
SSd pool double the random write IOPS
Test read IOPS by performing random reads, using an I/O block size of 4 KB and an I/O depth of at least 64
Axle
fio --name=read_iops --directory=/mnt/axle/fio --size=10G \
--time_based --runtime=60s --ramp_time=2s --direct=1 \
--verify=0 --bs=4K --iodepth=64 --rw=randread --group_reporting=1
read_iops: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=psync, iodepth=64
fio-3.27
Starting 1 process
read_iops: Laying out IO file (1 file / 10240MiB)
Jobs: 1 (f=1): [r(1)][100.0%][r=154MiB/s][r=39.4k IOPS][eta 00m:00s]
read_iops: (groupid=0, jobs=1): err= 0: pid=7036: Sun Sep 25 22:38:24 2022
read: IOPS=38.6k, BW=151MiB/s (158MB/s)(9052MiB/60001msec)
clat (usec): min=3, max=12893, avg=25.22, stdev=29.26
lat (usec): min=3, max=12893, avg=25.28, stdev=29.30
clat percentiles (usec):
| 1.00th=[ 7], 5.00th=[ 7], 10.00th=[ 24], 20.00th=[ 25],
| 30.00th=[ 25], 40.00th=[ 26], 50.00th=[ 26], 60.00th=[ 26],
| 70.00th=[ 27], 80.00th=[ 27], 90.00th=[ 29], 95.00th=[ 31],
| 99.00th=[ 39], 99.50th=[ 45], 99.90th=[ 186], 99.95th=[ 490],
| 99.99th=[ 1270]
bw ( KiB/s): min=98406, max=174432, per=100.00%, avg=154628.82, stdev=13670.63, samples=119
iops : min=24601, max=43608, avg=38656.93, stdev=3417.68, samples=119
lat (usec) : 4=0.01%, 10=8.65%, 20=0.33%, 50=90.63%, 100=0.23%
lat (usec) : 250=0.08%, 500=0.03%, 750=0.02%, 1000=0.01%
lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%
cpu : usr=3.16%, sys=94.23%, ctx=19021, majf=0, minf=1
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=2317219,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
READ: bw=151MiB/s (158MB/s), 151MiB/s-151MiB/s (158MB/s-158MB/s), io=9052MiB (9491MB), run=60001-60001msec
axle ssd
root@rotor[/mnt/axle-ssd/fio]# fio --name=read_iops --directory=/mnt/axle-ssd/fio --size=10G \
--time_based --runtime=60s --ramp_time=2s --direct=1 \
--verify=0 --bs=4K --iodepth=64 --rw=randread --group_reporting=1
read_iops: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=psync, iodepth=64
fio-3.27
Starting 1 process
read_iops: Laying out IO file (1 file / 10240MiB)
Jobs: 1 (f=1): [r(1)][100.0%][r=163MiB/s][r=41.8k IOPS][eta 00m:00s]
read_iops: (groupid=0, jobs=1): err= 0: pid=7333: Sun Sep 25 22:41:39 2022
read: IOPS=37.9k, BW=148MiB/s (155MB/s)(8881MiB/60001msec)
clat (usec): min=5, max=11624, avg=25.73, stdev=20.90
lat (usec): min=5, max=11624, avg=25.78, stdev=20.91
clat percentiles (usec):
| 1.00th=[ 8], 5.00th=[ 8], 10.00th=[ 8], 20.00th=[ 27],
| 30.00th=[ 27], 40.00th=[ 28], 50.00th=[ 28], 60.00th=[ 28],
| 70.00th=[ 28], 80.00th=[ 29], 90.00th=[ 30], 95.00th=[ 31],
| 99.00th=[ 37], 99.50th=[ 44], 99.90th=[ 81], 99.95th=[ 153],
| 99.99th=[ 881]
bw ( KiB/s): min=104311, max=172071, per=100.00%, avg=151720.29, stdev=9345.02, samples=119
iops : min=26077, max=43017, avg=37929.78, stdev=2336.29, samples=119
lat (usec) : 10=11.69%, 20=0.37%, 50=87.67%, 100=0.19%, 250=0.04%
lat (usec) : 500=0.02%, 750=0.01%, 1000=0.01%
lat (msec) : 2=0.01%, 4=0.01%, 10=0.01%, 20=0.01%
cpu : usr=3.18%, sys=95.56%, ctx=12585, majf=0, minf=1
IO depths : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
issued rwts: total=2273635,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
READ: bw=148MiB/s (155MB/s), 148MiB/s-148MiB/s (155MB/s-155MB/s), io=8881MiB (9313MB), run=60001-60001msec
Random reads Iops are similar for both pools.