FIO磁盘压测工具

fio，又称为Flexible IO Tester，是Jens Axboe编写的应用程序。Jens是Linux Kernel中block IO subsystem的维护者。FIO作为网络文件系统和磁盘的压测工具，多用于机型验证和文件系统的性能对比，本工具可以自动的分发fio命令到集群的机器列表，搜集小文件的iops和大文件的吞吐。

参数说明

rw=[mode]
rwmixwrite=30 在混合读写的模式下，写占30%

mode
read 顺序读
write 顺序写
readwrite 顺序混合读写
randwrite 随机写
randread 随机读
randrw 随机混合读写

fio -numjobs=1 -iodepth=1 -direct=1 -ioengine=libaio -sync=1 -rw=randread -bs=4k -size=100k -time_based -runtime=60 -name=Fio -directory=/mnt/test2 

结果说明

io=执行了多少M的IO
bw=平均IO带宽
iops=IOPS
runt=线程运行时间
slat=Submission Latency。盘需要多久将IO提交到kernel做处理。
clat=Completion Latency。命令提交到kernel到IO做完之间的时间，不包括submission latency
lat=Total Latency，包括 fio 从创建这个 I/O 单元到完成的总的时间。
bw=带宽
cpu=利用率
IO depths=io队列。用来控制同一时刻发送给OS多少个IO
IO submit=单个IO提交要提交的IO数
IO complete=Like the above submit number, but for completions instead.
IO issued=The number of read/write requests issued, and how many of them were short.
IO latencies=IO完延迟的分布

io=总共执行了多少size的IO
aggrb=group总带宽
minb=最小.平均带宽.
maxb=最大平均带宽.
mint=group中线程的最短运行时间.
maxt=group中线程的最长运行时间.

ios=所有group总共执行的IO数.
merge=总共发生的IO合并数.
ticks=Number of ticks we kept the disk busy.
io_queue=花费在队列上的总共时间.
util=磁盘利用率

 

Fio: (g=0): rw=randread, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=1
fio-3.7
Starting 1 process
Jobs: 1 (f=1): [r(1)][100.0%][r=4044KiB/s,w=0KiB/s][r=1011,w=0 IOPS][eta 00m:00s]
Fio: (groupid=0, jobs=1): err= 0: pid=22131: Tue Feb  4 21:08:19 2020
   read: IOPS=1042, BW=4171KiB/s (4271kB/s)(244MiB/60001msec)
    slat (nsec): min=5266, max=96336, avg=9332.63, stdev=1772.18
    clat (usec): min=678, max=24547, avg=945.78, stdev=147.56
     lat (usec): min=688, max=24558, avg=955.93, stdev=147.64
    clat percentiles (usec):
     |  1.00th=[  783],  5.00th=[  824], 10.00th=[  857], 20.00th=[  881],
     | 30.00th=[  906], 40.00th=[  922], 50.00th=[  947], 60.00th=[  963],
     | 70.00th=[  979], 80.00th=[ 1004], 90.00th=[ 1037], 95.00th=[ 1057],
     | 99.00th=[ 1139], 99.50th=[ 1172], 99.90th=[ 1418], 99.95th=[ 2606],
     | 99.99th=[ 5800]
   bw (  KiB/s): min= 3840, max= 4432, per=100.00%, avg=4171.03, stdev=118.37, samples=119
   iops        : min=  960, max= 1108, avg=1042.71, stdev=29.57, samples=119
  lat (usec)   : 750=0.22%, 1000=79.80%
  lat (msec)   : 2=19.92%, 4=0.04%, 10=0.03%, 50=0.01%
  cpu          : usr=0.64%, sys=2.22%, ctx=62565, majf=0, minf=32
  IO depths    : 1=100.0%, 2=0.0%, 4=0.0%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     issued rwts: total=62562,0,0,0 short=0,0,0,0 dropped=0,0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=1

Run status group 0 (all jobs):
   READ: bw=4171KiB/s (4271kB/s), 4171KiB/s-4171KiB/s (4271kB/s-4271kB/s), io=244MiB (256MB), run=60001-60001msec

 

示例

hostfile: 配置机器的list, #注释的机器会自动过滤掉

#!/bin/bash
################################################
#Author : Li Qingbing
#Date   : 2019-08-01
################################################
########## Config ###############
hostfile="hosts"
oplist=('randread' 'randwrite' 'read' 'write')
host=($(awk '{print $1}' $hostfile  | grep -v '#'))
blocksize=('1K' '4K' '8K' '16K' '32K' '64K' '128K')
filesize=('1G' '1G' '1G' '1G' '1G' '1G' '1G' )
mountpath='/data/redkv/kv-0'
toolpath='fio'
####################################################
len=${#host[@]}

for(( s = 0; s < ${#blocksize[@]}; s = s + 1 ))
do
    for(( op = 0; op < ${#oplist[@]}; op = op + 1 ))
    do
        cmd="$toolpath -numjobs=1 -iodepth=8 -direct=1 -ioengine=libaio -sync=1 -rw=${oplist[$op]}  -bs=${blocksize[$s]} -size=${filesize[$s]} -time_based -runtime=60 -name=Fio -directory=$mountpath/io"

        for(( client = 2; client <= $len; client = client + 1 ))
        do
            name=$(echo $RANDOM)
            for(( i = 2; i <= $client; i = i + 1 ))
            do
                num=$((i-1))
                echo "[`printf \"%2s\" $i`] ssh ${host[$num]} \"$cmd$i > /tmp/${name}.log &\""
                eval "ssh ${host[$num]} mkdir -p $mountpath/io$i"
                eval "ssh ${host[$num]} \"$cmd$i | tee /tmp/${name}.log &\" &"
            done
        done

        echo "Sleep 65 seconds"
        sleep 65

        for(( i = 1; i <= $client; i = i + 1 ))
        do
            num=$((i-1))
            eval "scp ${host[$num]}:/tmp/${name}.log ${host[$num]}.log.$name.${oplist[$op]}.${blocksize[$s]}.${filesize[$s]}"
        done

    done
done

处理数据

#!/bin/python

import os
import commands
import json
import argparse

g_dataType = ['read', 'write', 'randread', 'randwrite']
g_blockSize = ['1K', '4K', '8K', '16K', '32K', '64K', '128K', '1M']

def scan_files(path):
    if not os.path.exists(path):
        print("Give path cannot be found")
        exit(0)

    cmd = "/bin/ls *.log.*"
    ret, output = commands.getstatusoutput(cmd)
    if ret != 0:
        print("Command execute error, ret: %s, msg: %s" % (ret, output))
        exit(0)
    else:
        return output.split('\n')

def load_hosts(path):
    if not os.path.exists(path):
        print("Given path cannot find host file")
        exit(0)
    data = []
    with open(path) as fd:
        content = fd.read()
        for host in content.split("\n"):
            if not host.startswith("#") and host.strip():
                data.append(host)
    return data

def parse_file(files, path, hosts):
    if len(files) == 0:
        exit(0)

    out = []
    for host in hosts:
        ot = {}
        ot['host'] = host
        ot['data'] = {}
        ret = {}
        for t in g_dataType:
            ret[t] = {}
            for file in files:
                # 10.0.20.236.log.16003.write.16K.1G
                dt = str(file).split(".")
                if dt[-3] == t and host == ".".join(dt[0:4]):
                    ret[t][dt[-2]] = {}
                    with open(path + "/" + file) as fd:
                        content = fd.read().split("\n")
                        data = {}
                        for line in content:
                            # read: IOPS=77.1k, BW=2409MiB/s (2526MB/s)(141GiB/60001msec)
                            if "IOPS=" in line.strip():
                                item = line.strip().replace(',', '').split(' ')
                                data['iops'] = item[1].split('=')[1]
                                data['bw'] = item[2].split('=')[1]

                            # lat (usec): min=1929, max=19675, avg=10419.65, stdev=450.80
                            if line.strip().startswith("lat (usec):") or line.strip().startswith("lat (msec):"):
                                item = line.strip().replace(',', '').split(' ')
                                data['lat'] = {}
                                if item[1] == "(msec):":
                                    data['lat']['min'] = float(item[2].split('=')[1]) * 1000
                                    data['lat']['max'] = float(item[3].split('=')[1]) * 1000
                                    data['lat']['avg'] = float(item[4].split('=')[1]) * 1000
                                else:
                                    data['lat']['min'] = item[2].split('=')[1]
                                    data['lat']['max'] = item[3].split('=')[1]
                                    data['lat']['avg'] = item[4].split('=')[1]
                                data['lat']['stdev'] = item[5].split('=')[1]

                            if line.strip().startswith("clat percentiles"):
                                data['lat']['unit'] = line[-6:-2]
                                continue
                            #  99.00th=[  184], 99.50th=[  190], 99.90th=[  192], 99.95th=[  194]
                            pct =  ""
                            if "99.00th" in line.strip():
                                item = line.strip().strip("|").strip(",").split(",")
                                for v in item:
                                    m = v.split('=')[0][0:6].strip() # 99.00%
                                    n = v.split('=')[1][1:len(v.split('=')[1]) -1].strip()
                                    if data['lat']['unit'] and data['lat']['unit'] == "msec":
                                        n = int(float(n) * 1000)
                                    if m == "99.00":
                                        pct += "\033[31m%9sus\033[0m " % (str(n))
                                    else:
                                        pct += "%9sus " % (str(n))
                                data['lat']['pct'] = pct
                                break
                    ret[t][dt[-2]] = data
            ot['data'] = ret
        out.append(ot)
    return out

def format_data(data):
    if not data:
        exit(0)

    for item in data:
        for t in g_dataType:
            dt = item['data'][t]

            #u = {int(k[:-1]):v for k, v in dt.iteritems()}
            u = dt
            if dt:
                print "*"*140
                print "%40s | %s" % (t, item['host'])
                print "-"*140
                print "%10s %10s %10s %12s %12s %12s %13s %10s %10s %10s %10s" % ("BlockSize", "IOPS", "BandWidth", "Latency-min", "Latency-max", "Latency-avg","Latency-stdev", "Latency-P99", "Latency-P995", "Latency-P999", "Latency-P9995")
                print "-"*140
            else:
                continue
            for k in g_blockSize:
                if k not in u.keys():
                    continue
                print "%10s %10s %10s %10sus %10sus \033[32m%10sus\033[0m %13s %10s" % (k, u[k]['iops'], u[k]['bw'], u[k]['lat']['min'], u[k]['lat']['max'],u[k]['lat']['avg'],u[k]['lat']['stdev'],u[k]['lat']['pct'])
            print "\n"

def format_output_json(data):
    return json.dumps(data, indent=4)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='fio test result analysis tool')
    parser.add_argument('-f', '--host', type=str, default='./hosts', help='Host file, one host per line, canbe annotated with #')
    args = parser.parse_args()

    files = scan_files("./")
    hosts = load_hosts(args.host)
    data = parse_file(files, "./", hosts)
    format_data(data)