libeio性能测试

前言：

        之前搜了一下，貌似没有关于libeio的读写性能测试的数据，因此直接写了个测试程序去测试下。

写的没去试，接口都有，有需要可以自己去填下。

关于libeio：

        这个是库 Marc Lehmann 写的，关于这个大牛，有兴趣的可以去搜搜。

关于测试：

        测试环境我分别用ABC表示，采用随机读取文件中的某个位置开始的一定长度

        A表示我虚拟机中的测试，根据测试目录中的数据大小不同，会造成明显的差别

        B表示在服务器1中的测试，测试的目录是存放200多个flv文件的根目录，由于文件较小，因此速度很快

        C表示在服务器2中的测试，测试的目录是真实线上服务器中的有几千个影视数据的根目录，所有影视数据占近1T磁盘空间，由于文件较大，另外间隔一定时间就有同步数据的影响，及系统中的一些其他参数的影响，因此测试文件个数选取1000个。速度回相交B的测试结果要慢很多。

            

测试结果：

        A：

        1)选取有大文件的数据，目录中文件不多时，测试结构可以达到280左右M/s

        2)选取随便一个存有解压压缩包的目录，目录中文件有9000多个，文件较小，测试速度达到31.247337M/s

        B：这个环境比较简单，直接贴测试结果

all file list size:223

src file list size:223  dst_file_list_size:223

buf list is full out

test end,read size:21766.363750M cost:83.639529,speed:260.240152M/s

        C：

            1)选取真实影视数据目录的一个子目录，文件数在300多个，测试结果如下：

use pointer dir:/data/data2/

test begin

all file list size:329

src file list size:329  dst_file_list_size:329

test end,read size:3942.934089M cost:14.385984,speed:274.081640M/s

            2)选取所有影视数据的根目录，文件数在近万个，由于系统关系，打开几万个去测试会卡主，

    因此选取其中的1000个文件，测试结果如下：

第一次结果：

use pointer dir:/data/

test begin

all file list size:7803

src file list size:7796  dst_file_list_size:7796

test end,read size:11804.820662M cost:65.706916,speed:179.658724M/s

第二次结果(由于系统page cache，所以会较大)：

use pointer dir:/data/

test begin

all file list size:7803

src file list size:7796  dst_file_list_size:7796

test end,read size:11681.930182M cost:63.431233,speed:184.166847M/s

You have new mail in /var/mail/root

dg35_r710_9sd6t2x:/webservice/server/jiangwenlong/fileio # 

测试代码：

        依赖库：libeio,libapr,boost

     

   #include <apr.h>
#include <apr_lib.h>
#include <apr_general.h>
#include <apr_poll.h>
#include <apr_pools.h>
#include <eio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdlib.h>
#include <errno.h>

#include <apr_file_io.h>
#include <apr_strings.h>
#include <boost/filesystem.hpp>
#include <vector>

void want_poll();
void done_poll();
void apr_create_poll();
void apr_create_pool();
void apr_create_pipe();
void event_loop();
//TODO:read & write function for eio callback 
int write_cb(eio_req * req);
int read_cb(eio_req * req);
int open_cb(eio_req * req);
int close_cb(eio_req * req);

void read_dir();
void random_read();
void order_read();
void random_write();
void order_write();

static apr_pool_t *p = NULL;
static apr_pollset_t *pollset = NULL;
static int MAX_WATCH_SIZE = 100;
static apr_file_t *readp = NULL;
static apr_file_t *writep = NULL;

static long		  read_size=0;
static long		  write_size=0;

typedef struct _tag_FILE_INFO{
	int         fd;
	std::string file_path;
	int			file_size;
	bool        is_regular_file;
	time_t      last_write_time;
	char		*buf[256];
	int			pos ;
	_tag_FILE_INFO(){
		fd = 0;
		file_size = 0;
		last_write_time = 0;
		pos = 0;
	}
}FILE_INFO;

static char src_path[256] = "../";
static char dst_path[256] = "test_dst_data";
static std::vector<FILE_INFO>src_file_path_stat;
static std::vector<FILE_INFO>src_file_path_list;
static std::vector<std::string>dst_file_path_list;

//TODO:for eio callback want_poll
void want_poll()
{
	char c;
	apr_size_t size=1;
	if(apr_file_write(writep,&c,&size)
			!= APR_SUCCESS){
		printf("write pipe error\n");
		exit(1);
	}
}

//TODO:for eio_callback done_poll
void done_poll()
{
	char c;
	apr_size_t size=1;
	if(apr_file_read(readp,&c,&size)
			!= APR_SUCCESS){
		printf("read pipe error\n");
	}

}

//TODO:use apr lib create pipe
void apr_create_pipe()
{
	if(apr_file_pipe_create(&readp,&writep,p)
			!= APR_SUCCESS){
		abort();
	}
}

//TODO:use apr lib create pollset
void apr_create_poll()
{
	if(apr_pollset_create(&pollset,MAX_WATCH_SIZE,p,0)
			!= APR_SUCCESS){
		abort();
	}
}

//TODO:clean all what apr need clean
void apr_clean_all()
{
	if(apr_file_close(readp)
			!= APR_SUCCESS){
		printf("close pipe read fd error\n");
	}

	if(apr_file_close(writep)
			!= APR_SUCCESS){
		printf("close pipe write fd error\n");
	}
}

//TODO:use apr create pool
void apr_create_pool()
{
	if(apr_pool_create(&p,NULL) != APR_SUCCESS){
		abort();
	}
}

//TODO:for watch 
void event_loop()
{
	apr_pollfd_t pollfd;
	pollfd.desc_type = APR_POLL_FILE;
	pollfd.reqevents = APR_POLLIN;
	pollfd.desc.f  = readp;

	if(apr_pollset_add(pollset,&pollfd)
			!= APR_SUCCESS){
		printf("add fd to pollset error\n");
		exit(1);
	}
	
	while(eio_nreqs()){
		int num=0;
		const apr_pollfd_t *outfd=NULL;
		if(apr_pollset_poll(pollset,-1,&num,&outfd)
				!= APR_SUCCESS){
			printf("apr_poll error\n");
		}
		if(num != 1){
			printf("incorrect events\n");
		}else{
			int rc = eio_poll();
			if(rc == -1)
				printf("eio_poll error:%d\n",eio_poll());
		}
	}
	apr_pollset_remove(pollset,&pollfd);
}

int write_cb(eio_req * req)
{
	int rc = EIO_RESULT(req);
	if( rc <= 0 ){
		printf("write file:%s rc:%d,error msg:%s\n",(char*)req->data,rc,strerror(req->errorno));
		abort();
	}
	return 0;
}

int read_cb(eio_req * req)
{
	int rc = EIO_RESULT(req);
	if ( rc < 0){
		printf("read file:%s rc:%d error msg:%s\n",(char*)req->data,rc,strerror(req->errorno));
		abort();
	}
	FILE_INFO* info = (FILE_INFO*)req->data;
	//printf("file:%-20s\tread size:%d\n",info->file_path.c_str(),rc);
	read_size+=rc;
	return 0;
}

int open_cb(eio_req * req)
{
	int rc = EIO_RESULT(req);
	FILE_INFO * info = (FILE_INFO*)req->data;
	if(rc <= 0){
		printf("open file :%s ,error msg:%s\n",info->file_path.c_str(),strerror(req->errorno));
		return 0;
		abort();
	}
	info->fd = rc;
	//printf("open file fd:%d\n",info->fd);
	return 0;
}

int close_cb(eio_req *req)
{
	int rc = EIO_RESULT(req);
	if( rc != 0){
		printf("close file error,error msg:%s\n",strerror(req->errorno));
		abort();
	}
	return 0;
}

void read_dir()
{
	boost::filesystem::path spath(src_path);
	if(!boost::filesystem::is_directory(spath)){
		printf("error,src_path not directory\n");
		abort();
	}

	boost::filesystem::recursive_directory_iterator rcs_iter(spath);
	boost::filesystem::recursive_directory_iterator end;	
	//boost::filesystem::directory_iterator rcs_iter(spath);
	//boost::filesystem::directory_iterator end;
	for(;rcs_iter != end; ++rcs_iter){
		FILE_INFO finfo;
		try{
			finfo.is_regular_file = boost::filesystem::is_regular_file(*rcs_iter);
			finfo.last_write_time = boost::filesystem::last_write_time(*rcs_iter);
			finfo.file_path		  = (*rcs_iter).path().string();
			finfo.file_size		  = boost::filesystem::file_size(*rcs_iter);
			src_file_path_stat.push_back(finfo);	
			//printf("file:%s\n",finfo.file_path.c_str());
		}catch(...){

		}
	}
	printf("all file list size:%d\n",src_file_path_stat.size());
	std::vector<FILE_INFO>::iterator f_iter,f_end;
	f_iter = src_file_path_stat.begin();
	f_end  = src_file_path_stat.end();
	for(; f_iter != f_end; ++f_iter){
		if(f_iter->is_regular_file && f_iter->file_size > 0){
			src_file_path_list.push_back(*f_iter);
			boost::filesystem::path temp_path(f_iter->file_path);
			dst_file_path_list.push_back(temp_path.filename().string());
			//printf("filename:%s\n",temp_path.filename().string().c_str());
		}
	}
	
	printf("src file list size:%d\t dst_file_list_size:%d\n",src_file_path_list.size(),dst_file_path_list.size());
	
}

void random_read()
{
	srand(time(NULL));
	bool exit_flag = false;
	//while(!exit_flag){
		for(int i=0; i<src_file_path_list.size(); ++i){
			if(src_file_path_list.at(i).fd == 0){
				continue;
			}
			int temp_size = src_file_path_list.at(i).file_size / 256*1024 ;
			if(temp_size == 0){
				continue;
			}
					
			int read_bytes=rand()%(temp_size>100?100*256*1024:100*1024);
			int offset    =rand()%(1024*1024);
			offset=(offset+read_bytes)>src_file_path_list.at(i).file_size ? (rand()%100):offset;
			char *buf = NULL;
			buf = (char*)apr_palloc(p,read_bytes+1);
			if(buf == NULL){
				printf("use apr poll alloc memory error\n");
				continue;
			}
			//printf("read file:%-20s,file size:%-10d,offset:%-5d,length:%-10d\n",
			//src_file_path_list.at(i).file_path.c_str(),
			//src_file_path_list.at(i).file_size,offset,read_bytes);
			src_file_path_list.at(i).buf[src_file_path_list.at(i).pos++] = buf;
			eio_read(src_file_path_list.at(i).fd,buf,read_bytes,offset,0,read_cb,&src_file_path_list.at(i));
			if(src_file_path_list.at(i).pos >= 10){
				printf("buf list is full out\n");
				exit_flag = true;
				break;
			}
		}
	//}
}

int main(int argc,char **argv)
{
	apr_initialize();
	apr_create_pool();
	apr_create_pipe();
	apr_create_poll();
	if(eio_init(want_poll,done_poll)){
		abort();
	}
	if(argc == 2){
		printf("use pointer dir:%s\n",argv[1]);
		memset(src_path,0,256);
		sprintf(src_path,"%s",argv[1]);
	}
	printf("test begin\n");
	read_dir();
	
	for(int i = 0; i < src_file_path_list.size(); ++i){
		if(i > 1000)break;
		eio_open(src_file_path_list.at(i).file_path.c_str(),O_RDONLY,0644,0,open_cb,&src_file_path_list.at(i));
	}
	event_loop();

	apr_time_t begin = apr_time_now();
	random_read();
	event_loop();
	apr_time_t end = apr_time_now();

	for(int i = 0; i < src_file_path_list.size(); ++i){
		if(src_file_path_list.at(i).fd > 0)
			eio_close(src_file_path_list.at(i).fd,0,close_cb,NULL);
	}
	event_loop();
	double total_size = read_size*1.0/(1024*1024);
	double cost_time  = (end-begin)/1000000.0;
	double speed = total_size/cost_time;
	printf("test end,read size:%lfM cost:%lf,speed:%lfM/s\n",read_size*1.0/(1024*1024),(end-begin)/1000000.0,speed);
	apr_terminate();
	return 0;
}