研: 超大文件处理

给点信心啊。。。

加密，可正常输的版本，好像比之前的快了些，正常运行。

能处理1.6Ｇ左右，具体没看，，，，大了会直接跳出。

	//encryption
	//cout<<data_;
	/*
	 *ofstream out_lt("letter");
	 *for(auto i=0;i<length_r;i++)
	 *{//TODO:bug, if size of file too big ,core dump come 
	 *    int tmp=(int)data_[i];
	 *    //C multiply itself sk_ times, then mod sn_
	 *    int num_ecd=(int)pow((double)tmp,sk_) % sn_;
	 *    //out_lt<<num_ecd<<" ";//seprate with space
	 *    out_lt<<num_ecd;
	 *}
	 *out_lt.close();
	 */

	int	off=0;
	//const char *filename_read = filename.c_str();//filename 
	fd_r = open(filename_read,O_RDWR);
	assert(fd_r);

	filename_write="output_letter.data";
	fd_w = open(filename_write,O_RDWR|O_CREAT|O_TRUNC,0644);
	assert(fd_w);
	length_r = lseek(fd_r,0,SEEK_END);
	length_w = lseek(fd_w,length_r - 1,SEEK_CUR);
	cout<<"length_r:"<<length_r<<endl;
	cout<<"length_w:"<<length_w<<endl;
	int tp=	write(fd_w,"0",1);//fix addr cannot be access 
	assert(tp);

	lseek(fd_w,0,SEEK_SET);
	lseek(fd_r,0,SEEK_SET);
	
	ofstream out_lt("letter");
	auto page=length_r/BUF_SIZE;
	auto remain=length_r%BUF_SIZE;
	cout<<"page:"<<page<<endl;
	cout<<"length_r:"<<length_r<<endl;
	cout<<"size_:"<<size_<<endl;
	cout<<"\n======= off+BUF_SIZE :"<<off+BUF_SIZE<<"\nremain: "<<length_r - off <<"\nremain: "<<remain<< endl;

	for(auto k=0;k<length_r;k++)
	{
		//cout<<data_;
		int tmp=(int)data_[k];
		//C multiply itself sk_ times, then mod sn_
		int num_ecd=(int)pow((double)tmp,sk_) % sn_;
		//out_lt<<num_ecd<<" ";
		out_lt<<num_ecd;
	}

	for(auto k=0;k<page;k++)
	{
		lseek(fd_r,off,SEEK_SET);
		//cout<<"off: "<<off<<endl;
		data_ =(char*) mmap(NULL,BUF_SIZE,PROT_READ|PROT_WRITE,MAP_SHARED,fd_r,off);
		//p_w = (char*)mmap(NULL,BUF_SIZE,PROT_WRITE,MAP_SHARED,fd_w,off);
		p_w = (int*)mmap(NULL,BUF_SIZE,PROT_WRITE,MAP_SHARED,fd_w,off);
		/*
		 *for(auto i=off;i<off+BUF_SIZE-1;i++)
		 *{
		 *    //cout<<data_[i];
		 *    int tmp=(int)data_[i];
		 *    //C multiply itself sk_ times, then mod sn_
		 *    int num_ecd=(int)pow((double)tmp,sk_) % sn_;
		 *    //out_lt<<num_ecd<<" ";
		 *    out_lt<<num_ecd;
		 *    //cout<<num_ecd;
		 *    //memset(*p_w,num_ecd,sizeof(num_ecd));
		 *    //p_w++;
		 *}
		 */

		//memcpy(p_w,data_,BUF_SIZE);
		/*
		 *if(page==k)
		 *{
		 *    data_ = (char*)mmap(NULL,length_r - off ,PROT_READ|PROT_WRITE,MAP_SHARED,fd_r,off);
		 *    //p_w = (char*)mmap(NULL,length_r - off,PROT_WRITE,MAP_SHARED,fd_w,off);
		 *    p_w = (int*)mmap(NULL,length_r - off,PROT_WRITE,MAP_SHARED,fd_w,off);
		 *    //memcpy(p_w,data_,length_r -off );
		 *    //munmap(data_,length_r - off);
		 *    //munmap(p_w,length_r - off);
		 *    cout<<"length_r:"<<length_r<<endl;
		 *    cout<<"size_:"<<size_<<endl;
		 *    cout<<"\n======= off+BUF_SIZE :"<<off+BUF_SIZE<<"\nremain: "<<length_r - off <<"\nremain: "<<remain<< endl;
		 *    for(auto i=off;i<length_r;i++)
		 *    {
		 *        //cout<<data_[i];
		 *        int tmp=(int)data_[i];
		 *        //C multiply itself sk_ times, then mod sn_
		 *        int num_ecd=(int)pow((double)tmp,sk_) % sn_;
		 *        //out_lt<<num_ecd<<" ";
		 *        //out_lt<<num_ecd;
		 *        //memset(*p_w,num_ecd,sizeof(num_ecd));
		 *        //memcpy(p_w,num_ecd,sizeof(num_ecd) );
		 *        //p_w++;
		 *    }
		 *    //break;
		 *}
		 */
		off += BUF_SIZE;
	}
	out_lt.close();

	munmap(data_,length_r);
	munmap(p_w,length_r);
	close(fd_r);
	close(fd_w);

 

 

统计：

mmap + ofstream

time make run　

 

861M May 27 15:12 test.data

Now submit all the things, thanks!

real    1m41.327s
user    1m37.384s
sys    0m2.060s

 

1.6G May 27 15:16 test.data

....................
Now submit all the things, thanks!

real    3m2.381s
user    2m54.396s
sys    0m3.664s

都生成 4.8G May 27 15:25 letter

 

 

ifstream+ ofstream:

1.6G May 27 15:16 test.data

Now submit all the things, thanks!
177.80user 5.74system 3:07.56elapsed 97%CPU (0avgtext+0avgdata 2080656maxresident)k
3159640inputs+10028824outputs (47major+650847minor)pagefaults 0swaps

real    3m7.572s
user    2m57.812s
sys    0m5.744s

 

结论，没什么用，根本没提升，要用memcpy

 

 

对output_letter 进行memset