目前在做项目的时候,遇到两个服务之间传输文件,由于数据很大需要进行分片传输,一片一片的传输,然后到了另外一个服务上进行合成,组成一个完整的问题。如果是小的 文件,几百kB的,根本不需要分片的。
看一下代码,这部分是如何将数据一点一点的读取,然后然后写入到本地的。
#include "stdafx.h"
#include <windows.h>
#include <memory.h>
#include <iosteam>
#include <fstream>
using namespace std;
const char FILE_RD = "d:\\vs2013.iso";//原文件
const char FILE_WR = "d:\\vs2013_bak.iso";//待写入的文件
const size_t WR_SIZE = 1024*1024*10;//每次读写的大小,此处为10M
void hugefile_rw()
{
FILE* stream_rd,*stream_wr;
size_t num_read,num_written;
int rw_cnt = 0;
long long offset = 0;
if(0 != fopen_s(&stream_wr,FILE_WR,"wb")){//获取待写入文件的句柄
return;
}
DWORD dStart = GetTiclCount();
char *buf = new char[WR_SIZE];
if(0 == fopen_s(&stream_rd,FILE_RD,"rb"))
{
while(!feof(stream_rd)){
rw_cnt++;
//读
num_read = fread(buf,sizeof(char),WR_SIZE,stream_rd);
if(0==num_read) break;
//读取文件的时候,需要将读取到的文件长度也要设置一下,避免遇到“\0”被截断
string data = string(buf,num_read);
offset += num_read;//文件偏移
_fseeki64(stream_rd,offset,SEEK_SET);
//写
num_written = fwrite(buf,sizeof(char),num_read,stream_wr);
_fseeki64(stream_wr,offset,SEEK_SET);
cout<<"Cnt = "<<rw_cnt<<",num_read = "<<num_read <<",num_written = "<< num_written;
}
}
DWORD dEnd = GetTiclCount();
cout<<"读写时间 = "<< (dEnd - dStart)/1000<<"s"<<endl;
//资源释放
delete []buf;
fclose(stream_rd);
fclose(stream_wr);
}
int main(int argc,_TCHAR* argv[])
{
hugefile_rw();
getchar();
return 0;
}
通过网络传输的,需要将文件切片,具体代码如下:
bool CUpgradeMgr::UpgradePackageSendToSite(UINT_T id_site, const string& packagePath)
{
rsmmodule::RemoteSiteExtendData remotesite;
if (!rsmmodule::RSMModule_FindRemoteSiteByID(remotesite, id_site))
{
Error::SetLastError(Error::ErrorCode(ERROR_CODE(CMS_MODULE_CMS), ERROR_CODE(CMS_REMOTESITE_IS_NOT_EXIST)));
return false;
}
ifstream read_file_(packagePath.c_str(), ios::binary);
bumblebee::UINT_T chunkSize = 1 * 1024 * 1024; //1MB 每片,如果要分片,分多大一片? 默认大小为5M
if (read_file_.is_open())
{
read_file_.seekg(0, ios_base::end);//移动到文件尾
istream::pos_type file_size = read_file_.tellg();//取得当前位置的指针长度->即文件长度
read_file_.seekg(0, ios_base::beg);//移动到原来的位置
double totalSize = static_cast<double>(file_size);
bumblebee::UINT_T fileTotalSize = static_cast<bumblebee::UINT_T>(file_size);
bumblebee::UINT_T chunks = ceil(totalSize / chunkSize);
FILE* stream_rd;
size_t num_read;
bumblebee::UINT_T rw_cnt = 0;
long long offset = 0;
char *buf = new char[chunkSize];
memset(buf, 0, chunkSize);
if (0 == fopen_s(&stream_rd, packagePath.c_str(), "rb"))
{
while (!feof(stream_rd))
{ //读
num_read = fread(buf, sizeof(char), chunkSize, stream_rd);
string data_ = string(buf, num_read);
int size__ = data_.length();
if (0 == num_read) break;
offset += num_read;//文件偏移
_fseeki64(stream_rd, offset, SEEK_SET);
HttpResponse http_response;
http_response.content_type = "application/octet-stream";
http_response.send_timeout = 300000;
//http_response.file_name = "SiteBackup.backup";
http_response.content_disposition = "attachment";
string str_response;
rsmmodule::RSMModule_HttpSendPackage(str_response, id_site, "PUT", "/ISAPI/Bumblebee/Upgrade/UpgradeDevicePackage", fileTotalSize, chunks, rw_cnt, std::string(data_), 500);
rw_cnt++;
}
}
delete[] buf;
buf = nullptr;
fclose(stream_rd);
}
read_file_.close();
return true;
}
bool HttpSendPackage(std::string& response, const SiteCommRequestData& request_data, std::string Method, const std::string& url,int totalSize,int chunks,int chunk, const std::string& send_data = std::string())
{
if (request_data.sid.empty())
{
Base::Error::SetLastError(Base::Error::ErrorCode(CMS_MODULE_CMS, CMS_REMOTE_IS_OFFLINE));
return false;
}
std::string token = request_data.token;
if (token.empty())
{
token = CRemoteSiteMgr::Instance().CreateToken(request_data.sid);
}
// 发送数据
vsmnetlib::HttpRequest http_request;
http_request.connect_tiemout = request_data.connect_tiemout;
http_request.send_timeout = request_data.send_timeout;
http_request.recv_timeout = request_data.recv_timeout;
std::ostringstream ss_url;
std::string access_type = CRemoteSiteMgr::Instance().GetAccessType();
ss_url << access_type << "://" << request_data.ip << ":" << request_data.port << url << "?SID=" << request_data.sid;
if (request_data.client_type != -1)
{
ss_url << "&CT=" << request_data.client_type;
}
if (!token.empty())
{
ss_url << "&Token=" << token;
}
if (Method == "GET")
{
ss_url << "&MT=GET";
Method = "POST";
}
ss_url << "&size=" << totalSize;
ss_url << "&chunks=" << chunks;
ss_url << "&chunk=" << chunk;
if (!send_data.empty())
{
http_request.send_buff = const_cast<char*>(send_data.c_str());
http_request.send_size = static_cast<DWORD>(send_data.length());
}
http_request.http_method = Method;
http_request.http_url = ss_url.str();
// 返回值
LONG client_handle = 0;
vsmnetlib::HttpClientRecv recv_data;
LONG http_ret = vsmnetlib::CVsmNetLib::HttpClientSynSend(client_handle, recv_data, http_request);
bool ret = CheckHttpResponse(response, http_ret, recv_data, http_request.http_url);
if (!ret)
{
// 提前收到session无效,优化
if (!request_data.sid.empty())
{
auto last_errro = Base::Error::GetLastError();
std::string cascading_error = last_errro.GetCasadingErrorInfo();
if (!cascading_error.empty())
{
int error_module = 0;
int error_code = 0;
sscanf(cascading_error.c_str(), "(%d, %d)", &error_module, &error_code);
if (error_module == CMS_MODULE_CMS &&
error_code == CMS_SESSIONID_INVALID)
{
// 根据sid查找site
auto ptr_site = CRemoteSiteMgr::Instance().FindRemoteSiteBySession(request_data.sid);
if (ptr_site)
{
ptr_site->SessionInvalid();
}
}
}
}
}
vsmnetlib::CVsmNetLib::HttpClostClientLink(client_handle);
return ret;
}
以上代码为切片传输的格式,下面是要进行切片文件的接收以及文件的合成,主要是通过文件上传管理类来进行出来的。具体就不说明了,会涉及到公司的代码。
下面是将图片数据写到本地的,调用std::ofstream。
DWORD dwPictureLen; //图片大小
char* pPicBuffer; //图片数据,
string file_path = "D:\\12.jpg";
std::ofstream out_file(file_path.c_str(), ios::out | ios::binary | ios::trunc);
if (!out_file)
{
return ;
}
if (!out_file.write(pPic->pPicBuffer, pPic->dwPictureLen))
{
return ;
}
if (out_file.is_open())
{
out_file.close();
}