基于libcurl 实现web-uploader客户端大文件分片，断点续传(curl-uploader)

zhujinghao09

已于 2023-04-03 13:34:06 修改

阅读量1.9k

点赞数 1

分类专栏： libcurl C++ 嵌入式文章标签： c++ Powered by 金山文档

于 2023-02-09 17:38:14 首次发布

本文链接：https://blog.csdn.net/zhujinghao_09/article/details/128955861

版权

C++ 同时被 3 个专栏收录

7 篇文章 0 订阅

订阅专栏

libcurl

2 篇文章 1 订阅

订阅专栏

嵌入式

2 篇文章 0 订阅

订阅专栏

Linux Arm嵌入式设备 l 基于libcurl 实现大文件分片上传，断点续传，秒传,分片大小可控，使用libcurl 实现web-upload linux c/c++版本客户端(curl-uploader)。通过wireshark 抓包网页版客户端上传文件过程分析http post 流程，总结http 流程如下。

工程代码：

客户端curl-uploader: https://download.csdn.net/download/zhujinghao09/87431650

后台服务器： https://gitee.com/laminae_admin/webupload/

主要流程如下：

_check_url = base_url + "/upload/fileRecord//zone/upload/md5Check"; (MD5验证)

_upload_url = base_url + "/upload/fileRecord//zone/upload"; （上传分片）

_merge_url = base_url + "/upload/fileRecord//zone/upload/merge/"; （合并分片）

核心代码：

s1 文件MD5

/*
    计算文件MD5，支持分片计算，指定计算大小
*/
int Ufile::file_md5(std::string &md5_value,int start,int end)
{
    char buf[1024 * 16];
    MD5_CTX md5Context;
    MD5_Init(&md5Context);
    int  k =0;
    int s = start;
    int e = end;

     md5_value.clear();

    //_file.seekp(0, _file.end);
    auto fos = _file.tellg();


    if(s<=0)
        s= 0;
    if(s>_file_size){
        return -1;
    }
    if(e<0){
        return -1;
    }
    if(e>=_file_size){
        e = _file_size;
    }
    k = 0;
    if(s>0)
        _file.seekg(s);

    while (_file.good() && k <= e-s) {
        _file.read(buf, sizeof(buf));
        // gcount() 返回上次从文件流提取出的字符个数
        MD5_Update(&md5Context, buf, _file.gcount());
        k += _file.gcount();
    }

    if (_file.eof())
        _file.clear();    //清空结尾eof标志，可以再次打开该文件


    unsigned char result[MD5_DIGEST_LENGTH];
    MD5_Final(result, &md5Context);

    char hex[35];
    memset(hex, 0, sizeof(hex));
    for (int i = 0; i < MD5_DIGEST_LENGTH; ++i)
    {
        sprintf(hex + i * 2, "%02x", result[i]);
    }
    hex[32] = '\0';
    md5_value = string(hex);
    //回位
    _file.seekg(fos);


    return 0;
}

S2 文件分片读取，与计算分片MD5

/*
    读取分片数据
*/
int SliceFile::do_read_slice_data(int sliceIndex)
{
    if(_cur_slice_index==sliceIndex){
        return 0;
    }
    int once_read_size = 16*1024;

    int s = sliceIndex * _slice_size;
    int k = 0;

    int to_read_size = _slice_size;

    if(s + to_read_size > file_size()){
        to_read_size = file_size() - s;
    }

    _file.seekg(s);

    while (_file.good() && k < to_read_size) {
        _file.read(_slice_data+k, once_read_size);
        // gcount() 返回上次从文件流提取出的字符个数
        k += _file.gcount();
    }

    if (_file.eof())
        _file.clear();    //清空结尾eof标志，可以再次打开该文件


    _slice_data_size = k;
    _cur_slice_index = sliceIndex;


    return 0;

}

char* SliceFile::get_slice_data(int sliceIndex)
{
    if(sliceIndex != _cur_slice_index){
        do_read_slice_data(sliceIndex);
    }
    return _slice_data;
}

int SliceFile::get_slice_data_size(int sliceIndex)
{
    if(sliceIndex != _cur_slice_index){
            do_read_slice_data(sliceIndex);
    }

    return _slice_data_size;
}
/*
        计算分片MD5

*/
int SliceFile::SliceMd5(std::string &md5,int sliceIndex){

    if(sliceIndex <0 || sliceIndex > _slice_num){
        printf("invlaid sliceIndex:%d\n",sliceIndex);
        return -1;
    }
    return file_md5(md5,sliceIndex*_slice_size,(sliceIndex+1)*_slice_size);
}


/*
        打印文件信息，分片数量及分片md5
*/
void SliceFile::PrintInfo()
{
    printf("\n\n=====================File========================\n");
    printf("file_path: %s\n",_file_path.c_str());
    printf("file_name: %s\n",_file_name.c_str());


    printf("\n\n=====================MD5========================\n");
    printf("file_size:  %d\n",_file_size);
    printf("slice_size: %d\n",_slice_size);
    printf("slice_num:  %d\n",_slice_num);

    printf("fileMD5: %s\n",file_md5().c_str());
    for(int i=0;i<_slice_num;i++){
        std::string md5;

        SliceMd5(md5,i);
        printf("sliceIndex:%d,     zoneMd5:%s\n",i,md5.c_str());
    }

    printf("=====================MD5========================\n");

    //printf("function = %s,line = %d\n",__FUNCTION__,__LINE__);
    printf("文件使用的设备号: %lu\r\n", _file_state.st_dev); //32为int数据，高12位主设备号，低20位次设备号
    printf("索引节点号: %lu\r\n", _file_state.st_ino);
    printf("文件对应的模式，文件，目录等: %u\r\n", _file_state.st_mode);
    printf("文件的硬连接数: %lu\r\n", _file_state.st_nlink);
    printf("所有者用户识别号: %u\r\n", _file_state.st_uid);
    printf("组识别号: %u\r\n", _file_state.st_gid);
    printf("设备文件的设备号: %lu\r\n", _file_state.st_rdev);
    printf("文件大小: %ld\r\n", _file_state.st_size);
    printf("访问日期: %s", ctime(&_file_state.st_atime));
    printf("最后修改日期: %s", ctime(&_file_state.st_mtime));
    printf("文件创建时间: %s", ctime(&_file_state.st_ctime));

    printf("=====================File========================\n\n");
}

S3 libcurl POST 上传二进制分片数据


/*
 *
 * return
 *   0 sucess
 *
 */
int WebUploader::curlPostFormDataWithBin(SliceFile& file,int sliceIndex,const char* _url, struct curl_httppost* formpost,CURL_CALL_RESULT* result)
{
    CURLcode res;

    struct curl_slist * _http_headers;
    char *progress_data = (char*)NULL;

    BOOL prevent_cache = TRUE;

    // http_headers = curl_slist_append(http_headers, "Accept-Charset: utf-8");//WEB暂不识别
    // http_headers = curl_slist_append(http_headers, "Accept-Encoding: utf-8");//WEB暂不识别
    //返回为JSON时使用。
    _http_headers = curl_slist_append(_http_headers, "Content-Type: multipart/form-data");
    //_http_headers = curl_slist_append(_http_headers, "Content-Type:application/json;charset=UTF-8");
    //http_headers = curl_slist_append(http_headers, "Content-Type: application/x-www-form-urlencoded; charset=UTF-8");
    //curl_easy_setopt(_curlHandle, CURLOPT_SSLVERSION, 3);
    curl_easy_setopt(_curlHandle, CURLOPT_SSL_VERIFYHOST, 0);
    curl_easy_setopt(_curlHandle, CURLOPT_SSL_VERIFYPEER, 0L);//忽略证书检查
    //curl_easy_setopt(_curlHandle, CURLOPT_SSLCERT,"ca.pem");
    //curl_easy_setopt(_curlHandle, CURLOPT_CAINFO,"ca.pem");
    //curl_easy_setopt(_curlHandle, CURLOPT_CAPATH, "/etc/ssl");

    curl_easy_setopt(_curlHandle, CURLOPT_URL, _url);
    curl_easy_setopt(_curlHandle, CURLOPT_POST, 1L);

    curl_easy_setopt(_curlHandle, CURLOPT_HTTPHEADER, _http_headers);    //设置发送http头部
    curl_easy_setopt(_curlHandle, CURLOPT_HTTPPOST, formpost);

    curl_easy_setopt(_curlHandle, CURLOPT_TIMEOUT, 10L);            //设置超时时间

    //重要!!!多线程下禁用控制域名解析的alarm 超时
    curl_easy_setopt(_curlHandle, CURLOPT_NOSIGNAL, prevent_cache);


    curl_easy_setopt(_curlHandle, CURLOPT_PROGRESSDATA, &file);
    curl_easy_setopt(_curlHandle, CURLOPT_NOPROGRESS, FALSE);
    curl_easy_setopt(_curlHandle, CURLOPT_PROGRESSFUNCTION,WebUploader::web_service_progress_func);

    //提交post字符串
//    if(_content_ptr && _content_size>0){
//        curl_easy_setopt(_curlHandle, CURLOPT_POSTFIELDS, _content_ptr);
//        curl_easy_setopt(_curlHandle, CURLOPT_POSTFIELDSIZE, _content_size);
//    }

    curl_easy_setopt(_curlHandle, CURLOPT_WRITEDATA, result);
    curl_easy_setopt(_curlHandle, CURLOPT_WRITEFUNCTION,WebUploader::web_service_rsp_write_func);


    curl_easy_setopt(_curlHandle, CURLOPT_VERBOSE, 1L);

    printf("enter curl ws call: curl=%p, ws_result_ptr=%p....\n",
            _curlHandle,
            result);

    //printf( "curl_easy_perform()....\n");
    res = curl_easy_perform(_curlHandle);
    result->curl_res = res;


    return (int)res;
}

/*
 * {"success":true,"code":10000,"message":"...............","data":{"zoneNowIndex":1,"fileZone":{"id":"1619576327126519808","zoneName":"8c6ca01dcd6c57160a83e7e8d675b3e1.mp4.temp","zonePath":"d://temp/webupload/fileData/webuploadfile/temp/cbe607627b5e27ee460acde1005f160d","zoneMd5":"8c6ca01dcd6c57160a83e7e8d675b3e1","zoneRecordDate":"2023-01-29 06:01:01","zoneCheckDate":null,"zoneTotalCount":2,"zoneTotalMd5":"cbe607627b5e27ee460acde1005f160d","zoneNowIndex":1,"zoneStartSize":5242880,"zoneEndSize":7271720,"zoneTotalSize":7271720,"zoneSuffix":".mp4","fileRecordId":"1619576324489478146"},"isExist":false}}
 *
return
     0  already upload sucess
    -1  error
*/
int WebUploader::curlPostSlice(SliceFile& file,int sliceIndex,string &zoneMd5)
{
    struct curl_httppost *formpost = NULL;
    struct curl_httppost *lastptr = NULL;

    CURLFORMcode  resultForm;
    int ret =0;

    struct curl_httppost *p = NULL;


    resultForm = curl_formadd( &formpost, &lastptr, CURLFORM_COPYNAME, "id",
            CURLFORM_COPYCONTENTS, (const char *) "WU_FILE_0", CURLFORM_END );

    resultForm = curl_formadd( &formpost, &lastptr, CURLFORM_COPYNAME, "name",
            CURLFORM_COPYCONTENTS, (const char *) file.file_name().c_str(), CURLFORM_END );

    //application/octet-stream
    //video/mp4

    resultForm = curl_formadd( &formpost, &lastptr, CURLFORM_COPYNAME, "type",
            CURLFORM_COPYCONTENTS, (const char *) file.content_type().c_str(), CURLFORM_END );

    resultForm = curl_formadd( &formpost, &lastptr, CURLFORM_COPYNAME, "lastModifiedDate",
            CURLFORM_COPYCONTENTS, (const char *) file.lastModifiedDate().c_str(), CURLFORM_END );

    resultForm = curl_formadd( &formpost, &lastptr, CURLFORM_COPYNAME, "size",
            CURLFORM_COPYCONTENTS, (const char *) to_string(file.file_size()).c_str(), CURLFORM_END );

    resultForm = curl_formadd( &formpost, &lastptr, CURLFORM_COPYNAME, "chunk",
            CURLFORM_COPYCONTENTS, (const char *) to_string(sliceIndex).c_str(), CURLFORM_END );

    resultForm = curl_formadd( &formpost, &lastptr, CURLFORM_COPYNAME, "fileMd5",
            CURLFORM_COPYCONTENTS, (const char *) file.file_md5().c_str(), CURLFORM_END );


    resultForm = curl_formadd( &formpost, &lastptr, CURLFORM_COPYNAME, "contentType",
            CURLFORM_COPYCONTENTS, (const char *) file.content_type().c_str(), CURLFORM_END );

    resultForm = curl_formadd( &formpost, &lastptr, CURLFORM_COPYNAME, "zoneTotalMd5",
            CURLFORM_COPYCONTENTS, (const char *) file.file_md5().c_str(), CURLFORM_END );

    resultForm = curl_formadd( &formpost, &lastptr, CURLFORM_COPYNAME, "zoneMd5",
            CURLFORM_COPYCONTENTS, (const char *) zoneMd5.c_str(), CURLFORM_END );


    resultForm = curl_formadd( &formpost, &lastptr, CURLFORM_COPYNAME, "zoneTotalCount",
            CURLFORM_COPYCONTENTS, (const char *) to_string(file.slice_num()).c_str(), CURLFORM_END );


    resultForm = curl_formadd( &formpost, &lastptr, CURLFORM_COPYNAME, "zoneNowIndex",
            CURLFORM_COPYCONTENTS, (const char *) to_string(sliceIndex).c_str(), CURLFORM_END );

    resultForm = curl_formadd( &formpost, &lastptr, CURLFORM_COPYNAME, "zoneTotalSize",
            CURLFORM_COPYCONTENTS, (const char *) to_string(file.file_size()).c_str(), CURLFORM_END );


    resultForm = curl_formadd( &formpost, &lastptr, CURLFORM_COPYNAME, "zoneStartSize",
            CURLFORM_COPYCONTENTS, (const char *) to_string(file.slice_size()*sliceIndex).c_str(), CURLFORM_END );

    //Content-Disposition: form-data; name="zoneEndSize"
    //
    //386726
    int zoneEndSize = file.slice_size()*(sliceIndex+1) > file.file_size()? file.file_size(): file.slice_size()*(sliceIndex+1);
    resultForm = curl_formadd( &formpost, &lastptr, CURLFORM_COPYNAME, "zoneEndSize",
            CURLFORM_COPYCONTENTS, (const char *) to_string(zoneEndSize).c_str(), CURLFORM_END );

    //Content-Disposition: form-data;name="file";filename="yolov5s_zhc_foodSmokePhone_13330_300_960_2022-05-12.rknn"
    //Content-Type:    application/octet-stream
    //二进制流上传：
    resultForm = curl_formadd(&formpost,&lastptr,
            CURLFORM_COPYNAME,     "file",
            CURLFORM_BUFFER,        file.file_name().c_str(),
            CURLFORM_BUFFERPTR,     file.get_slice_data(sliceIndex),
            CURLFORM_BUFFERLENGTH,  file.get_slice_data_size(sliceIndex),
            CURLFORM_CONTENTTYPE,    "application/octet-stream",
            CURLFORM_END);
    //curl_formadd

    if(resultForm!=CURL_FORMADD_OK){
        printf("curl_formadd faild:%d\n",resultForm);
        return -1;
    }
    p = formpost;
    do{
        printf("%s,%s,%d\n",p->name, p->contents,p->contentslength);
        p = p->next;
    }while(p!=lastptr);

    clear_ws_reault(&_ws_result);
    ret = curlPostFormDataWithBin(file,sliceIndex,_upload_url.c_str(),formpost,&_ws_result);

    if(ret == 0 && strstr(_ws_result.json_data,"\"success\":true,\"code\":10000,")){
        //upload sucess
        ret = 0;
    }else{
        printf("curlPostFormDataWithBin fiald ret:%d,%d\n",ret,_ws_result.curl_res);
    }


    return ret;

}

S4 分片上传主流程

/*
 *
 *
 *
   return 0  already upload sucess
   return 1  no upload
   return -1  system error
 **/
int WebUploader::CheckSlice(SliceFile& file,int sliceIndex,string &zoneMd5)
{
    int ret = -1;

    string ctx = "checkType=2&zoneTotalMd5=";
    ctx += file.file_md5();
    ctx += "&zoneMd5=";
    ctx += zoneMd5;

    char* content_type = "Content-Type: application/x-www-form-urlencoded; charset=UTF-8";
    clear_ws_reault(&_ws_result);
    curlPost(_check_url.c_str(),content_type,ctx.c_str(),ctx.size(),&_ws_result);

    if(_ws_result.curl_res != CURLE_OK){
        ret = -1;
        goto end;
    }
    if(strstr(_ws_result.json_data,"\"success\":true,\"code\":10000,")){
        //already upload
        ret = 0;
        goto end;
    }else if(strstr(_ws_result.json_data,"\"success\":false,\"code\":99999,")){
        //no upload
        ret = 1;
        goto end;
    }else{
        ret = -1;
    }
end:
    return ret;
}

/*
 *
 * return:
 *  0 sucess
 *  1 faild
 */
int WebUploader::MergeFileSlices(SliceFile& file)
{
    int ret = -1;
    string zoneMd5;
    string ctx="";

    if(!file.remote_path().empty()){
        ctx += "remotePath=";
        ctx += file.remote_path();
    }
    string url = string(_merge_url.c_str());
    url += file.file_md5();
    char* content_type = "Content-Type: application/x-www-form-urlencoded; charset=UTF-8";
    clear_ws_reault(&_ws_result);
    curlPost(url.c_str(),content_type,ctx.c_str(),ctx.size(),&_ws_result);

    if(_ws_result.curl_res != CURLE_OK){
        ret = 1;
        goto end;
    }
    if(strstr(_ws_result.json_data,"\"success\":true,\"code\":10000,")){
        //Merge  sucess
         ret = 0;
        goto end;
    }else{
        //Merge  faild
        ret = 1;
        goto end;
    }
end:
    return ret;
}

/*
   return 0  already upload sucess
   return -1   error
 */
int WebUploader::UploadFileSlice(SliceFile& file,int sliceIndex)
{
    int ret = 0;
    std::string zoneMd5;

    printf("\n\n");

    zoneMd5.clear();
    file.SliceMd5(zoneMd5,sliceIndex);

    printf("========================= step 2.%d-1 CheckSlice==========================\n",sliceIndex);

    ret = CheckSlice(file,sliceIndex,zoneMd5);


    printf("CheckSlice:%d result:%d\n",sliceIndex,ret);
    //already upload
    if(ret==0){
        return 0;
    }else if(ret==1){
        printf("========================= step 2.%d-2 curlPostSlice==========================\n",sliceIndex);
        //no upload
        //do cur post
        ret = curlPostSlice(file,sliceIndex,zoneMd5);
        printf("curlPostSlice:%d result:%d\n",sliceIndex,ret);
        return ret;
    }else{
        printf("CheckSlice sys error sliceIndex:%d,zoneMd5:%s\n",sliceIndex,zoneMd5.c_str());
        return -1;
    }

    return 0;
}
/*
   return 0  already upload sucess
   return -1   error
 */
int WebUploader::UploadFile(SliceFile& file)
{
    int ret = 0;
    //"contentType" = "image/jpeg"
    do{
        //step1

        printf("========================= step 1 CheckFile==========================\n");
        int checkRet = CheckFile(file);
        if(checkRet==0){
            ret = 0;
            break;
        }else if(checkRet==-1){
            printf("CheckFile sys error...\n");
            ret = -1;
            break;
        }

        //step2
        printf("========================= step 2 UploadFileSlice==========================\n");
        for(int i =0;i<file.slice_num();i++){
            ret = UploadFileSlice(file,i);
            if(ret !=0){
                printf("upload file:%s,slice:%d faild\n",file.file_path().c_str(),i);
                break;
            }
        }
        if(ret != 0){
            break;
        }
        //step3
        printf("========================= step 3 MergeFileSlices==========================\n");
        ret = MergeFileSlices(file);
        if(ret != 0){
            printf("upload file:%s,Merge faild\n",file.file_path().c_str());
            break;
        }
    }while(0);

    return ret;
}

/*
   return 0  already upload sucess
   return -1   error
 */
int WebUploader::UploadFile(string filePathName,string remotePath,ON_FTP_POST_FILE_PROGRESS fn,int sliceSize)
{
    int ret = -1;

    SliceFile file(this,fn,remotePath,sliceSize);

    ret = file.open(filePathName);
    if(ret!=0){
        printf("file open faild");
        return -1;
    }
    file.PrintInfo();


    ret = UploadFile(file);
    if(ret != 0){
        printf("upload file faild");
        goto end;
    }

end:
    file.close();

    return ret;
}

分析工具：

wireshark

参考链接：

1 web-uploader: http://fex.baidu.com/webuploader/ 百度

2 后台服务器： https://gitee.com/laminae_admin/webupload/

3 文件md5: https://www.shuzhiduo.com/A/6pdDlkoOdw/

4 libcurl上传文件 https://juejin.cn/post/6901995720740765704

5 libcurl curl_formadd https://wenku.baidu.com/view/db8e88c26194dd88d0d233d4b14e852458fb39d7.html?_wkts_=1675932603111&bdQuery=libcurl+curl_formadd+%E6%96%87%E4%BB%B6