minio 使用aws c++ 多线程下载

eric_clz

已于 2023-03-28 16:49:24 修改

阅读量592

点赞数

分类专栏： aws 文章标签： c++ aws

于 2023-03-28 15:20:04 首次发布

本文链接：https://blog.csdn.net/m0_37826678/article/details/129817010

版权

aws 专栏收录该内容

1 篇文章 0 订阅

订阅专栏

C++ 下载 Minio 文件

下载 aws-sdk-cpp

git clone --recurse-submodules GitHub - aws/aws-sdk-cpp: AWS SDK for C++

麒麟上编译AWS

cmake ../aws-sdk-cpp -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=/home/ht706/aws/tcs/install -DCMAKE_INSTALL_PREFIX=/home/ht706/aws/tcs/install -DLibCrypto_INCLUDE_DIR=/home/ht706/aws/tcs/install/include -DBUILD_SHARED_LIBS=on -DNO_WGNU=OFF -DBUILD_ONLY="s3" -DENABLE_TESTING=OFF

多线程下载

// 只要设置HTTP请求头的Range属性, 就可以实现从指定位置开始下载
/*
表示头500个字节：Range: bytes=0-499
表示第二个500字节：Range: bytes=500-999
表示最后500个字节：Range: bytes=-500
表示500字节以后的范围：Range: bytes=500-
*/

    std::vector<MultiDownloadCtx> downloadParts;
    for (size_t i = 0; i < partNum; i++)
    {
        std::stringstream rangeStream;
        rangeStream << "bytes=" << i * DOWNLOAD_DATA_SIZE_THRESHOLD << '-' <<
            ((i == partNum - 1) ? filesize - 1
                : ((i + 1) * DOWNLOAD_DATA_SIZE_THRESHOLD - 1));

        downloadParts.emplace_back();
        downloadParts.back().m_partNumber = i;
        downloadParts.back().getObjectRequest
            .WithBucket(fromBucket)
            .WithKey(objectFile)
            .WithRange(rangeStream.str());
    }

    for (int i = 0; i < downloadParts.size(); i++)
    {
        MultiDownloadCtx& ctx = downloadParts[i];

        m_threadPool->AddJob([&]()-> void {

            objectFile;
            int partSize = ctx.m_partNumber == partNum - 1 ?
                (int)(filesize -(size_t)ctx.m_partNumber * DOWNLOAD_DATA_SIZE_THRESHOLD)
                : DOWNLOAD_DATA_SIZE_THRESHOLD;
            ByteArrayStreamBuf* buf = appender.GetBuffer(m_threadPool->GetThreadIdx());

            LogDEBUG("Start downloading part %d, range: %s, part size: %d\n",
                    ctx.m_partNumber, ctx.getObjectRequest.GetRange().c_str(),
                partSize);


            // Clear the stream as aws sdk expects a clean stream returned from
            // response stream factory each time when retry on network errors
            ctx.getObjectRequest.SetResponseStreamFactory([&buf]()-> Aws::IOStream*
                { buf->reset(); return Aws::New<Aws::IOStream>("SR_MULTI_PART_DOWNLOAD", buf); });

            Aws::S3::Model::GetObjectOutcome outcome = s3_client->GetObject(
                ctx.getObjectRequest);

            buf->updateSize(partSize);
            if (outcome.IsSuccess())
            {
                LogDEBUG("Download part %d succeed, download size: %d",
                    ctx.m_partNumber, partSize);
                ctx.m_outcome = RemoteStorageRequestOutcome::SUCCESS;
                appender.WritePartToOutputStream(m_threadPool->GetThreadIdx(),ctx.m_partNumber);

                doDownloadStatus(objectFile, filesize, partSize);
            }
            else
            {
                ctx.m_outcome = handleError(outcome.GetError());
            }
            });
    }

    m_threadPool->WaitAll();

运行现象