minio 使用aws c++ 多线程下载

多线程下载"/>

minio 使用aws c++ 多线程下载

C++ 下载 Minio 文件

下载 aws-sdk-cpp

1. git clone --recurse-submodules GitHub - aws/aws-sdk-cpp: AWS SDK for C++

麒麟上编译AWS

cmake ../aws-sdk-cpp -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH=/home/ht706/aws/tcs/install -DCMAKE_INSTALL_PREFIX=/home/ht706/aws/tcs/install -DLibCrypto_INCLUDE_DIR=/home/ht706/aws/tcs/install/include -DBUILD_SHARED_LIBS=on -DNO_WGNU=OFF -DBUILD_ONLY="s3" -DENABLE_TESTING=OFF

多线程下载

// 只要设置HTTP请求头的Range属性, 就可以实现从指定位置开始下载
/*
表示头500个字节：Range: bytes=0-499
表示第二个500字节：Range: bytes=500-999
表示最后500个字节：Range: bytes=-500
表示500字节以后的范围：Range: bytes=500-
*/std::vector<MultiDownloadCtx> downloadParts;for (size_t i = 0; i < partNum; i++){std::stringstream rangeStream;rangeStream << "bytes=" << i * DOWNLOAD_DATA_SIZE_THRESHOLD << '-' <<((i == partNum - 1) ? filesize - 1: ((i + 1) * DOWNLOAD_DATA_SIZE_THRESHOLD - 1));downloadParts.emplace_back();downloadParts.back().m_partNumber = i;downloadParts.back().getObjectRequest.WithBucket(fromBucket).WithKey(objectFile).WithRange(rangeStream.str());}for (int i = 0; i < downloadParts.size(); i++){MultiDownloadCtx& ctx = downloadParts[i];m_threadPool->AddJob([&]()-> void {objectFile;int partSize = ctx.m_partNumber == partNum - 1 ?(int)(filesize -(size_t)ctx.m_partNumber * DOWNLOAD_DATA_SIZE_THRESHOLD): DOWNLOAD_DATA_SIZE_THRESHOLD;ByteArrayStreamBuf* buf = appender.GetBuffer(m_threadPool->GetThreadIdx());LogDEBUG("Start downloading part %d, range: %s, part size: %d\n",ctx.m_partNumber, ctx.getObjectRequest.GetRange().c_str(),partSize);// Clear the stream as aws sdk expects a clean stream returned from// response stream factory each time when retry on network errorsctx.getObjectRequest.SetResponseStreamFactory([&buf]()-> Aws::IOStream*{ buf->reset(); return Aws::New<Aws::IOStream>("SR_MULTI_PART_DOWNLOAD", buf); });Aws::S3::Model::GetObjectOutcome outcome = s3_client->GetObject(ctx.getObjectRequest);buf->updateSize(partSize);if (outcome.IsSuccess()){LogDEBUG("Download part %d succeed, download size: %d",ctx.m_partNumber, partSize);ctx.m_outcome = RemoteStorageRequestOutcome::SUCCESS;appender.WritePartToOutputStream(m_threadPool->GetThreadIdx(),ctx.m_partNumber);doDownloadStatus(objectFile, filesize, partSize);}else{ctx.m_outcome = handleError(outcome.GetError());}});}m_threadPool->WaitAll();

运行现象

视频

petal_20230328_164338

交流qq ：690759587  qq群：691104042