#include "S3DataAdaptor.h" #include #include #include #include #include #include #define STRINGIFY_HELPER(val) #val #define STRINGIFY(val) STRINGIFY_HELPER(val) #define AWS_ALLOCATE_TAG __FILE__ ":" STRINGIFY(__LINE__) std::once_flag S3INIT_FLAG; std::once_flag S3SHUTDOWN_FLAG; Aws::SDKOptions AWS_SDK_OPTIONS; // https://github.com/aws/aws-sdk-cpp/issues/1430 class PreallocatedIOStream : public Aws::IOStream { public: PreallocatedIOStream(char *buf, size_t size) : Aws::IOStream(new Aws::Utils::Stream::PreallocatedStreamBuf( reinterpret_cast(buf), size)) {} PreallocatedIOStream(const char *buf, size_t size) : PreallocatedIOStream(const_cast(buf), size) {} ~PreallocatedIOStream() { // corresponding new in constructor delete rdbuf(); } }; Aws::String GetObjectRequestRange(uint64_t offset, uint64_t len) { auto range = "bytes=" + std::to_string(offset) + "-" + std::to_string(offset + len); return {range.data(), range.size()}; } S3DataAdaptor::S3DataAdaptor() { auto initSDK = [&]() { Aws::InitAPI(AWS_SDK_OPTIONS); }; std::call_once(S3INIT_FLAG, initSDK); auto &s3_config = GetGlobalConfig().s3_config; setenv("AWS_EC2_METADATA_DISABLED", "true", 1); clientCfg_ = Aws::New(AWS_ALLOCATE_TAG, true); clientCfg_->scheme = Aws::Http::Scheme::HTTP; clientCfg_->verifySSL = false; clientCfg_->maxConnections = 10; clientCfg_->endpointOverride = s3_config.address; clientCfg_->executor = Aws::MakeShared("S3Adapter.S3Client", s3_config.bg_threads); s3Client_ = Aws::New(AWS_ALLOCATE_TAG, Aws::Auth::AWSCredentials(s3_config.access_key, s3_config.secret_access_key), *clientCfg_, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, false); } S3DataAdaptor::~S3DataAdaptor() { if (clientCfg_ != nullptr) { Aws::Delete(clientCfg_); clientCfg_ = nullptr; } if (s3Client_ != nullptr) { Aws::Delete(s3Client_); s3Client_ = nullptr; } auto shutdownSDK = [&]() { Aws::ShutdownAPI(AWS_SDK_OPTIONS); }; std::call_once(S3SHUTDOWN_FLAG, shutdownSDK); } folly::Future S3DataAdaptor::DownLoad(const std::string &key, size_t start, size_t size, ByteBuffer &buffer) { Aws::S3::Model::GetObjectRequest request; request.SetBucket(GetGlobalConfig().s3_config.bucket); request.SetKey(Aws::String{key.c_str(), key.size()}); request.SetRange(GetObjectRequestRange(start, size)); request.SetResponseStreamFactory( [&buffer]() { return Aws::New(AWS_ALLOCATE_TAG, buffer.data, buffer.len); }); auto promise = std::make_shared < folly::Promise < int >> (); Aws::S3::GetObjectResponseReceivedHandler handler = [&buffer, size, promise]( const Aws::S3::S3Client */*client*/, const Aws::S3::Model::GetObjectRequest &/*request*/, const Aws::S3::Model::GetObjectOutcome &response, const std::shared_ptr &awsCtx) { if (response.IsSuccess()) { promise->setValue(OK); } else if (response.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY) { promise->setValue(NOT_FOUND); } else { LOG(ERROR) << "GetObjectAsync error: " << response.GetError().GetExceptionName() << "message: " << response.GetError().GetMessage(); promise->setValue(S3_INTERNAL_ERROR); } }; s3Client_->GetObjectAsync(request, handler, nullptr); return promise->getFuture(); } folly::Future S3DataAdaptor::UpLoad(const std::string &key, size_t size, const ByteBuffer &buffer, const std::map &headers) { Aws::S3::Model::PutObjectRequest request; request.SetBucket(GetGlobalConfig().s3_config.bucket); request.SetKey(key); request.SetMetadata(headers); request.SetBody(Aws::MakeShared(AWS_ALLOCATE_TAG, buffer.data, buffer.len)); auto promise = std::make_shared < folly::Promise < int >> (); Aws::S3::PutObjectResponseReceivedHandler handler = [promise]( const Aws::S3::S3Client */*client*/, const Aws::S3::Model::PutObjectRequest &/*request*/, const Aws::S3::Model::PutObjectOutcome &response, const std::shared_ptr &awsCtx) { LOG_IF(ERROR, !response.IsSuccess()) << "PutObjectAsync error: " << response.GetError().GetExceptionName() << "message: " << response.GetError().GetMessage(); promise->setValue(response.IsSuccess() ? OK : S3_INTERNAL_ERROR); }; s3Client_->PutObjectAsync(request, handler, nullptr); return promise->getFuture(); } folly::Future S3DataAdaptor::Delete(const std::string &key) { Aws::S3::Model::DeleteObjectRequest request; request.SetBucket(GetGlobalConfig().s3_config.bucket); request.SetKey(key); auto promise = std::make_shared < folly::Promise < int >> (); Aws::S3::DeleteObjectResponseReceivedHandler handler = [promise]( const Aws::S3::S3Client */*client*/, const Aws::S3::Model::DeleteObjectRequest &/*request*/, const Aws::S3::Model::DeleteObjectOutcome &response, const std::shared_ptr &awsCtx) { if (response.IsSuccess()) { promise->setValue(OK); } else if (response.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY) { promise->setValue(NOT_FOUND); } else { LOG(ERROR) << "DeleteObjectAsync error: " << response.GetError().GetExceptionName() << "message: " << response.GetError().GetMessage(); promise->setValue(S3_INTERNAL_ERROR); } }; s3Client_->DeleteObjectAsync(request, handler, nullptr); return promise->getFuture(); } folly::Future S3DataAdaptor::Head(const std::string &key, size_t &size, std::map &headers) { Aws::S3::Model::HeadObjectRequest request; request.SetBucket(GetGlobalConfig().s3_config.bucket); request.SetKey(key); auto promise = std::make_shared < folly::Promise < int >> (); Aws::S3::HeadObjectResponseReceivedHandler handler = [promise, &size, &headers]( const Aws::S3::S3Client */*client*/, const Aws::S3::Model::HeadObjectRequest &/*request*/, const Aws::S3::Model::HeadObjectOutcome &response, const std::shared_ptr &awsCtx) { if (response.IsSuccess()) { headers = response.GetResult().GetMetadata(); size = response.GetResult().GetContentLength(); promise->setValue(OK); } else if (response.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY) { promise->setValue(NOT_FOUND); } else { LOG(ERROR) << "HeadObjectAsync error: " << response.GetError().GetExceptionName() << "message: " << response.GetError().GetMessage(); promise->setValue(S3_INTERNAL_ERROR); } }; s3Client_->HeadObjectAsync(request, handler, nullptr); return promise->getFuture(); }