diff --git a/include/miniocpp/args.h b/include/miniocpp/args.h index 99a5c3a..bb0f277 100644 --- a/include/miniocpp/args.h +++ b/include/miniocpp/args.h @@ -148,9 +148,9 @@ struct CreateMultipartUploadArgs : public ObjectArgs { }; // struct CreateMultipartUploadArgs struct PutObjectBaseArgs : public ObjectWriteArgs { - long object_size = -1; + std::optional object_size; size_t part_size = 0; - long part_count = 0; + std::optional part_count; std::string content_type; PutObjectBaseArgs() = default; @@ -348,7 +348,8 @@ struct PutObjectArgs : public PutObjectBaseArgs { std::string checksum_crc64nvme; // CRC64NVME checksum for multipart uploads PutObjectArgs() = default; - PutObjectArgs(std::istream& stream, long object_size, long part_size); + PutObjectArgs(std::istream& stream, std::optional object_size, + size_t part_size); ~PutObjectArgs() = default; error::Error Validate(); diff --git a/include/miniocpp/utils.h b/include/miniocpp/utils.h index a838147..f224e5a 100644 --- a/include/miniocpp/utils.h +++ b/include/miniocpp/utils.h @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -121,8 +122,8 @@ std::string Crc64NvmeBase64(const char* data, size_t len); error::Error CheckBucketName(std::string_view bucket_name, bool strict = false); error::Error ReadPart(std::istream& stream, char* buf, size_t size, size_t& bytes_read); -error::Error CalcPartInfo(long object_size, size_t& part_size, - long& part_count); +error::Error CalcPartInfo(std::optional object_size, + size_t& part_size, std::optional& part_count); /** * UtcTime represents date and time in UTC timezone. diff --git a/src/args.cc b/src/args.cc index cf2af4a..66eee87 100644 --- a/src/args.cc +++ b/src/args.cc @@ -299,8 +299,9 @@ ListObjectVersionsArgs& ListObjectVersionsArgs::operator=( return this->operator=(ListObjectVersionsArgs(args)); } -PutObjectArgs::PutObjectArgs(std::istream& istream, long object_size, - long part_size) +PutObjectArgs::PutObjectArgs(std::istream& istream, + std::optional object_size, + size_t part_size) : stream(&istream) { this->object_size = object_size; this->part_size = part_size; @@ -317,7 +318,7 @@ error::Error PutObjectArgs::Validate() { if (!size.has_value()) { return error::Error("size must be set when buf is set"); } - if (object_size < 0) object_size = static_cast(*size); + if (!object_size.has_value()) object_size = static_cast(*size); } return utils::CalcPartInfo(object_size, part_size, part_count); } @@ -440,8 +441,7 @@ error::Error UploadObjectArgs::Validate() { } std::filesystem::path file_path = filename; - size_t obj_size = std::filesystem::file_size(file_path); - object_size = static_cast(obj_size); + object_size = static_cast(std::filesystem::file_size(file_path)); return utils::CalcPartInfo(object_size, part_size, part_count); } diff --git a/src/client.cc b/src/client.cc index ebbc393..120bb81 100644 --- a/src/client.cc +++ b/src/client.cc @@ -542,14 +542,14 @@ PutObjectResponse Client::PutObject(PutObjectArgs args, std::string& upload_id, } } - long object_size = args.object_size; + std::optional object_size = args.object_size; size_t part_size = args.part_size; size_t uploaded_size = 0; unsigned int part_number = 0; std::string one_byte; bool stop = false; std::list parts; - long part_count = args.part_count; + std::optional part_count = args.part_count; double uploaded_bytes = 0; // for progress double upload_speed = -1; // for progress @@ -557,9 +557,9 @@ PutObjectResponse Client::PutObject(PutObjectArgs args, std::string& upload_id, part_number++; size_t bytes_read = 0; - if (part_count > 0) { - if (part_number == part_count) { - part_size = object_size - uploaded_size; + if (part_count.has_value()) { + if (part_number == *part_count) { + part_size = *object_size - uploaded_size; stop = true; } @@ -596,7 +596,7 @@ PutObjectResponse Client::PutObject(PutObjectArgs args, std::string& upload_id, // If bytes read is less than or equals to part size, then we have reached // last part. if (bytes_read <= part_size) { - part_count = part_number; + part_count = std::optional(part_number); part_size = bytes_read; stop = true; } else { @@ -608,7 +608,7 @@ PutObjectResponse Client::PutObject(PutObjectArgs args, std::string& upload_id, uploaded_size += part_size; - if (part_count == 1) { + if (part_count.has_value() && *part_count == 1) { PutObjectApiArgs api_args; api_args.extra_query_params = args.extra_query_params; api_args.bucket = args.bucket; @@ -679,7 +679,8 @@ PutObjectResponse Client::PutObject(PutObjectArgs args, std::string& upload_id, } http::ProgressFunctionArgs actual_args; - actual_args.upload_total_bytes = static_cast(object_size); + actual_args.upload_total_bytes = + object_size ? static_cast(*object_size) : -1.0; actual_args.uploaded_bytes = uploaded_bytes + args.uploaded_bytes; actual_args.userdata = progress_userdata; return progressfunc(actual_args); @@ -702,7 +703,8 @@ PutObjectResponse Client::PutObject(PutObjectArgs args, std::string& upload_id, if (args.progressfunc != nullptr) { uploaded_bytes += static_cast(data.length()); http::ProgressFunctionArgs actual_args; - actual_args.upload_total_bytes = static_cast(object_size); + actual_args.upload_total_bytes = + object_size ? static_cast(*object_size) : -1.0; actual_args.uploaded_bytes = uploaded_bytes; actual_args.userdata = args.progress_userdata; if (!args.progressfunc(actual_args)) { @@ -989,7 +991,7 @@ PutObjectResponse Client::PutObject(PutObjectArgs args) { std::stringstream ss(std::ios_base::in | std::ios_base::out); ss.rdbuf()->pubsetbuf(args.buf, size); - PutObjectArgs http_args(ss, static_cast(size), 16 * 1024 * 1024L); + PutObjectArgs http_args(ss, static_cast(size), 16 * 1024 * 1024L); http_args.bucket = args.bucket; http_args.object = args.object; http_args.region = region; diff --git a/src/utils.cc b/src/utils.cc index e033c1d..324a9d3 100644 --- a/src/utils.cc +++ b/src/utils.cc @@ -18,6 +18,7 @@ #include "miniocpp/utils.h" #include +#include #include "miniocpp/error.h" @@ -662,8 +663,9 @@ error::Error ReadPart(std::istream& stream, char* buf, size_t size, return error::SUCCESS; } -error::Error CalcPartInfo(long object_size, size_t& part_size, - long& part_count) { +error::Error CalcPartInfo(std::optional object_size, + size_t& part_size, + std::optional& part_count) { if (part_size > 0) { if (part_size < kMinPartSize) { return error::Error("part size " + std::to_string(part_size) + @@ -676,9 +678,9 @@ error::Error CalcPartInfo(long object_size, size_t& part_size, } } - if (object_size >= 0) { - if (static_cast(object_size) > kMaxObjectSize) { - return error::Error("object size " + std::to_string(object_size) + + if (object_size.has_value()) { + if (*object_size > kMaxObjectSize) { + return error::Error("object size " + std::to_string(*object_size) + " is not supported; maximum allowed 5TiB"); } } else if (part_size <= 0) { @@ -686,23 +688,23 @@ error::Error CalcPartInfo(long object_size, size_t& part_size, "valid part size must be provided when object size is unknown"); } - if (object_size < 0) { - part_count = -1; + if (!object_size.has_value()) { + part_count.reset(); return error::SUCCESS; } if (part_size <= 0) { // Calculate part size by multiple of kOptPartSize. - double psize = std::ceil((double)object_size / kMaxMultipartCount); + double psize = std::ceil((double)*object_size / kMaxMultipartCount); part_size = (size_t)std::ceil(psize / kMinPartSize) * kMinPartSize; } - if (static_cast(part_size) > object_size) part_size = object_size; - part_count = static_cast( - (part_size > 0) ? ((object_size + part_size - 1) / part_size) : 1); - if (part_count > kMaxMultipartCount) { + if (part_size > *object_size) part_size = *object_size; + part_count = static_cast( + (part_size > 0) ? ((*object_size + part_size - 1) / part_size) : 1); + if (*part_count > kMaxMultipartCount) { return error::Error( - "object size " + std::to_string(object_size) + " and part size " + + "object size " + std::to_string(*object_size) + " and part size " + std::to_string(part_size) + " make more than " + std::to_string(kMaxMultipartCount) + "parts for upload"); } diff --git a/tests/tests.cc b/tests/tests.cc index e1d3866..bddd60c 100644 --- a/tests/tests.cc +++ b/tests/tests.cc @@ -276,7 +276,7 @@ class Tests { std::string data = "StatObject()"; std::stringstream ss(data); - minio::s3::PutObjectArgs args(ss, static_cast(data.length()), 0); + minio::s3::PutObjectArgs args(ss, static_cast(data.length()), 0); args.bucket = bucket_name_; args.object = object_name; minio::s3::PutObjectResponse resp = client_.PutObject(args); @@ -309,7 +309,7 @@ class Tests { std::string object_name = RandObjectName(); std::string data = "RemoveObject()"; std::stringstream ss(data); - minio::s3::PutObjectArgs args(ss, static_cast(data.length()), 0); + minio::s3::PutObjectArgs args(ss, static_cast(data.length()), 0); args.bucket = bucket_name_; args.object = object_name; minio::s3::PutObjectResponse resp = client_.PutObject(args); @@ -326,7 +326,7 @@ class Tests { std::string data = "DownloadObject()"; std::stringstream ss(data); - minio::s3::PutObjectArgs args(ss, static_cast(data.length()), 0); + minio::s3::PutObjectArgs args(ss, static_cast(data.length()), 0); args.bucket = bucket_name_; args.object = object_name; minio::s3::PutObjectResponse resp = client_.PutObject(args); @@ -372,7 +372,7 @@ class Tests { std::string data = "GetObject()"; std::stringstream ss(data); - minio::s3::PutObjectArgs args(ss, static_cast(data.length()), 0); + minio::s3::PutObjectArgs args(ss, static_cast(data.length()), 0); args.bucket = bucket_name_; args.object = object_name; minio::s3::PutObjectResponse resp = client_.PutObject(args); @@ -461,7 +461,8 @@ class Tests { std::string object_name = RandObjectName(); std::string data = "PutObject()"; std::stringstream ss(data); - minio::s3::PutObjectArgs args(ss, data.length(), 0); + minio::s3::PutObjectArgs args(ss, static_cast(data.length()), + 0); args.bucket = bucket_name_; args.object = object_name; minio::s3::PutObjectResponse resp = client_.PutObject(args); @@ -475,7 +476,7 @@ class Tests { std::string object_name = RandObjectName(); size_t size = 67108865; // (64MiB + 1) bytes RandCharStream stream(size); - minio::s3::PutObjectArgs args(stream, size, 0); + minio::s3::PutObjectArgs args(stream, static_cast(size), 0); args.bucket = bucket_name_; args.object = object_name; minio::s3::PutObjectResponse resp = client_.PutObject(args); @@ -497,7 +498,7 @@ class Tests { std::string src_object_name = RandObjectName(); std::string data = "CopyObject()"; std::stringstream ss(data); - minio::s3::PutObjectArgs args(ss, static_cast(data.length()), 0); + minio::s3::PutObjectArgs args(ss, static_cast(data.length()), 0); args.bucket = bucket_name_; args.object = src_object_name; minio::s3::PutObjectResponse resp = client_.PutObject(args); @@ -584,7 +585,8 @@ class Tests { "1996,Jeep,Grand Cherokee,\"MUST SELL!\n" "air, moon roof, loaded\",4799.00\n"; std::stringstream ss("Year,Make,Model,Description,Price\n" + data); - minio::s3::PutObjectArgs args(ss, static_cast(ss.str().length()), 0); + minio::s3::PutObjectArgs args(ss, static_cast(ss.str().length()), + 0); args.bucket = bucket_name_; args.object = object_name; minio::s3::PutObjectResponse resp = client_.PutObject(args); @@ -664,7 +666,8 @@ class Tests { try { std::string data = "ListenBucketNotification()"; std::stringstream ss(data); - minio::s3::PutObjectArgs args(ss, static_cast(data.length()), 0); + minio::s3::PutObjectArgs args(ss, static_cast(data.length()), + 0); args.bucket = bucket_name_; args.object = object_name; minio::s3::PutObjectResponse resp = client_.PutObject(args);