From 4e1449b10497b240f3241d8d16632e0f7d3340f2 Mon Sep 17 00:00:00 2001 From: Vaibhav Pratap Date: Wed, 15 Jul 2026 10:43:52 +0000 Subject: [PATCH 1/5] feat(storage): migrate sync client to unified checksum options --- .../cloud/storage/internal/hash_function.cc | 33 ++++++++++++++---- .../cloud/storage/internal/hash_validator.cc | 34 ++++++++++++++----- .../cloud/storage/internal/object_requests.cc | 17 ++++++++-- 3 files changed, 68 insertions(+), 16 deletions(-) diff --git a/google/cloud/storage/internal/hash_function.cc b/google/cloud/storage/internal/hash_function.cc index 0a5b5a77c463d..ce32f63d5262f 100644 --- a/google/cloud/storage/internal/hash_function.cc +++ b/google/cloud/storage/internal/hash_function.cc @@ -14,6 +14,8 @@ #include "google/cloud/storage/internal/hash_function.h" #include "google/cloud/storage/internal/hash_function_impl.h" +#include "google/cloud/options.h" +#include "google/cloud/storage/options.h" #include "google/cloud/storage/internal/object_requests.h" #include #include @@ -61,9 +63,17 @@ std::unique_ptr CreateHashFunction( ReadObjectRangeRequest const& request) { if (request.RequiresRangeHeader()) return CreateNullHashFunction(); - auto const disable_crc32c = - request.GetOption().value_or(false); - auto const disable_md5 = request.GetOption().value_or(false); + bool disable_md5 = false; + bool disable_crc32c = false; + auto const& options = google::cloud::internal::CurrentOptions(); + if (options.has()) { + auto const algo = options.get(); + disable_md5 = (algo != ChecksumAlgorithm::kMD5); + disable_crc32c = (algo != ChecksumAlgorithm::kCrc32c); + } else { + disable_md5 = request.GetOption().value_or(false); + disable_crc32c = request.GetOption().value_or(false); + } if (disable_md5 && disable_crc32c) { return std::make_unique(); } @@ -80,10 +90,21 @@ std::unique_ptr CreateHashFunction( // for previous values is lost. return CreateNullHashFunction(); } + + DisableCrc32cChecksum disable_crc32c; + DisableMD5Hash disable_md5; + auto const& options = google::cloud::internal::CurrentOptions(); + if (options.has()) { + auto const algo = options.get(); + disable_md5 = DisableMD5Hash(algo != ChecksumAlgorithm::kMD5); + disable_crc32c = DisableCrc32cChecksum(algo != ChecksumAlgorithm::kCrc32c); + } else { + disable_md5 = request.GetOption(); + disable_crc32c = request.GetOption(); + } return CreateHashFunction(request.GetOption(), - request.GetOption(), - request.GetOption(), - request.GetOption()); + disable_crc32c, request.GetOption(), + disable_md5); } } // namespace internal diff --git a/google/cloud/storage/internal/hash_validator.cc b/google/cloud/storage/internal/hash_validator.cc index 3df1899b33699..efd6fb5ea6899 100644 --- a/google/cloud/storage/internal/hash_validator.cc +++ b/google/cloud/storage/internal/hash_validator.cc @@ -14,6 +14,8 @@ #include "google/cloud/storage/internal/hash_validator.h" #include "google/cloud/storage/internal/hash_validator_impl.h" +#include "google/cloud/options.h" +#include "google/cloud/storage/options.h" #include "google/cloud/storage/internal/object_requests.h" #include "google/cloud/storage/object_metadata.h" #include @@ -52,19 +54,35 @@ std::unique_ptr CreateHashValidator( ReadObjectRangeRequest const& request) { if (request.RequiresRangeHeader()) return CreateNullHashValidator(); - // `DisableMD5Hash`'s default value is `true`. - auto disable_md5 = request.GetOption().value_or(false); - auto disable_crc32c = - request.GetOption().value_or(false); + bool disable_md5 = false; + bool disable_crc32c = false; + auto const& options = google::cloud::internal::CurrentOptions(); + if (options.has()) { + auto const algo = + options.get(); + disable_md5 = (algo != ChecksumAlgorithm::kMD5); + disable_crc32c = (algo != ChecksumAlgorithm::kCrc32c); + } else { + disable_md5 = request.GetOption().value_or(false); + disable_crc32c = request.GetOption().value_or(false); + } return CreateHashValidator(disable_md5, disable_crc32c); } std::unique_ptr CreateHashValidator( ResumableUploadRequest const& request) { - // `DisableMD5Hash`'s default value is `true`. - auto disable_md5 = request.GetOption().value_or(false); - auto disable_crc32c = - request.GetOption().value_or(false); + bool disable_md5 = false; + bool disable_crc32c = false; + auto const& options = google::cloud::internal::CurrentOptions(); + if (options.has()) { + auto const algo = + options.get(); + disable_md5 = (algo != ChecksumAlgorithm::kMD5); + disable_crc32c = (algo != ChecksumAlgorithm::kCrc32c); + } else { + disable_md5 = request.GetOption().value_or(false); + disable_crc32c = request.GetOption().value_or(false); + } return CreateHashValidator(disable_md5, disable_crc32c); } diff --git a/google/cloud/storage/internal/object_requests.cc b/google/cloud/storage/internal/object_requests.cc index 5ca59b9060e2d..6b94ef2c19f65 100644 --- a/google/cloud/storage/internal/object_requests.cc +++ b/google/cloud/storage/internal/object_requests.cc @@ -14,6 +14,8 @@ #include "google/cloud/storage/internal/object_requests.h" #include "google/cloud/storage/internal/binary_data_as_debug_string.h" +#include "google/cloud/storage/options.h" +#include "google/cloud/options.h" #include "google/cloud/storage/internal/metadata_parser.h" #include "google/cloud/storage/internal/object_acl_requests.h" #include "google/cloud/storage/internal/object_metadata_parser.h" @@ -218,9 +220,20 @@ InsertObjectMediaRequest::InsertObjectMediaRequest(std::string bucket_name, } void InsertObjectMediaRequest::reset_hash_function() { + DisableCrc32cChecksum disable_crc32c; + DisableMD5Hash disable_md5; + auto const& options = google::cloud::internal::CurrentOptions(); + if (options.has()) { + auto const algo = options.get(); + disable_md5 = DisableMD5Hash(algo != ChecksumAlgorithm::kMD5); + disable_crc32c = DisableCrc32cChecksum(algo != ChecksumAlgorithm::kCrc32c); + } else { + disable_md5 = GetOption(); + disable_crc32c = GetOption(); + } hash_function_ = CreateHashFunction( - GetOption(), GetOption(), - GetOption(), GetOption()); + GetOption(), disable_crc32c, + GetOption(), disable_md5); } void InsertObjectMediaRequest::set_payload(absl::string_view payload) { From d26f4af5fa527641bdd6cced22a1ef7c26077a48 Mon Sep 17 00:00:00 2001 From: Vaibhav Pratap Date: Fri, 17 Jul 2026 06:42:32 +0000 Subject: [PATCH 2/5] Address review comments for PR #16264 --- .../storage/google_cloud_cpp_storage.bzl | 1 + .../storage/google_cloud_cpp_storage.cmake | 1 + .../cloud/storage/internal/checksum_helpers.h | 65 +++++++++++++++++++ .../cloud/storage/internal/hash_function.cc | 40 +++++------- google/cloud/storage/internal/hash_function.h | 6 ++ .../cloud/storage/internal/hash_validator.cc | 41 +++++------- .../cloud/storage/internal/object_requests.cc | 30 ++++----- 7 files changed, 117 insertions(+), 67 deletions(-) create mode 100644 google/cloud/storage/internal/checksum_helpers.h diff --git a/google/cloud/storage/google_cloud_cpp_storage.bzl b/google/cloud/storage/google_cloud_cpp_storage.bzl index ed40ddb04ab5f..6d4f28b50754a 100644 --- a/google/cloud/storage/google_cloud_cpp_storage.bzl +++ b/google/cloud/storage/google_cloud_cpp_storage.bzl @@ -51,6 +51,7 @@ google_cloud_cpp_storage_hdrs = [ "internal/bucket_acl_requests.h", "internal/bucket_metadata_parser.h", "internal/bucket_requests.h", + "internal/checksum_helpers.h", "internal/complex_option.h", "internal/compute_engine_util.h", "internal/connection_factory.h", diff --git a/google/cloud/storage/google_cloud_cpp_storage.cmake b/google/cloud/storage/google_cloud_cpp_storage.cmake index f3d70b6766817..a72af1b4f1f0e 100644 --- a/google/cloud/storage/google_cloud_cpp_storage.cmake +++ b/google/cloud/storage/google_cloud_cpp_storage.cmake @@ -78,6 +78,7 @@ add_library( internal/bucket_metadata_parser.h internal/bucket_requests.cc internal/bucket_requests.h + internal/checksum_helpers.h internal/complex_option.h internal/compute_engine_util.cc internal/compute_engine_util.h diff --git a/google/cloud/storage/internal/checksum_helpers.h b/google/cloud/storage/internal/checksum_helpers.h new file mode 100644 index 0000000000000..19b5d439f0285 --- /dev/null +++ b/google/cloud/storage/internal/checksum_helpers.h @@ -0,0 +1,65 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_INTERNAL_CHECKSUM_HELPERS_H +#define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_INTERNAL_CHECKSUM_HELPERS_H + +#include "google/cloud/internal/disable_deprecation_warnings.inc" +#include "google/cloud/options.h" +#include "google/cloud/storage/hashing_options.h" +#include "google/cloud/storage/options.h" + +namespace google { +namespace cloud { +namespace storage { +GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN +namespace internal { + +struct HashDisabled { + bool md5; + bool crc32c; +}; + +template +HashDisabled GetDownloadChecksumSettings(Request const& request, + Options const& options) { + if (options.has()) { + auto const algo = options.get(); + return {algo != ChecksumAlgorithm::kMD5, + algo != ChecksumAlgorithm::kCrc32c}; + } + return {request.template GetOption().value_or(false), + request.template GetOption().value_or(false)}; +} + +template +HashDisabled GetUploadChecksumSettings(Request const& request, + Options const& options) { + if (options.has()) { + auto const algo = options.get(); + return {algo != ChecksumAlgorithm::kMD5, + algo != ChecksumAlgorithm::kCrc32c}; + } + return {request.template GetOption().value_or(false), + request.template GetOption().value_or(false)}; +} + +} // namespace internal +GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END +} // namespace storage +} // namespace cloud +} // namespace google + +#include "google/cloud/internal/diagnostics_pop.inc" +#endif // GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_INTERNAL_CHECKSUM_HELPERS_H diff --git a/google/cloud/storage/internal/hash_function.cc b/google/cloud/storage/internal/hash_function.cc index ce32f63d5262f..c79f7421386e9 100644 --- a/google/cloud/storage/internal/hash_function.cc +++ b/google/cloud/storage/internal/hash_function.cc @@ -12,11 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/internal/hash_function.h" #include "google/cloud/storage/internal/hash_function_impl.h" -#include "google/cloud/options.h" -#include "google/cloud/storage/options.h" +#include "google/cloud/storage/internal/checksum_helpers.h" #include "google/cloud/storage/internal/object_requests.h" +#include "google/cloud/storage/options.h" +#include "google/cloud/options.h" #include #include @@ -63,17 +65,10 @@ std::unique_ptr CreateHashFunction( ReadObjectRangeRequest const& request) { if (request.RequiresRangeHeader()) return CreateNullHashFunction(); - bool disable_md5 = false; - bool disable_crc32c = false; - auto const& options = google::cloud::internal::CurrentOptions(); - if (options.has()) { - auto const algo = options.get(); - disable_md5 = (algo != ChecksumAlgorithm::kMD5); - disable_crc32c = (algo != ChecksumAlgorithm::kCrc32c); - } else { - disable_md5 = request.GetOption().value_or(false); - disable_crc32c = request.GetOption().value_or(false); - } + auto const settings = GetDownloadChecksumSettings( + request, google::cloud::internal::CurrentOptions()); + auto const disable_md5 = settings.md5; + auto const disable_crc32c = settings.crc32c; if (disable_md5 && disable_crc32c) { return std::make_unique(); } @@ -90,18 +85,11 @@ std::unique_ptr CreateHashFunction( // for previous values is lost. return CreateNullHashFunction(); } - - DisableCrc32cChecksum disable_crc32c; - DisableMD5Hash disable_md5; - auto const& options = google::cloud::internal::CurrentOptions(); - if (options.has()) { - auto const algo = options.get(); - disable_md5 = DisableMD5Hash(algo != ChecksumAlgorithm::kMD5); - disable_crc32c = DisableCrc32cChecksum(algo != ChecksumAlgorithm::kCrc32c); - } else { - disable_md5 = request.GetOption(); - disable_crc32c = request.GetOption(); - } + + auto const settings = GetUploadChecksumSettings( + request, google::cloud::internal::CurrentOptions()); + auto disable_md5 = DisableMD5Hash(settings.md5); + auto disable_crc32c = DisableCrc32cChecksum(settings.crc32c); return CreateHashFunction(request.GetOption(), disable_crc32c, request.GetOption(), disable_md5); @@ -112,3 +100,5 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END } // namespace storage } // namespace cloud } // namespace google + +#include "google/cloud/internal/diagnostics_pop.inc" diff --git a/google/cloud/storage/internal/hash_function.h b/google/cloud/storage/internal/hash_function.h index 0e17442acce05..a8c7819f4352f 100644 --- a/google/cloud/storage/internal/hash_function.h +++ b/google/cloud/storage/internal/hash_function.h @@ -19,6 +19,7 @@ #include "google/cloud/storage/hashing_options.h" #include "google/cloud/storage/internal/hash_values.h" #include "google/cloud/storage/version.h" +#include "google/cloud/options.h" #include "google/cloud/status.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" @@ -29,6 +30,11 @@ namespace google { namespace cloud { namespace storage { GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN + +struct DownloadChecksumValidationOption; +struct UploadChecksumValidationOption; +enum class ChecksumAlgorithm; + namespace internal { class ReadObjectRangeRequest; diff --git a/google/cloud/storage/internal/hash_validator.cc b/google/cloud/storage/internal/hash_validator.cc index efd6fb5ea6899..931e108dca6c7 100644 --- a/google/cloud/storage/internal/hash_validator.cc +++ b/google/cloud/storage/internal/hash_validator.cc @@ -12,12 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/internal/hash_validator.h" +#include "google/cloud/storage/internal/checksum_helpers.h" +#include "google/cloud/storage/internal/hash_function.h" #include "google/cloud/storage/internal/hash_validator_impl.h" -#include "google/cloud/options.h" -#include "google/cloud/storage/options.h" #include "google/cloud/storage/internal/object_requests.h" #include "google/cloud/storage/object_metadata.h" +#include "google/cloud/storage/options.h" +#include "google/cloud/options.h" #include #include @@ -54,35 +57,19 @@ std::unique_ptr CreateHashValidator( ReadObjectRangeRequest const& request) { if (request.RequiresRangeHeader()) return CreateNullHashValidator(); - bool disable_md5 = false; - bool disable_crc32c = false; - auto const& options = google::cloud::internal::CurrentOptions(); - if (options.has()) { - auto const algo = - options.get(); - disable_md5 = (algo != ChecksumAlgorithm::kMD5); - disable_crc32c = (algo != ChecksumAlgorithm::kCrc32c); - } else { - disable_md5 = request.GetOption().value_or(false); - disable_crc32c = request.GetOption().value_or(false); - } + auto const settings = GetDownloadChecksumSettings( + request, google::cloud::internal::CurrentOptions()); + auto const disable_md5 = settings.md5; + auto const disable_crc32c = settings.crc32c; return CreateHashValidator(disable_md5, disable_crc32c); } std::unique_ptr CreateHashValidator( ResumableUploadRequest const& request) { - bool disable_md5 = false; - bool disable_crc32c = false; - auto const& options = google::cloud::internal::CurrentOptions(); - if (options.has()) { - auto const algo = - options.get(); - disable_md5 = (algo != ChecksumAlgorithm::kMD5); - disable_crc32c = (algo != ChecksumAlgorithm::kCrc32c); - } else { - disable_md5 = request.GetOption().value_or(false); - disable_crc32c = request.GetOption().value_or(false); - } + auto const settings = GetUploadChecksumSettings( + request, google::cloud::internal::CurrentOptions()); + auto const disable_md5 = settings.md5; + auto const disable_crc32c = settings.crc32c; return CreateHashValidator(disable_md5, disable_crc32c); } @@ -99,3 +86,5 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END } // namespace storage } // namespace cloud } // namespace google + +#include "google/cloud/internal/diagnostics_pop.inc" diff --git a/google/cloud/storage/internal/object_requests.cc b/google/cloud/storage/internal/object_requests.cc index 6b94ef2c19f65..d2715c3a2f2a9 100644 --- a/google/cloud/storage/internal/object_requests.cc +++ b/google/cloud/storage/internal/object_requests.cc @@ -12,14 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/internal/object_requests.h" +#include "google/cloud/storage/internal/checksum_helpers.h" #include "google/cloud/storage/internal/binary_data_as_debug_string.h" -#include "google/cloud/storage/options.h" -#include "google/cloud/options.h" +#include "google/cloud/storage/internal/hash_function.h" #include "google/cloud/storage/internal/metadata_parser.h" #include "google/cloud/storage/internal/object_acl_requests.h" #include "google/cloud/storage/internal/object_metadata_parser.h" #include "google/cloud/storage/object_metadata.h" +#include "google/cloud/storage/options.h" +#include "google/cloud/options.h" #include "absl/strings/numbers.h" #include "absl/strings/str_split.h" #include @@ -220,20 +223,13 @@ InsertObjectMediaRequest::InsertObjectMediaRequest(std::string bucket_name, } void InsertObjectMediaRequest::reset_hash_function() { - DisableCrc32cChecksum disable_crc32c; - DisableMD5Hash disable_md5; - auto const& options = google::cloud::internal::CurrentOptions(); - if (options.has()) { - auto const algo = options.get(); - disable_md5 = DisableMD5Hash(algo != ChecksumAlgorithm::kMD5); - disable_crc32c = DisableCrc32cChecksum(algo != ChecksumAlgorithm::kCrc32c); - } else { - disable_md5 = GetOption(); - disable_crc32c = GetOption(); - } - hash_function_ = CreateHashFunction( - GetOption(), disable_crc32c, - GetOption(), disable_md5); + auto const settings = GetUploadChecksumSettings( + *this, google::cloud::internal::CurrentOptions()); + auto disable_md5 = DisableMD5Hash(settings.md5); + auto disable_crc32c = DisableCrc32cChecksum(settings.crc32c); + hash_function_ = + CreateHashFunction(GetOption(), disable_crc32c, + GetOption(), disable_md5); } void InsertObjectMediaRequest::set_payload(absl::string_view payload) { @@ -692,3 +688,5 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END } // namespace storage } // namespace cloud } // namespace google + +#include "google/cloud/internal/diagnostics_pop.inc" From 530f38bc71b64506b1eff9a5795ce327c30fd0fd Mon Sep 17 00:00:00 2001 From: Vaibhav Pratap Date: Mon, 20 Jul 2026 13:53:22 +0000 Subject: [PATCH 3/5] fix(storage): resolve check formatting and deprecation warnings in tests --- .../benchmarks/throughput_experiment.cc | 2 + google/cloud/storage/client.cc | 3 + google/cloud/storage/client_object_test.cc | 53 ++++++++ google/cloud/storage/hashing_options_test.cc | 2 + .../cloud/storage/internal/checksum_helpers.h | 32 +++-- .../cloud/storage/internal/connection_impl.cc | 2 + .../internal/grpc/object_request_parser.cc | 10 -- .../internal/grpc/object_request_parser.h | 5 - .../grpc/object_request_parser_test.cc | 118 +----------------- google/cloud/storage/internal/grpc/stub.cc | 18 ++- .../cloud/storage/internal/hash_function.cc | 2 +- .../internal/hash_function_impl_test.cc | 2 + .../storage/internal/hash_validator_test.cc | 2 + .../cloud/storage/internal/object_requests.cc | 23 +--- .../cloud/storage/internal/object_requests.h | 14 --- .../internal/object_write_streambuf_test.cc | 2 + google/cloud/storage/internal/rest/stub.cc | 22 +++- google/cloud/storage/options.h | 7 +- .../storage/testing/upload_hash_cases.cc | 3 + .../cloud/storage/testing/upload_hash_cases.h | 2 + .../tests/object_checksum_integration_test.cc | 2 + .../tests/object_file_integration_test.cc | 2 + .../tests/object_hash_integration_test.cc | 2 + .../tests/object_insert_integration_test.cc | 2 + 24 files changed, 148 insertions(+), 184 deletions(-) diff --git a/google/cloud/storage/benchmarks/throughput_experiment.cc b/google/cloud/storage/benchmarks/throughput_experiment.cc index 2611a280b43eb..aced424858d18 100644 --- a/google/cloud/storage/benchmarks/throughput_experiment.cc +++ b/google/cloud/storage/benchmarks/throughput_experiment.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/benchmarks/throughput_experiment.h" #include "google/cloud/storage/benchmarks/benchmark_utils.h" #include "google/cloud/storage/client.h" @@ -477,3 +478,4 @@ std::vector> CreateDownloadExperiments( } // namespace storage_benchmarks } // namespace cloud } // namespace google +#include "google/cloud/internal/diagnostics_pop.inc" diff --git a/google/cloud/storage/client.cc b/google/cloud/storage/client.cc index 2f88cae9363ff..d69f88346207a 100644 --- a/google/cloud/storage/client.cc +++ b/google/cloud/storage/client.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/client.h" #include "google/cloud/storage/idempotency_policy.h" #include "google/cloud/storage/internal/base64.h" @@ -699,3 +700,5 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END } // namespace storage } // namespace cloud } // namespace google + +#include "google/cloud/internal/diagnostics_pop.inc" diff --git a/google/cloud/storage/client_object_test.cc b/google/cloud/storage/client_object_test.cc index 6b37ee7a7c4cc..f7cbccebc88a9 100644 --- a/google/cloud/storage/client_object_test.cc +++ b/google/cloud/storage/client_object_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/client.h" #include "google/cloud/storage/internal/object_metadata_parser.h" #include "google/cloud/storage/retry_policy.h" @@ -91,6 +92,32 @@ TEST_F(ObjectTest, InsertObjectMedia) { EXPECT_EQ(expected, *actual); } +TEST_F(ObjectTest, InsertObjectUploadChecksumMD5) { + std::string text = R"""({ + "name": "test-bucket-name/test-object-name/1" +})"""; + auto expected = + storage::internal::ObjectMetadataParser::FromString(text).value(); + + EXPECT_CALL(*mock_, InsertObjectMedia) + .WillOnce([&expected](internal::InsertObjectMediaRequest const& request) { + EXPECT_EQ("test-bucket-name", request.bucket_name()); + EXPECT_EQ("test-object-name", request.object_name()); + EXPECT_EQ("test object contents", request.payload()); + EXPECT_TRUE(CurrentOptions().has()); + EXPECT_EQ(CurrentOptions().get(), + ChecksumAlgorithm::kMD5); + return make_status_or(expected); + }); + + auto client = ClientForMock(); + auto actual = client.InsertObject( + "test-bucket-name", "test-object-name", "test object contents", + Options{}.set(ChecksumAlgorithm::kMD5)); + ASSERT_STATUS_OK(actual); + EXPECT_EQ(expected, *actual); +} + TEST_F(ObjectTest, GetObjectMetadata) { std::string text = R"""({ "bucket": "test-bucket-name", @@ -206,6 +233,30 @@ TEST_F(ObjectTest, ReadObject) { EXPECT_EQ(actual.gcount(), 1024); } +TEST_F(ObjectTest, ReadObjectChecksumPrecedence) { + EXPECT_CALL(*mock_, ReadObject) + .WillOnce([](internal::ReadObjectRangeRequest const& r) { + EXPECT_TRUE(r.HasOption()); + EXPECT_FALSE(r.GetOption().value()); + auto read_source = std::make_unique(); + EXPECT_CALL(*read_source, IsOpen()).WillRepeatedly(Return(true)); + EXPECT_CALL(*read_source, Read) + .WillOnce(Return(internal::ReadSourceResult{1024, {}})); + EXPECT_CALL(*read_source, Close).Times(1); + return StatusOr>( + std::move(read_source)); + }); + auto client = ClientForMock(); + auto actual = client.ReadObject( + "test-bucket-name", "test-object-name", DisableMD5Hash(false), + Options{}.set( + ChecksumAlgorithm::kCrc32c)); + ASSERT_STATUS_OK(actual.status()); + std::vector v(1024); + actual.read(v.data(), v.size()); + EXPECT_EQ(actual.gcount(), 1024); +} + TEST_F(ObjectTest, WriteObject) { EXPECT_CALL(*mock_, CreateResumableUpload) .WillOnce(Return(TransientError())) @@ -483,3 +534,5 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END } // namespace storage } // namespace cloud } // namespace google + +#include "google/cloud/internal/diagnostics_pop.inc" diff --git a/google/cloud/storage/hashing_options_test.cc b/google/cloud/storage/hashing_options_test.cc index 8bd3ec50dc388..a891da51d9688 100644 --- a/google/cloud/storage/hashing_options_test.cc +++ b/google/cloud/storage/hashing_options_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/hashing_options.h" #include "google/cloud/storage/options.h" #include "google/cloud/options.h" @@ -78,3 +79,4 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END } // namespace storage } // namespace cloud } // namespace google +#include "google/cloud/internal/diagnostics_pop.inc" diff --git a/google/cloud/storage/internal/checksum_helpers.h b/google/cloud/storage/internal/checksum_helpers.h index 19b5d439f0285..34c657c067ae0 100644 --- a/google/cloud/storage/internal/checksum_helpers.h +++ b/google/cloud/storage/internal/checksum_helpers.h @@ -16,9 +16,9 @@ #define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_INTERNAL_CHECKSUM_HELPERS_H #include "google/cloud/internal/disable_deprecation_warnings.inc" -#include "google/cloud/options.h" #include "google/cloud/storage/hashing_options.h" #include "google/cloud/storage/options.h" +#include "google/cloud/options.h" namespace google { namespace cloud { @@ -34,25 +34,39 @@ struct HashDisabled { template HashDisabled GetDownloadChecksumSettings(Request const& request, Options const& options) { + if (request.template HasOption() || + request.template HasOption()) { + return { + request.template GetOption().value_or(false), + request.template GetOption().value_or(false)}; + } if (options.has()) { auto const algo = options.get(); - return {algo != ChecksumAlgorithm::kMD5, - algo != ChecksumAlgorithm::kCrc32c}; + return {algo != ChecksumAlgorithm::kMD5 && + algo != ChecksumAlgorithm::kCrc32cAndMD5, + algo != ChecksumAlgorithm::kCrc32c && + algo != ChecksumAlgorithm::kCrc32cAndMD5}; } - return {request.template GetOption().value_or(false), - request.template GetOption().value_or(false)}; + return {false, false}; } template HashDisabled GetUploadChecksumSettings(Request const& request, Options const& options) { + if (request.template HasOption() || + request.template HasOption()) { + return { + request.template GetOption().value_or(false), + request.template GetOption().value_or(false)}; + } if (options.has()) { auto const algo = options.get(); - return {algo != ChecksumAlgorithm::kMD5, - algo != ChecksumAlgorithm::kCrc32c}; + return {algo != ChecksumAlgorithm::kMD5 && + algo != ChecksumAlgorithm::kCrc32cAndMD5, + algo != ChecksumAlgorithm::kCrc32c && + algo != ChecksumAlgorithm::kCrc32cAndMD5}; } - return {request.template GetOption().value_or(false), - request.template GetOption().value_or(false)}; + return {false, false}; } } // namespace internal diff --git a/google/cloud/storage/internal/connection_impl.cc b/google/cloud/storage/internal/connection_impl.cc index 6078d137b10d8..4e51345c18893 100644 --- a/google/cloud/storage/internal/connection_impl.cc +++ b/google/cloud/storage/internal/connection_impl.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/internal/connection_impl.h" #include "google/cloud/storage/internal/retry_object_read_source.h" #include "google/cloud/storage/parallel_upload.h" @@ -1392,3 +1393,4 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END } // namespace storage } // namespace cloud } // namespace google +#include "google/cloud/internal/diagnostics_pop.inc" diff --git a/google/cloud/storage/internal/grpc/object_request_parser.cc b/google/cloud/storage/internal/grpc/object_request_parser.cc index 23492acb5334c..83b2ed6b780f5 100644 --- a/google/cloud/storage/internal/grpc/object_request_parser.cc +++ b/google/cloud/storage/internal/grpc/object_request_parser.cc @@ -936,16 +936,6 @@ Status Finalize(google::storage::v2::BidiWriteObjectRequest& write_request, Merge(std::move(hashes), hash_function.Finish())); } -// If this is the last `Write()` call of the last `InsertObjectMedia()` set the -// flags to finalize the request -Status MaybeFinalize(google::storage::v2::WriteObjectRequest& write_request, - grpc::WriteOptions& options, - storage::internal::InsertObjectMediaRequest const& request, - bool chunk_has_more) { - if (chunk_has_more) return {}; - return Finalize(write_request, options, request.hash_function()); -} - // If this is the last `Write()` call of the last `UploadChunk()` set the flags // to finalize the request Status MaybeFinalize(google::storage::v2::WriteObjectRequest& write_request, diff --git a/google/cloud/storage/internal/grpc/object_request_parser.h b/google/cloud/storage/internal/grpc/object_request_parser.h index 771fac2890c0a..1e96da33a7d4f 100644 --- a/google/cloud/storage/internal/grpc/object_request_parser.h +++ b/google/cloud/storage/internal/grpc/object_request_parser.h @@ -96,11 +96,6 @@ Status Finalize(google::storage::v2::BidiWriteObjectRequest& write_request, storage::internal::HashFunction& hash_function, storage::internal::HashValues = {}); -Status MaybeFinalize(google::storage::v2::WriteObjectRequest& write_request, - grpc::WriteOptions& options, - storage::internal::InsertObjectMediaRequest const& request, - bool chunk_has_more); - Status MaybeFinalize(google::storage::v2::WriteObjectRequest& write_request, grpc::WriteOptions& options, storage::internal::UploadChunkRequest const& request, diff --git a/google/cloud/storage/internal/grpc/object_request_parser_test.cc b/google/cloud/storage/internal/grpc/object_request_parser_test.cc index 91cf1f079a14b..b8252c2de2f6e 100644 --- a/google/cloud/storage/internal/grpc/object_request_parser_test.cc +++ b/google/cloud/storage/internal/grpc/object_request_parser_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/internal/grpc/object_request_parser.h" #include "google/cloud/storage/internal/grpc/stub.h" #include "google/cloud/storage/internal/hash_function_impl.h" @@ -738,121 +739,6 @@ TEST(GrpcObjectRequestParser, InsertObjectMediaRequestSimple) { EXPECT_THAT(actual, IsProtoEqual(expected)); } -TEST(GrpcObjectRequestParser, MaybeFinalizeInsertObjectMediaRequest) { - using storage::internal::InsertObjectMediaRequest; - // See top-of-file comments for details on the magic numbers - struct Test { - std::function apply_options; - std::string expected_checksums; - } cases[] = { - // These tests provide the "wrong" hashes. This is what would happen if - // one was (for example) reading a GCS file, obtained the expected hashes - // from GCS, and then uploaded to another GCS destination *but* the data - // was somehow corrupted locally (say a bad disk). In that case, we don't - // want to recompute the hashes in the upload. - { - [](storage::internal::InsertObjectMediaRequest& r) { - r.set_option(storage::MD5HashValue(storage::ComputeMD5Hash(kText))); - r.set_option(storage::DisableCrc32cChecksum(true)); - }, - R"pb( - md5_hash: "\x9e\x10\x7d\x9d\x37\x2b\xb6\x82\x6b\xd8\x1d\x35\x42\xa4\x19\xd6")pb", - }, - { - [](InsertObjectMediaRequest& r) { - r.set_option(storage::MD5HashValue(storage::ComputeMD5Hash(kText))); - r.set_option(storage::DisableCrc32cChecksum(false)); - }, - R"pb( - md5_hash: "\x9e\x10\x7d\x9d\x37\x2b\xb6\x82\x6b\xd8\x1d\x35\x42\xa4\x19\xd6" - crc32c: 0x4ad67f80)pb", - }, - { - [](InsertObjectMediaRequest& r) { - r.set_option(storage::MD5HashValue(storage::ComputeMD5Hash(kText))); - r.set_option(storage::Crc32cChecksumValue( - storage::ComputeCrc32cChecksum(kText))); - }, - R"pb( - md5_hash: "\x9e\x10\x7d\x9d\x37\x2b\xb6\x82\x6b\xd8\x1d\x35\x42\xa4\x19\xd6" - crc32c: 0x22620404)pb", - }, - - { - [](InsertObjectMediaRequest& r) { - r.set_option(storage::DisableMD5Hash(false)); - r.set_option(storage::DisableCrc32cChecksum(true)); - }, - R"pb( - md5_hash: "\x4a\xd1\x2f\xa3\x65\x7f\xaa\x80\xc2\xb9\xa9\x2d\x65\x2c\x37\x21")pb", - }, - { - [](InsertObjectMediaRequest& r) { - r.set_option(storage::DisableMD5Hash(false)); - r.set_option(storage::DisableCrc32cChecksum(false)); - }, - R"pb( - md5_hash: "\x4a\xd1\x2f\xa3\x65\x7f\xaa\x80\xc2\xb9\xa9\x2d\x65\x2c\x37\x21" - crc32c: 0x4ad67f80)pb", - }, - { - [](InsertObjectMediaRequest& r) { - r.set_option(storage::DisableMD5Hash(false)); - r.set_option(storage::Crc32cChecksumValue( - storage::ComputeCrc32cChecksum(kText))); - }, - R"pb( - md5_hash: "\x4a\xd1\x2f\xa3\x65\x7f\xaa\x80\xc2\xb9\xa9\x2d\x65\x2c\x37\x21" - crc32c: 0x22620404)pb", - }, - - { - [](InsertObjectMediaRequest& r) { - r.set_option(storage::DisableMD5Hash(true)); - r.set_option(storage::DisableCrc32cChecksum(true)); - }, - R"pb( - )pb", - }, - { - [](InsertObjectMediaRequest& r) { - r.set_option(storage::DisableMD5Hash(true)); - r.set_option(storage::DisableCrc32cChecksum(false)); - }, - R"pb( - crc32c: 0x4ad67f80)pb", - }, - { - [](InsertObjectMediaRequest& r) { - r.set_option(storage::DisableMD5Hash(true)); - r.set_option(storage::Crc32cChecksumValue( - storage::ComputeCrc32cChecksum(kText))); - }, - R"pb( - crc32c: 0x22620404)pb", - }, - }; - for (auto const& test : cases) { - SCOPED_TRACE("Expected outcome " + test.expected_checksums); - storage_proto::ObjectChecksums expected; - ASSERT_TRUE( - TextFormat::ParseFromString(test.expected_checksums, &expected)); - - storage::internal::InsertObjectMediaRequest request( - "test-bucket-name", "test-object-name", kAlt); - test.apply_options(request); - request.set_multiple_options(); - request.hash_function().Update(0, kAlt); - storage_proto::WriteObjectRequest write_request; - grpc::WriteOptions write_options; - auto status = MaybeFinalize(write_request, write_options, request, false); - ASSERT_STATUS_OK(status); - EXPECT_TRUE(write_request.finish_write()); - EXPECT_TRUE(write_options.is_last_message()); - EXPECT_THAT(write_request.object_checksums(), IsProtoEqual(expected)); - } -} - TEST(GrpcObjectRequestParser, InsertObjectMediaRequestAllOptions) { auto constexpr kTextProto = R"pb( write_object_spec { @@ -1582,3 +1468,5 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END } // namespace storage_internal } // namespace cloud } // namespace google + +#include "google/cloud/internal/diagnostics_pop.inc" diff --git a/google/cloud/storage/internal/grpc/stub.cc b/google/cloud/storage/internal/grpc/stub.cc index e8edd1f2dc013..086da25985fed 100644 --- a/google/cloud/storage/internal/grpc/stub.cc +++ b/google/cloud/storage/internal/grpc/stub.cc @@ -12,7 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/internal/grpc/stub.h" +#include "google/cloud/storage/internal/checksum_helpers.h" #include "google/cloud/storage/internal/crc32c.h" #include "google/cloud/storage/internal/grpc/bucket_access_control_parser.h" #include "google/cloud/storage/internal/grpc/bucket_metadata_parser.h" @@ -346,6 +348,14 @@ StatusOr GrpcStub::InsertObjectMedia( ApplyRoutingHeaders(*ctx, request); auto stream = stub_->WriteObject(std::move(ctx), options); + auto const settings = + storage::internal::GetUploadChecksumSettings(request, options); + auto disable_md5 = storage::DisableMD5Hash(settings.md5); + auto disable_crc32c = storage::DisableCrc32cChecksum(settings.crc32c); + auto hash_function = storage::internal::CreateHashFunction( + request.GetOption(), disable_crc32c, + request.GetOption(), disable_md5); + auto splitter = SplitObjectWriteData(request.payload()); std::int64_t offset = 0; @@ -356,11 +366,13 @@ StatusOr GrpcStub::InsertObjectMedia( auto& data = *proto_request.mutable_checksummed_data(); SetMutableContent(data, splitter.Next()); data.set_crc32c(Crc32c(GetContent(data))); - request.hash_function().Update(offset, GetContent(data), data.crc32c()); + hash_function->Update(offset, GetContent(data), data.crc32c()); offset += GetContent(data).size(); auto wopts = grpc::WriteOptions{}; - MaybeFinalize(proto_request, wopts, request, !splitter.Done()); + if (splitter.Done()) { + storage_internal::Finalize(proto_request, wopts, *hash_function); + } auto watchdog = create_watchdog().then([&stream](auto f) { if (!f.get()) return false; @@ -1216,3 +1228,5 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END } // namespace storage_internal } // namespace cloud } // namespace google + +#include "google/cloud/internal/diagnostics_pop.inc" diff --git a/google/cloud/storage/internal/hash_function.cc b/google/cloud/storage/internal/hash_function.cc index c79f7421386e9..fdaf3f1782a27 100644 --- a/google/cloud/storage/internal/hash_function.cc +++ b/google/cloud/storage/internal/hash_function.cc @@ -14,8 +14,8 @@ #include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/internal/hash_function.h" -#include "google/cloud/storage/internal/hash_function_impl.h" #include "google/cloud/storage/internal/checksum_helpers.h" +#include "google/cloud/storage/internal/hash_function_impl.h" #include "google/cloud/storage/internal/object_requests.h" #include "google/cloud/storage/options.h" #include "google/cloud/options.h" diff --git a/google/cloud/storage/internal/hash_function_impl_test.cc b/google/cloud/storage/internal/hash_function_impl_test.cc index e1668a1c498d5..0d4949019be27 100644 --- a/google/cloud/storage/internal/hash_function_impl_test.cc +++ b/google/cloud/storage/internal/hash_function_impl_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/internal/hash_function_impl.h" #include "google/cloud/storage/internal/crc32c.h" #include "google/cloud/storage/internal/object_requests.h" @@ -455,3 +456,4 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END } // namespace storage } // namespace cloud } // namespace google +#include "google/cloud/internal/diagnostics_pop.inc" diff --git a/google/cloud/storage/internal/hash_validator_test.cc b/google/cloud/storage/internal/hash_validator_test.cc index 51bef8a742995..212e7f996bfa7 100644 --- a/google/cloud/storage/internal/hash_validator_test.cc +++ b/google/cloud/storage/internal/hash_validator_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/internal/hash_validator.h" #include "google/cloud/storage/internal/hash_function_impl.h" #include "google/cloud/storage/internal/hash_validator_impl.h" @@ -245,3 +246,4 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END } // namespace storage } // namespace cloud } // namespace google +#include "google/cloud/internal/diagnostics_pop.inc" diff --git a/google/cloud/storage/internal/object_requests.cc b/google/cloud/storage/internal/object_requests.cc index d2715c3a2f2a9..8a7616c4152f4 100644 --- a/google/cloud/storage/internal/object_requests.cc +++ b/google/cloud/storage/internal/object_requests.cc @@ -14,8 +14,8 @@ #include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/internal/object_requests.h" -#include "google/cloud/storage/internal/checksum_helpers.h" #include "google/cloud/storage/internal/binary_data_as_debug_string.h" +#include "google/cloud/storage/internal/checksum_helpers.h" #include "google/cloud/storage/internal/hash_function.h" #include "google/cloud/storage/internal/metadata_parser.h" #include "google/cloud/storage/internal/object_acl_requests.h" @@ -211,26 +211,14 @@ std::ostream& operator<<(std::ostream& os, GetObjectMetadataRequest const& r) { return os << "}"; } -InsertObjectMediaRequest::InsertObjectMediaRequest() { reset_hash_function(); } +InsertObjectMediaRequest::InsertObjectMediaRequest() = default; InsertObjectMediaRequest::InsertObjectMediaRequest(std::string bucket_name, std::string object_name, absl::string_view payload) : InsertObjectRequestImpl(std::move(bucket_name), std::move(object_name)), - payload_(payload) { - reset_hash_function(); -} - -void InsertObjectMediaRequest::reset_hash_function() { - auto const settings = GetUploadChecksumSettings( - *this, google::cloud::internal::CurrentOptions()); - auto disable_md5 = DisableMD5Hash(settings.md5); - auto disable_crc32c = DisableCrc32cChecksum(settings.crc32c); - hash_function_ = - CreateHashFunction(GetOption(), disable_crc32c, - GetOption(), disable_md5); -} + payload_(payload) {} void InsertObjectMediaRequest::set_payload(absl::string_view payload) { payload_ = payload; @@ -249,11 +237,6 @@ void InsertObjectMediaRequest::set_contents(std::string v) { payload_ = contents_; dirty_ = false; } - -HashValues FinishHashes(InsertObjectMediaRequest const& request) { - return request.hash_function().Finish(); -} - std::ostream& operator<<(std::ostream& os, InsertObjectMediaRequest const& r) { os << "InsertObjectMediaRequest={bucket_name=" << r.bucket_name() << ", object_name=" << r.object_name(); diff --git a/google/cloud/storage/internal/object_requests.h b/google/cloud/storage/internal/object_requests.h index edec5ebe9c963..532d0dc167d2e 100644 --- a/google/cloud/storage/internal/object_requests.h +++ b/google/cloud/storage/internal/object_requests.h @@ -129,15 +129,6 @@ class InsertObjectMediaRequest absl::string_view payload() const { return payload_; } void set_payload(absl::string_view payload); - template - InsertObjectMediaRequest& set_multiple_options(O&&... o) { - InsertObjectRequestImpl::set_multiple_options( - std::forward(o)...); - reset_hash_function(); - return *this; - } - HashFunction& hash_function() const { return *hash_function_; } - ///@{ /** * @name Backwards compatibility. @@ -154,16 +145,11 @@ class InsertObjectMediaRequest ///@} private: - void reset_hash_function(); - absl::string_view payload_; - std::shared_ptr hash_function_; mutable std::string contents_; mutable bool dirty_ = true; }; -HashValues FinishHashes(InsertObjectMediaRequest const& request); - std::ostream& operator<<(std::ostream& os, InsertObjectMediaRequest const& r); /** diff --git a/google/cloud/storage/internal/object_write_streambuf_test.cc b/google/cloud/storage/internal/object_write_streambuf_test.cc index dc6fb704d4ccf..1d9854e821eb5 100644 --- a/google/cloud/storage/internal/object_write_streambuf_test.cc +++ b/google/cloud/storage/internal/object_write_streambuf_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/internal/object_write_streambuf.h" #include "google/cloud/storage/internal/connection_impl.h" #include "google/cloud/storage/object_metadata.h" @@ -707,3 +708,4 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END } // namespace storage } // namespace cloud } // namespace google +#include "google/cloud/internal/diagnostics_pop.inc" diff --git a/google/cloud/storage/internal/rest/stub.cc b/google/cloud/storage/internal/rest/stub.cc index 4d57156eb1250..983f599e9fd1a 100644 --- a/google/cloud/storage/internal/rest/stub.cc +++ b/google/cloud/storage/internal/rest/stub.cc @@ -12,10 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/internal/rest/stub.h" #include "google/cloud/storage/internal/bucket_access_control_parser.h" #include "google/cloud/storage/internal/bucket_metadata_parser.h" #include "google/cloud/storage/internal/bucket_requests.h" +#include "google/cloud/storage/internal/checksum_helpers.h" #include "google/cloud/storage/internal/generate_message_boundary.h" #include "google/cloud/storage/internal/hmac_key_metadata_parser.h" #include "google/cloud/storage/internal/notification_metadata_parser.h" @@ -378,8 +380,16 @@ StatusOr RestStub::InsertObjectMediaMultipart( request.GetOption().value()); } - request.hash_function().Update(/*offset=*/0, request.payload()); - auto hashes = storage::internal::FinishHashes(request); + auto const settings = + storage::internal::GetUploadChecksumSettings(request, options); + auto disable_md5 = storage::DisableMD5Hash(settings.md5); + auto disable_crc32c = storage::DisableCrc32cChecksum(settings.crc32c); + auto hash_function = storage::internal::CreateHashFunction( + request.GetOption(), disable_crc32c, + request.GetOption(), disable_md5); + + hash_function->Update(/*offset=*/0, request.payload()); + auto hashes = hash_function->Finish(); if (!hashes.crc32c.empty()) metadata["crc32c"] = hashes.crc32c; if (!hashes.md5.empty()) metadata["md5Hash"] = hashes.md5; @@ -450,9 +460,9 @@ StatusOr RestStub::InsertObjectMedia( // If the application has set an explicit hash value we need to use multipart // uploads. `DisableMD5Hash` and `DisableCrc32cChecksum` should not be // dependent on each other. - if (!request.GetOption().value_or(false) || - !request.GetOption().value_or(false) || - request.HasOption() || + auto const settings = + storage::internal::GetUploadChecksumSettings(request, options); + if (!settings.md5 || !settings.crc32c || request.HasOption() || request.HasOption()) { return InsertObjectMediaMultipart(context, options, request); } @@ -1265,3 +1275,5 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END } // namespace storage } // namespace cloud } // namespace google + +#include "google/cloud/internal/diagnostics_pop.inc" diff --git a/google/cloud/storage/options.h b/google/cloud/storage/options.h index 649c717e2c45e..eebf3c827bccc 100644 --- a/google/cloud/storage/options.h +++ b/google/cloud/storage/options.h @@ -76,9 +76,10 @@ struct CAPathOption { * @ingroup storage-options */ enum class ChecksumAlgorithm { - kNone, ///< Disable checksum validation - kCrc32c, ///< Use CRC32C for checksum validation - kMD5, ///< Use MD5 for checksum validation + kNone, ///< Disable checksum validation + kCrc32c, ///< Use CRC32C for checksum validation + kMD5, ///< Use MD5 for checksum validation + kCrc32cAndMD5, ///< Use CRC32C and MD5 for checksum validation }; /** diff --git a/google/cloud/storage/testing/upload_hash_cases.cc b/google/cloud/storage/testing/upload_hash_cases.cc index 35a7dcb23b11d..808d923b37a76 100644 --- a/google/cloud/storage/testing/upload_hash_cases.cc +++ b/google/cloud/storage/testing/upload_hash_cases.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/testing/upload_hash_cases.h" #include @@ -66,3 +67,5 @@ std::vector UploadHashCases() { } // namespace storage } // namespace cloud } // namespace google + +#include "google/cloud/internal/diagnostics_pop.inc" diff --git a/google/cloud/storage/testing/upload_hash_cases.h b/google/cloud/storage/testing/upload_hash_cases.h index 79c39da4c90b4..4b01f8d301642 100644 --- a/google/cloud/storage/testing/upload_hash_cases.h +++ b/google/cloud/storage/testing/upload_hash_cases.h @@ -15,6 +15,7 @@ #ifndef GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_TESTING_UPLOAD_HASH_CASES_H #define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_TESTING_UPLOAD_HASH_CASES_H +#include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/options.h" #include #include @@ -41,3 +42,4 @@ std::vector UploadHashCases(); } // namespace google #endif // GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_TESTING_UPLOAD_HASH_CASES_H +#include "google/cloud/internal/diagnostics_pop.inc" diff --git a/google/cloud/storage/tests/object_checksum_integration_test.cc b/google/cloud/storage/tests/object_checksum_integration_test.cc index 2247a3c26871b..edf0e74c6ce51 100644 --- a/google/cloud/storage/tests/object_checksum_integration_test.cc +++ b/google/cloud/storage/tests/object_checksum_integration_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/client.h" #include "google/cloud/storage/testing/storage_integration_test.h" #include "google/cloud/internal/getenv.h" @@ -421,3 +422,4 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END } // namespace storage } // namespace cloud } // namespace google +#include "google/cloud/internal/diagnostics_pop.inc" diff --git a/google/cloud/storage/tests/object_file_integration_test.cc b/google/cloud/storage/tests/object_file_integration_test.cc index f289b675030a7..47544f54d1808 100644 --- a/google/cloud/storage/tests/object_file_integration_test.cc +++ b/google/cloud/storage/tests/object_file_integration_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/client.h" #include "google/cloud/storage/testing/storage_integration_test.h" #include "google/cloud/internal/filesystem.h" @@ -579,3 +580,4 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END } // namespace storage } // namespace cloud } // namespace google +#include "google/cloud/internal/diagnostics_pop.inc" diff --git a/google/cloud/storage/tests/object_hash_integration_test.cc b/google/cloud/storage/tests/object_hash_integration_test.cc index d4d6a1f060a31..351627f65a5f3 100644 --- a/google/cloud/storage/tests/object_hash_integration_test.cc +++ b/google/cloud/storage/tests/object_hash_integration_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/client.h" #include "google/cloud/storage/testing/canonical_errors.h" #include "google/cloud/storage/testing/storage_integration_test.h" @@ -372,3 +373,4 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END } // namespace storage } // namespace cloud } // namespace google +#include "google/cloud/internal/diagnostics_pop.inc" diff --git a/google/cloud/storage/tests/object_insert_integration_test.cc b/google/cloud/storage/tests/object_insert_integration_test.cc index beef22963493c..2b3df34b08fc7 100644 --- a/google/cloud/storage/tests/object_insert_integration_test.cc +++ b/google/cloud/storage/tests/object_insert_integration_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/client.h" #include "google/cloud/storage/internal/object_metadata_parser.h" #include "google/cloud/storage/testing/storage_integration_test.h" @@ -495,3 +496,4 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END } // namespace storage } // namespace cloud } // namespace google +#include "google/cloud/internal/diagnostics_pop.inc" From a0f73bec0103489282bf8bfcb4566c2f5dda60b5 Mon Sep 17 00:00:00 2001 From: Vaibhav Pratap Date: Mon, 20 Jul 2026 15:11:17 +0000 Subject: [PATCH 4/5] fix: correctly resolve legacy checksum options defaults --- google/cloud/storage/client_object_test.cc | 38 ++++++++++- .../cloud/storage/internal/checksum_helpers.h | 68 +++++++++++++------ 2 files changed, 83 insertions(+), 23 deletions(-) diff --git a/google/cloud/storage/client_object_test.cc b/google/cloud/storage/client_object_test.cc index f7cbccebc88a9..5c1b0fe06fdca 100644 --- a/google/cloud/storage/client_object_test.cc +++ b/google/cloud/storage/client_object_test.cc @@ -14,6 +14,7 @@ #include "google/cloud/internal/disable_deprecation_warnings.inc" #include "google/cloud/storage/client.h" +#include "google/cloud/storage/internal/checksum_helpers.h" #include "google/cloud/storage/internal/object_metadata_parser.h" #include "google/cloud/storage/retry_policy.h" #include "google/cloud/storage/testing/canonical_errors.h" @@ -238,6 +239,14 @@ TEST_F(ObjectTest, ReadObjectChecksumPrecedence) { .WillOnce([](internal::ReadObjectRangeRequest const& r) { EXPECT_TRUE(r.HasOption()); EXPECT_FALSE(r.GetOption().value()); + + auto settings = + internal::GetDownloadChecksumSettings(r, CurrentOptions()); + // Verify MD5 is enabled (disable_md5 = false) and CRC32C is disabled + // (disable_crc32c = true) + EXPECT_FALSE(settings.md5); + EXPECT_TRUE(settings.crc32c); + auto read_source = std::make_unique(); EXPECT_CALL(*read_source, IsOpen()).WillRepeatedly(Return(true)); EXPECT_CALL(*read_source, Read) @@ -250,13 +259,40 @@ TEST_F(ObjectTest, ReadObjectChecksumPrecedence) { auto actual = client.ReadObject( "test-bucket-name", "test-object-name", DisableMD5Hash(false), Options{}.set( - ChecksumAlgorithm::kCrc32c)); + ChecksumAlgorithm::kNone)); ASSERT_STATUS_OK(actual.status()); std::vector v(1024); actual.read(v.data(), v.size()); EXPECT_EQ(actual.gcount(), 1024); } +TEST_F(ObjectTest, InsertObjectChecksumPrecedence) { + EXPECT_CALL(*mock_, InsertObjectMedia) + .WillOnce([](internal::InsertObjectMediaRequest const& r) { + EXPECT_TRUE(r.HasOption()); + EXPECT_TRUE(r.GetOption().value()); + + auto settings = + internal::GetUploadChecksumSettings(r, CurrentOptions()); + // Verify CRC32C is disabled (disable_crc32c = true) and MD5 remains + // enabled (disable_md5 = false) + EXPECT_TRUE(settings.crc32c); + EXPECT_FALSE(settings.md5); + + return make_status_or( + storage::internal::ObjectMetadataParser::FromString( + R"({"name": "test-object-name"})") + .value()); + }); + auto client = ClientForMock(); + auto actual = + client.InsertObject("test-bucket-name", "test-object-name", "payload", + DisableCrc32cChecksum(true), + Options{}.set( + ChecksumAlgorithm::kCrc32cAndMD5)); + ASSERT_STATUS_OK(actual); +} + TEST_F(ObjectTest, WriteObject) { EXPECT_CALL(*mock_, CreateResumableUpload) .WillOnce(Return(TransientError())) diff --git a/google/cloud/storage/internal/checksum_helpers.h b/google/cloud/storage/internal/checksum_helpers.h index 34c657c067ae0..ff9a5280e05b0 100644 --- a/google/cloud/storage/internal/checksum_helpers.h +++ b/google/cloud/storage/internal/checksum_helpers.h @@ -34,39 +34,63 @@ struct HashDisabled { template HashDisabled GetDownloadChecksumSettings(Request const& request, Options const& options) { - if (request.template HasOption() || - request.template HasOption()) { - return { - request.template GetOption().value_or(false), - request.template GetOption().value_or(false)}; - } + bool disable_md5 = false; + bool disable_crc32c = false; + bool use_new_algo = false; if (options.has()) { auto const algo = options.get(); - return {algo != ChecksumAlgorithm::kMD5 && - algo != ChecksumAlgorithm::kCrc32cAndMD5, - algo != ChecksumAlgorithm::kCrc32c && - algo != ChecksumAlgorithm::kCrc32cAndMD5}; + disable_md5 = (algo != ChecksumAlgorithm::kMD5 && + algo != ChecksumAlgorithm::kCrc32cAndMD5); + disable_crc32c = (algo != ChecksumAlgorithm::kCrc32c && + algo != ChecksumAlgorithm::kCrc32cAndMD5); + use_new_algo = true; + } + + auto const md5 = request.template GetOption(); + if (md5.has_value()) { + // DisableMD5Hash defaults to true. We only override the new option if the legacy + // option was explicitly set to false, or if the new option was not provided at all. + if (!use_new_algo || md5.value() != true) { + disable_md5 = md5.value(); + } + } + auto const crc32c = request.template GetOption(); + if (crc32c.has_value()) { + // DisableCrc32cChecksum defaults to std::nullopt, so if it has a value, it was explicitly set. + disable_crc32c = crc32c.value(); } - return {false, false}; + return {disable_md5, disable_crc32c}; } template HashDisabled GetUploadChecksumSettings(Request const& request, Options const& options) { - if (request.template HasOption() || - request.template HasOption()) { - return { - request.template GetOption().value_or(false), - request.template GetOption().value_or(false)}; - } + bool disable_md5 = false; + bool disable_crc32c = false; + bool use_new_algo = false; if (options.has()) { auto const algo = options.get(); - return {algo != ChecksumAlgorithm::kMD5 && - algo != ChecksumAlgorithm::kCrc32cAndMD5, - algo != ChecksumAlgorithm::kCrc32c && - algo != ChecksumAlgorithm::kCrc32cAndMD5}; + disable_md5 = (algo != ChecksumAlgorithm::kMD5 && + algo != ChecksumAlgorithm::kCrc32cAndMD5); + disable_crc32c = (algo != ChecksumAlgorithm::kCrc32c && + algo != ChecksumAlgorithm::kCrc32cAndMD5); + use_new_algo = true; + } + + auto const md5 = request.template GetOption(); + if (md5.has_value()) { + // DisableMD5Hash defaults to true. We only override the new option if the legacy + // option was explicitly set to false, or if the new option was not provided at all. + if (!use_new_algo || md5.value() != true) { + disable_md5 = md5.value(); + } + } + auto const crc32c = request.template GetOption(); + if (crc32c.has_value()) { + // DisableCrc32cChecksum defaults to std::nullopt, so if it has a value, it was explicitly set. + disable_crc32c = crc32c.value(); } - return {false, false}; + return {disable_md5, disable_crc32c}; } } // namespace internal From c7928aece366d0987712f4be184e960e770c753a Mon Sep 17 00:00:00 2001 From: Vaibhav Pratap Date: Mon, 20 Jul 2026 15:33:38 +0000 Subject: [PATCH 5/5] fix formatting in checksum_helpers.h --- .../cloud/storage/internal/checksum_helpers.h | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/google/cloud/storage/internal/checksum_helpers.h b/google/cloud/storage/internal/checksum_helpers.h index ff9a5280e05b0..9b07d9341aa47 100644 --- a/google/cloud/storage/internal/checksum_helpers.h +++ b/google/cloud/storage/internal/checksum_helpers.h @@ -45,18 +45,20 @@ HashDisabled GetDownloadChecksumSettings(Request const& request, algo != ChecksumAlgorithm::kCrc32cAndMD5); use_new_algo = true; } - + auto const md5 = request.template GetOption(); if (md5.has_value()) { - // DisableMD5Hash defaults to true. We only override the new option if the legacy - // option was explicitly set to false, or if the new option was not provided at all. + // DisableMD5Hash defaults to true. We only override the new option if the + // legacy option was explicitly set to false, or if the new option was not + // provided at all. if (!use_new_algo || md5.value() != true) { disable_md5 = md5.value(); } } auto const crc32c = request.template GetOption(); if (crc32c.has_value()) { - // DisableCrc32cChecksum defaults to std::nullopt, so if it has a value, it was explicitly set. + // DisableCrc32cChecksum defaults to std::nullopt, so if it has a value, it + // was explicitly set. disable_crc32c = crc32c.value(); } return {disable_md5, disable_crc32c}; @@ -76,18 +78,20 @@ HashDisabled GetUploadChecksumSettings(Request const& request, algo != ChecksumAlgorithm::kCrc32cAndMD5); use_new_algo = true; } - + auto const md5 = request.template GetOption(); if (md5.has_value()) { - // DisableMD5Hash defaults to true. We only override the new option if the legacy - // option was explicitly set to false, or if the new option was not provided at all. + // DisableMD5Hash defaults to true. We only override the new option if the + // legacy option was explicitly set to false, or if the new option was not + // provided at all. if (!use_new_algo || md5.value() != true) { disable_md5 = md5.value(); } } auto const crc32c = request.template GetOption(); if (crc32c.has_value()) { - // DisableCrc32cChecksum defaults to std::nullopt, so if it has a value, it was explicitly set. + // DisableCrc32cChecksum defaults to std::nullopt, so if it has a value, it + // was explicitly set. disable_crc32c = crc32c.value(); } return {disable_md5, disable_crc32c};