From f5b88267636314c2ef2771ced044652895303cb4 Mon Sep 17 00:00:00 2001 From: Gabor Gyimesi Date: Wed, 8 Oct 2025 15:40:04 +0200 Subject: [PATCH 1/2] MINIFICPP-2703 Update GCP extension to use unified credentials --- cmake/GoogleCloudCpp.cmake | 4 +-- .../GCPCredentialsControllerService.cpp | 36 +++++-------------- .../GCPCredentialsControllerService.h | 9 +++-- extensions/gcp/processors/GCSProcessor.cpp | 14 +++++--- extensions/gcp/processors/GCSProcessor.h | 6 ++-- .../GCPCredentialsControllerServiceTests.cpp | 7 ---- 6 files changed, 26 insertions(+), 50 deletions(-) diff --git a/cmake/GoogleCloudCpp.cmake b/cmake/GoogleCloudCpp.cmake index 071f83d5fe..0d34e78568 100644 --- a/cmake/GoogleCloudCpp.cmake +++ b/cmake/GoogleCloudCpp.cmake @@ -49,8 +49,8 @@ set(GOOGLE_CLOUD_CPP_ENABLE_MACOS_OPENSSL_CHECK OFF CACHE INTERNAL macos-openssl set(BUILD_TESTING OFF CACHE INTERNAL testing-off) set(GOOGLE_CLOUD_CPP_ENABLE_WERROR OFF CACHE INTERNAL warnings-off) FetchContent_Declare(google-cloud-cpp - URL https://github.com/googleapis/google-cloud-cpp/archive/refs/tags/v2.38.0.tar.gz - URL_HASH SHA256=f1493b2dce9b379714342f2be7ccb483d70d13aac09d4a90ae3b4756693b72fc + URL https://github.com/googleapis/google-cloud-cpp/archive/refs/tags/v2.45.0.tar.gz + URL_HASH SHA256=3d1b5eb696832f9071bf7ef0b3f0c9fd27c1a39d5edcb8a9976c65193319fd01 PATCH_COMMAND "${PC}" SYSTEM) if (WIN32) diff --git a/extensions/gcp/controllerservices/GCPCredentialsControllerService.cpp b/extensions/gcp/controllerservices/GCPCredentialsControllerService.cpp index eb9842fc45..5101f6fd92 100644 --- a/extensions/gcp/controllerservices/GCPCredentialsControllerService.cpp +++ b/extensions/gcp/controllerservices/GCPCredentialsControllerService.cpp @@ -21,8 +21,7 @@ #include "core/Resource.h" #include "google/cloud/storage/client.h" #include "utils/ProcessorConfigUtils.h" - -namespace gcs = ::google::cloud::storage; +#include "utils/file/FileUtils.h" namespace org::apache::nifi::minifi::extensions::gcp { @@ -30,43 +29,24 @@ void GCPCredentialsControllerService::initialize() { setSupportedProperties(Properties); } -std::shared_ptr GCPCredentialsControllerService::createDefaultCredentials() const { - auto default_credentials = gcs::oauth2::CreateServiceAccountCredentialsFromDefaultPaths(); - if (!default_credentials.ok()) { - logger_->log_error("{}", default_credentials.status().message()); - return nullptr; - } - return *default_credentials; -} - -std::shared_ptr GCPCredentialsControllerService::createCredentialsFromJsonPath() const { +std::shared_ptr GCPCredentialsControllerService::createCredentialsFromJsonPath() const { const auto json_path = getProperty(JsonFilePath.name); if (!json_path) { logger_->log_error("Missing or invalid {}", JsonFilePath.name); return nullptr; } - auto json_path_credentials = gcs::oauth2::CreateServiceAccountCredentialsFromJsonFilePath(*json_path); - if (!json_path_credentials.ok()) { - logger_->log_error("{}", json_path_credentials.status().message()); - return nullptr; - } - return *json_path_credentials; + return google::cloud::MakeServiceAccountCredentials(utils::file::get_content(*json_path)); } -std::shared_ptr GCPCredentialsControllerService::createCredentialsFromJsonContents() const { +std::shared_ptr GCPCredentialsControllerService::createCredentialsFromJsonContents() const { auto json_contents = getProperty(JsonContents.name); if (!json_contents) { logger_->log_error("Missing or invalid {}", JsonContents.name); return nullptr; } - auto json_path_credentials = gcs::oauth2::CreateServiceAccountCredentialsFromJsonContents(*json_contents); - if (!json_path_credentials.ok()) { - logger_->log_error("{}", json_path_credentials.status().message()); - return nullptr; - } - return *json_path_credentials; + return google::cloud::MakeServiceAccountCredentials(*json_contents); } void GCPCredentialsControllerService::onEnable() { @@ -79,15 +59,15 @@ void GCPCredentialsControllerService::onEnable() { credentials_location = CredentialsLocation::USE_DEFAULT_CREDENTIALS; } if (*credentials_location == CredentialsLocation::USE_DEFAULT_CREDENTIALS) { - credentials_ = createDefaultCredentials(); + credentials_ = google::cloud::MakeGoogleDefaultCredentials(); } else if (*credentials_location == CredentialsLocation::USE_COMPUTE_ENGINE_CREDENTIALS) { - credentials_ = gcs::oauth2::CreateComputeEngineCredentials(); + credentials_ = google::cloud::MakeComputeEngineCredentials(); } else if (*credentials_location == CredentialsLocation::USE_JSON_FILE) { credentials_ = createCredentialsFromJsonPath(); } else if (*credentials_location == CredentialsLocation::USE_JSON_CONTENTS) { credentials_ = createCredentialsFromJsonContents(); } else if (*credentials_location == CredentialsLocation::USE_ANONYMOUS_CREDENTIALS) { - credentials_ = gcs::oauth2::CreateAnonymousCredentials(); + credentials_ = google::cloud::MakeInsecureCredentials(); } if (!credentials_) logger_->log_error("Couldn't create valid credentials"); diff --git a/extensions/gcp/controllerservices/GCPCredentialsControllerService.h b/extensions/gcp/controllerservices/GCPCredentialsControllerService.h index 20d7c65b21..4f5fc219b4 100644 --- a/extensions/gcp/controllerservices/GCPCredentialsControllerService.h +++ b/extensions/gcp/controllerservices/GCPCredentialsControllerService.h @@ -28,7 +28,7 @@ #include "core/PropertyDefinitionBuilder.h" #include "utils/Enum.h" -#include "google/cloud/storage/oauth2/credentials.h" +#include "google/cloud/credentials.h" namespace org::apache::nifi::minifi::extensions::gcp { enum class CredentialsLocation { @@ -113,12 +113,11 @@ class GCPCredentialsControllerService : public core::controller::ControllerServi [[nodiscard]] const auto& getCredentials() const { return credentials_; } protected: - [[nodiscard]] std::shared_ptr createDefaultCredentials() const; - [[nodiscard]] std::shared_ptr createCredentialsFromJsonPath() const; - [[nodiscard]] std::shared_ptr createCredentialsFromJsonContents() const; + [[nodiscard]] std::shared_ptr createCredentialsFromJsonPath() const; + [[nodiscard]] std::shared_ptr createCredentialsFromJsonContents() const; - std::shared_ptr credentials_; + std::shared_ptr credentials_; std::shared_ptr logger_ = core::logging::LoggerFactory::getLogger(uuid_); }; } // namespace org::apache::nifi::minifi::extensions::gcp diff --git a/extensions/gcp/processors/GCSProcessor.cpp b/extensions/gcp/processors/GCSProcessor.cpp index ded36f8a23..91d48d64a6 100644 --- a/extensions/gcp/processors/GCSProcessor.cpp +++ b/extensions/gcp/processors/GCSProcessor.cpp @@ -27,7 +27,7 @@ namespace gcs = ::google::cloud::storage; namespace org::apache::nifi::minifi::extensions::gcp { -std::shared_ptr GCSProcessor::getCredentials(core::ProcessContext& context) const { +std::shared_ptr GCSProcessor::getCredentials(core::ProcessContext& context) const { auto gcp_credentials_controller_service = utils::parseOptionalControllerService(context, GCSProcessor::GCPCredentials, getUUID()); if (gcp_credentials_controller_service) { return gcp_credentials_controller_service->getCredentials(); @@ -51,10 +51,14 @@ void GCSProcessor::onSchedule(core::ProcessContext& context, core::ProcessSessio } gcs::Client GCSProcessor::getClient() const { - auto options = gcs::ClientOptions(gcp_credentials_); - if (endpoint_url_) - options.set_endpoint(*endpoint_url_); - return gcs::Client(options, *retry_policy_); + auto options = google::cloud::Options{} + .set(gcp_credentials_) + .set(retry_policy_); + + if (endpoint_url_) { + options.set(*endpoint_url_); + } + return gcs::Client(options); } } // namespace org::apache::nifi::minifi::extensions::gcp diff --git a/extensions/gcp/processors/GCSProcessor.h b/extensions/gcp/processors/GCSProcessor.h index e3361a786c..1ec2b6641a 100644 --- a/extensions/gcp/processors/GCSProcessor.h +++ b/extensions/gcp/processors/GCSProcessor.h @@ -27,7 +27,7 @@ #include "minifi-cpp/core/PropertyDefinition.h" #include "core/PropertyDefinitionBuilder.h" #include "minifi-cpp/core/PropertyValidator.h" -#include "google/cloud/storage/oauth2/credentials.h" +#include "google/cloud/credentials.h" #include "google/cloud/storage/client.h" #include "google/cloud/storage/retry_policy.h" @@ -64,10 +64,10 @@ class GCSProcessor : public core::ProcessorImpl { protected: virtual google::cloud::storage::Client getClient() const; - std::shared_ptr getCredentials(core::ProcessContext& context) const; + std::shared_ptr getCredentials(core::ProcessContext& context) const; std::optional endpoint_url_; - std::shared_ptr gcp_credentials_; + std::shared_ptr gcp_credentials_; google::cloud::storage::RetryPolicyOption::Type retry_policy_ = std::make_shared(6); }; diff --git a/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp b/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp index 969865a746..8c7438807a 100644 --- a/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp +++ b/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp @@ -80,13 +80,6 @@ class GCPCredentialsTests : public ::testing::Test { std::shared_ptr gcp_credentials_ = std::dynamic_pointer_cast(gcp_credentials_node_->getControllerServiceImplementation()); }; -TEST_F(GCPCredentialsTests, DefaultGCPCredentialsWithoutEnv) { - minifi::utils::Environment::unsetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS"); - plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::CredentialsLoc, magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_DEFAULT_CREDENTIALS)); - ASSERT_NO_THROW(test_controller_.runSession(plan_)); - EXPECT_EQ(nullptr, gcp_credentials_->getCredentials()); -} - TEST_F(GCPCredentialsTests, DefaultGCPCredentialsWithEnv) { auto temp_directory = test_controller_.createTempDirectory(); auto path = create_mock_json_file(temp_directory); From 19049a122e95ab317e6e06c11de0872bd51a04b0 Mon Sep 17 00:00:00 2001 From: Gabor Gyimesi Date: Wed, 11 Feb 2026 11:34:10 +0100 Subject: [PATCH 2/2] Review update --- .../GCPCredentialsControllerService.cpp | 5 +++++ .../gcp/tests/GCPCredentialsControllerServiceTests.cpp | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/extensions/gcp/controllerservices/GCPCredentialsControllerService.cpp b/extensions/gcp/controllerservices/GCPCredentialsControllerService.cpp index 5101f6fd92..9e93c29422 100644 --- a/extensions/gcp/controllerservices/GCPCredentialsControllerService.cpp +++ b/extensions/gcp/controllerservices/GCPCredentialsControllerService.cpp @@ -36,6 +36,11 @@ std::shared_ptr GCPCredentialsControllerService::cre return nullptr; } + if (!utils::file::exists(*json_path)) { + logger_->log_error("JSON file for GCP credentials '{}' does not exist", *json_path); + return nullptr; + } + return google::cloud::MakeServiceAccountCredentials(utils::file::get_content(*json_path)); } diff --git a/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp b/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp index 8c7438807a..6c33a402e6 100644 --- a/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp +++ b/extensions/gcp/tests/GCPCredentialsControllerServiceTests.cpp @@ -106,6 +106,16 @@ TEST_F(GCPCredentialsTests, CredentialsFromJsonWithProperty) { EXPECT_NE(nullptr, gcp_credentials_->getCredentials()); } +TEST_F(GCPCredentialsTests, CredentialsFromJsonWithInvalidPath) { + auto temp_directory = test_controller_.createTempDirectory(); + auto path = create_mock_json_file(temp_directory); + ASSERT_TRUE(path.has_value()); + plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::CredentialsLoc, magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_JSON_FILE)); + plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::JsonFilePath, "/invalid/path/to/credentials.json"); + ASSERT_NO_THROW(test_controller_.runSession(plan_)); + EXPECT_EQ(nullptr, gcp_credentials_->getCredentials()); +} + TEST_F(GCPCredentialsTests, CredentialsFromComputeEngineVM) { plan_->setProperty(gcp_credentials_node_, GCPCredentialsControllerService::CredentialsLoc, magic_enum::enum_name(minifi_gcp::CredentialsLocation::USE_COMPUTE_ENGINE_CREDENTIALS)); ASSERT_NO_THROW(test_controller_.runSession(plan_));