From 155a932a8d0acd09c464b8c5f40abd751c5fd82e Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 14 Jan 2026 19:00:55 +0000 Subject: [PATCH] Merge pull request #93866 from scanhex12/gs_storage_support Google cloud storage support for data lakes catalogs --- src/Common/FailPoint.cpp | 3 ++- src/Databases/DataLake/ICatalog.cpp | 17 ++++++++++++-- .../integration/test_database_iceberg/test.py | 23 +++++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 0adbd8102bf9..cb498d92beeb 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -117,7 +117,8 @@ static struct InitFiu REGULAR(sleep_in_logs_flush) \ ONCE(smt_commit_exception_before_op) \ ONCE(backup_add_empty_memory_table) \ - REGULAR(refresh_task_stop_racing_for_running_refresh) + REGULAR(refresh_task_stop_racing_for_running_refresh) \ + ONCE(database_iceberg_gcs) namespace FailPoints diff --git a/src/Databases/DataLake/ICatalog.cpp b/src/Databases/DataLake/ICatalog.cpp index c7ab78030411..eceaa73e55b6 100644 --- a/src/Databases/DataLake/ICatalog.cpp +++ b/src/Databases/DataLake/ICatalog.cpp @@ -5,10 +5,18 @@ #include +#include + namespace DB::ErrorCodes { extern const int NOT_IMPLEMENTED; extern const int LOGICAL_ERROR; + extern const int BAD_ARGUMENTS; +} + +namespace DB::FailPoints +{ + extern const char database_iceberg_gcs[]; } namespace DataLake @@ -49,9 +57,14 @@ StorageType parseStorageTypeFromString(const std::string & type) } if (capitalize_first_letter(storage_type_str) == "File") storage_type_str = "Local"; - - if (capitalize_first_letter(storage_type_str) == "S3a") + else if (capitalize_first_letter(storage_type_str) == "S3a" || storage_type_str == "oss" || storage_type_str == "gs") + { + fiu_do_on(DB::FailPoints::database_iceberg_gcs, + { + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Google cloud storage converts to S3"); + }); storage_type_str = "S3"; + } auto storage_type = magic_enum::enum_cast(capitalize_first_letter(storage_type_str)); diff --git a/tests/integration/test_database_iceberg/test.py b/tests/integration/test_database_iceberg/test.py index 1edacc5a68c7..2de6586e3787 100644 --- a/tests/integration/test_database_iceberg/test.py +++ b/tests/integration/test_database_iceberg/test.py @@ -798,3 +798,26 @@ def test_cluster_joins(started_cluster): ) assert res == "Jack\tBlack\nJack\tSilver\nJohn\tBlack\nJohn\tSilver\n" + +def test_gcs(started_cluster): + node = started_cluster.instances["node1"] + + node.query("SYSTEM ENABLE FAILPOINT database_iceberg_gcs") + node.query( + f""" + DROP DATABASE IF EXISTS {CATALOG_NAME}; + SET allow_database_iceberg = 1; + """ + ) + + with pytest.raises(Exception) as err: + node.query( + f""" + CREATE DATABASE {CATALOG_NAME} + ENGINE = DataLakeCatalog('http://rest:8181/v1', 'gcs', 'dummy') + SETTINGS + catalog_type = 'rest', + warehouse = 'demo', + """ + ) + assert "Google cloud storage converts to S3" in str(err.value)