From 0b1f31f6399ef0a6e56ff5003bab63812cad7caf Mon Sep 17 00:00:00 2001 From: Adam Fiedler Date: Tue, 27 Jan 2026 16:56:23 +0100 Subject: [PATCH] feat: add new fields regarding NULL values jira: CQ-1959 risk: low --- .../physical_model/column.py | 2 + .../logical_model/dataset/dataset.py | 10 ++++ .../declarative_ldm_with_sql_dataset.json | 60 ++++++++++++------- .../tests/catalog/test_catalog_data_source.py | 20 +++++-- 4 files changed, 68 insertions(+), 24 deletions(-) diff --git a/packages/gooddata-sdk/src/gooddata_sdk/catalog/data_source/declarative_model/physical_model/column.py b/packages/gooddata-sdk/src/gooddata_sdk/catalog/data_source/declarative_model/physical_model/column.py index 95344c1a8..45ccb6a15 100644 --- a/packages/gooddata-sdk/src/gooddata_sdk/catalog/data_source/declarative_model/physical_model/column.py +++ b/packages/gooddata-sdk/src/gooddata_sdk/catalog/data_source/declarative_model/physical_model/column.py @@ -16,6 +16,8 @@ class CatalogDeclarativeColumn(Base): is_primary_key: Optional[bool] = None referenced_table_id: Optional[str] = None referenced_table_column: Optional[str] = None + is_nullable: Optional[bool] = None + null_value: Optional[str] = None @staticmethod def client_class() -> type[DeclarativeColumn]: diff --git a/packages/gooddata-sdk/src/gooddata_sdk/catalog/workspace/declarative_model/workspace/logical_model/dataset/dataset.py b/packages/gooddata-sdk/src/gooddata_sdk/catalog/workspace/declarative_model/workspace/logical_model/dataset/dataset.py index 54f382206..be37191c2 100644 --- a/packages/gooddata-sdk/src/gooddata_sdk/catalog/workspace/declarative_model/workspace/logical_model/dataset/dataset.py +++ b/packages/gooddata-sdk/src/gooddata_sdk/catalog/workspace/declarative_model/workspace/logical_model/dataset/dataset.py @@ -82,6 +82,8 @@ class CatalogDeclarativeAttribute(Base): tags: Optional[list[str]] = None is_hidden: Optional[bool] = None locale: Optional[str] = None + is_nullable: Optional[bool] = None + null_value: Optional[str] = None @staticmethod def client_class() -> type[DeclarativeAttribute]: @@ -97,6 +99,8 @@ class CatalogDeclarativeFact(Base): description: Optional[str] = None tags: Optional[list[str]] = None is_hidden: Optional[bool] = None + is_nullable: Optional[bool] = None + null_value: Optional[str] = None @staticmethod def client_class() -> type[DeclarativeFact]: @@ -121,6 +125,8 @@ class CatalogDeclarativeAggregatedFact(Base): source_column_data_type: Optional[str] = None description: Optional[str] = None tags: Optional[list[str]] = None + is_nullable: Optional[bool] = None + null_value: Optional[str] = None @staticmethod def client_class() -> type[DeclarativeAggregatedFact]: @@ -171,6 +177,8 @@ class CatalogDeclarativeLabel(Base): locale: Optional[str] = None translations: Optional[list[CatalogDeclarativeLabelTranslation]] = None geo_area_config: Optional[CatalogGeoAreaConfig] = None + is_nullable: Optional[bool] = None + null_value: Optional[str] = None @staticmethod def client_class() -> type[DeclarativeLabel]: @@ -203,6 +211,8 @@ class CatalogDeclarativeReference(Base): source_columns: Optional[list[str]] = None source_column_data_types: Optional[list[str]] = None sources: Optional[list[CatalogDeclarativeReferenceSource]] = None + is_nullable: Optional[bool] = None + null_value: Optional[str] = None @staticmethod def client_class() -> type[DeclarativeReference]: diff --git a/packages/gooddata-sdk/tests/catalog/expected/declarative_ldm_with_sql_dataset.json b/packages/gooddata-sdk/tests/catalog/expected/declarative_ldm_with_sql_dataset.json index 230eb75ac..b7623f8c1 100644 --- a/packages/gooddata-sdk/tests/catalog/expected/declarative_ldm_with_sql_dataset.json +++ b/packages/gooddata-sdk/tests/catalog/expected/declarative_ldm_with_sql_dataset.json @@ -40,7 +40,8 @@ "description": "Campaign channel id", "tags": [ "Campaign channels" - ] + ], + "is_nullable": false }, { "id": "campaign_channels.category", @@ -51,7 +52,8 @@ "description": "Category", "tags": [ "Campaign channels" - ] + ], + "is_nullable": true }, { "id": "type", @@ -62,7 +64,8 @@ "description": "Type", "tags": [ "Campaign channels" - ] + ], + "is_nullable": true } ], "facts": [ @@ -74,7 +77,8 @@ "description": "Budget", "tags": [ "Campaign channels" - ] + ], + "is_nullable": true }, { "id": "spend", @@ -84,7 +88,8 @@ "description": "Spend", "tags": [ "Campaign channels" - ] + ], + "is_nullable": true } ], "dataSourceTableId": { @@ -121,7 +126,8 @@ "description": "Campaign name", "tags": [ "Campaigns" - ] + ], + "is_nullable": true }, { "id": "campaigns.campaign_id", @@ -132,7 +138,8 @@ "description": "Campaign id", "tags": [ "Campaigns" - ] + ], + "is_nullable": false } ], "facts": [], @@ -170,7 +177,8 @@ "description": "Customer id", "tags": [ "Customers" - ] + ], + "is_nullable": false }, { "id": "customers.customer_name", @@ -181,7 +189,8 @@ "description": "Customer name", "tags": [ "Customers" - ] + ], + "is_nullable": true }, { "id": "customers.region", @@ -192,7 +201,8 @@ "description": "Region", "tags": [ "Customers" - ] + ], + "is_nullable": true }, { "id": "customers.state", @@ -206,7 +216,8 @@ "tags": [ "Customers" ], - "valueType": "GEO" + "valueType": "GEO", + "is_nullable": true } ], "sourceColumn": "state", @@ -215,7 +226,8 @@ "description": "State", "tags": [ "Customers" - ] + ], + "is_nullable": true } ], "facts": [], @@ -322,7 +334,8 @@ "description": "Order id", "tags": [ "Order lines" - ] + ], + "is_nullable": true }, { "id": "order_lines.order_line_id", @@ -333,7 +346,8 @@ "description": "Order line id", "tags": [ "Order lines" - ] + ], + "is_nullable": false }, { "id": "order_lines.order_status", @@ -344,7 +358,8 @@ "description": "Order status", "tags": [ "Order lines" - ] + ], + "is_nullable": true } ], "facts": [ @@ -356,7 +371,8 @@ "description": "Price", "tags": [ "Order lines" - ] + ], + "is_nullable": true }, { "id": "order_lines.quantity", @@ -366,7 +382,8 @@ "description": "Quantity", "tags": [ "Order lines" - ] + ], + "is_nullable": true } ], "dataSourceTableId": { @@ -413,7 +430,8 @@ "description": "Product name", "tags": [ "Products" - ] + ], + "is_nullable": true }, { "id": "products.category", @@ -424,7 +442,8 @@ "description": "Category", "tags": [ "Products" - ] + ], + "is_nullable": true }, { "id": "products.product_id", @@ -435,7 +454,8 @@ "description": "Product id", "tags": [ "Products" - ] + ], + "is_nullable": false } ], "facts": [], diff --git a/packages/gooddata-sdk/tests/catalog/test_catalog_data_source.py b/packages/gooddata-sdk/tests/catalog/test_catalog_data_source.py index b08b696a9..4cbeb58a3 100644 --- a/packages/gooddata-sdk/tests/catalog/test_catalog_data_source.py +++ b/packages/gooddata-sdk/tests/catalog/test_catalog_data_source.py @@ -44,6 +44,7 @@ ) from gooddata_sdk.catalog.data_source.entity_model.data_source import DatabaseAttributes from gooddata_sdk.catalog.entity import ClientSecretCredentialsFromFile +from tests_support.compare_utils import deep_eq from tests_support.file_utils import load_json from tests_support.vcrpy_utils import get_vcr @@ -93,7 +94,11 @@ def test_generate_logical_model(test_config: dict): """ # Filter out SQL-based datasets (those have sql property set, no data_source_table_id) table_based_datasets = [ds for ds in declarative_model.ldm.datasets if ds.sql is None] - assert table_based_datasets == generated_declarative_model.ldm.datasets + + for i, dataset in enumerate(table_based_datasets): + print(f"Dataset: {dataset.id}") + assert deep_eq(dataset, generated_declarative_model.ldm.datasets[i]) + assert len(declarative_model.ldm.date_instances) == len(generated_declarative_model.ldm.date_instances) @@ -117,7 +122,10 @@ def test_scan_pdm_and_generate_logical_model(test_config: dict): """ # Filter out SQL-based datasets (those have sql property set, no data_source_table_id) table_based_datasets = [ds for ds in declarative_model.ldm.datasets if ds.sql is None] - assert table_based_datasets == generated_declarative_model.ldm.datasets + for i, dataset in enumerate(table_based_datasets): + print(f"Dataset: {dataset.id}") + assert deep_eq(dataset, generated_declarative_model.ldm.datasets[i]) + assert len(declarative_model.ldm.date_instances) == len(generated_declarative_model.ldm.date_instances) @@ -182,7 +190,9 @@ def test_generate_logical_model_with_sql_datasets(test_config: dict): # and remove sort once fixed generated_declarative_model.ldm.datasets.sort(key=lambda dataset: dataset.id) expected_ldm.ldm.datasets.sort(key=lambda dataset: dataset.id) - assert expected_ldm.ldm.datasets == generated_declarative_model.ldm.datasets + for i, dataset in enumerate(expected_ldm.ldm.datasets): + print(f"Dataset: {dataset.id}") + assert deep_eq(dataset, generated_declarative_model.ldm.datasets[i]) assert len(expected_ldm.ldm.date_instances) == len(generated_declarative_model.ldm.date_instances) @@ -208,7 +218,9 @@ def test_scan_pdm_and_generate_logical_model_with_sql_datasets(test_config: dict # and remove sort once fixed generated_declarative_model.ldm.datasets.sort(key=lambda dataset: dataset.id) expected_ldm.ldm.datasets.sort(key=lambda dataset: dataset.id) - assert expected_ldm.ldm.datasets == generated_declarative_model.ldm.datasets + for i, dataset in enumerate(expected_ldm.ldm.datasets): + print(f"Dataset {dataset.id}") + assert deep_eq(dataset, generated_declarative_model.ldm.datasets[i]) assert len(expected_ldm.ldm.date_instances) == len(generated_declarative_model.ldm.date_instances)