test: add aggregated dataset to testing ldm

fiedlr · fiedlr · commit 3e11e65bc2d9 · 2025-08-05T17:40:02.000+02:00
risk: nonprod
diff --git a/gooddata-sdk/gooddata_sdk/catalog/workspace/declarative_model/workspace/logical_model/dataset/dataset.py b/gooddata-sdk/gooddata_sdk/catalog/workspace/declarative_model/workspace/logical_model/dataset/dataset.py
@@ -42,6 +42,7 @@ class CatalogDeclarativeDataset(Base):
     attributes: Optional[list[CatalogDeclarativeAttribute]] = None
     facts: Optional[list[CatalogDeclarativeFact]] = None
     aggregated_facts: Optional[list[CatalogDeclarativeAggregatedFact]] = None
+    precedence: Optional[int] = None
     data_source_table_id: Optional[CatalogDataSourceTableIdentifier] = None
     sql: Optional[CatalogDeclarativeDatasetSql] = None
     tags: Optional[list[str]] = None
diff --git a/gooddata-sdk/gooddata_sdk/catalog/workspace/entity_model/content_objects/dataset.py b/gooddata-sdk/gooddata_sdk/catalog/workspace/entity_model/content_objects/dataset.py
@@ -145,6 +145,14 @@ def generate_facts_from_api(self) -> list[CatalogFact]:
         default=attr.Factory(lambda self: self.generate_facts_from_api(), takes_self=True),
     )
 
+    # TODO: Doublecheck if we shouldn't do something like for facts
+    aggregated_facts: Optional[list[CatalogAggregatedFact]] = attr.field(
+        default=attr.Factory(lambda self: self.json_api_attributes.get("aggregatedFacts"), takes_self=True),
+    )
+    precedence: Optional[int] = attr.field(
+        default=attr.Factory(lambda self: self.json_api_attributes.get("precedence"), takes_self=True)
+    )
+
     grain: Optional[list] = attr.field(
         default=attr.Factory(lambda self: self.json_api_attributes.get("grain"), takes_self=True)
     )
diff --git a/gooddata-sdk/gooddata_sdk/catalog/workspace/model_container.py b/gooddata-sdk/gooddata_sdk/catalog/workspace/model_container.py
@@ -6,6 +6,7 @@
 
 from gooddata_sdk.catalog.types import ValidObjects
 from gooddata_sdk.catalog.workspace.entity_model.content_objects.dataset import (
+    CatalogAggregatedFact,
     CatalogAttribute,
     CatalogDataset,
     CatalogFact,
@@ -46,6 +47,10 @@ def datasets(self) -> list[CatalogDataset]:
     def facts(self) -> list[CatalogFact]:
         return [f for d in self._datasets for f in d.facts]
 
+    @property
+    def aggregated_facts(self) -> list[CatalogAggregatedFact]:
+        return [f for d in self._datasets for f in d.aggregated_facts]
+
     @property
     def attributes(self) -> list[CatalogAttribute]:
         return [a for d in self._datasets for a in d.attributes]
diff --git a/gooddata-sdk/tests/catalog/expected/declarative_workspaces.json b/gooddata-sdk/tests/catalog/expected/declarative_workspaces.json
@@ -2376,6 +2376,54 @@
               ],
               "title": "Campaign channels"
             },
+            {
+              "attributes": [],
+              "description": "Campaign channels per categories",
+              "facts": [],
+              "aggregatedFacts": [
+                {
+                  "description": "Budget Agg",
+                  "id": "budget_agg",
+                  "sourceColumn": "budget",
+                  "sourceColumnDataType": "INT",
+                  "tags": [
+                    "Campaign channels per category"
+                  ],
+                  "sourceFactReference": {
+                    "reference": {
+                      "id": "budget",
+                      "type": "fact"
+                    },
+                    "operation": "SUM"
+                  }
+                }
+              ],
+              "grain": [],
+              "id": "campaign_channels_per_category",
+              "precedence":  1,
+              "references": [
+                {
+                  "identifier": {
+                    "id": "campaign_channels",
+                    "type": "dataset"
+                  },
+                  "multivalue": false,
+                  "sourceColumns": [
+                    "category",
+                    "budget"
+                  ],
+                  "sourceColumnDataTypes": [ "STRING", "INT" ]
+                }
+              ],
+              "tags": [
+                "Campaign channels per category"
+              ],
+              "sql": {
+                "statement": "SELECT category, SUM(budget) FROM campaign_channels GROUP BY category",
+                "dataSourceId": "demo-test-ds"
+              },
+              "title": "Campaign channels per category"
+            },
             {
               "attributes": [
                 {
@@ -2412,6 +2460,7 @@
               },
               "description": "Campaigns",
               "facts": [],
+              "aggregatedFacts": [],
               "grain": [
                 {
                   "id": "campaign_id",
@@ -2494,6 +2543,7 @@
               },
               "description": "Customers",
               "facts": [],
+              "aggregatedFacts": [],
               "grain": [
                 {
                   "id": "customer_id",
@@ -2575,6 +2625,7 @@
                   "title": "Quantity"
                 }
               ],
+              "aggregatedFacts": [],
               "grain": [
                 {
                   "id": "order_line_id",
@@ -2724,6 +2775,7 @@
               },
               "description": "Products",
               "facts": [],
+              "aggregatedFacts": [],
               "grain": [
                 {
                   "id": "product_id",
diff --git a/gooddata-sdk/tests/catalog/load/gooddata_layouts/default/workspaces/demo/ldm/datasets/campaign_channels_per_category.yaml b/gooddata-sdk/tests/catalog/load/gooddata_layouts/default/workspaces/demo/ldm/datasets/campaign_channels_per_category.yaml
@@ -0,0 +1,42 @@
+# (C) 2025 GoodData Corporation
+id: campaign_channels_per_category
+references:
+  - identifier:
+      id: campaign_channels
+      type: dataset
+    multivalue: false
+    sources:
+      - column: category
+        dataType: STRING
+        target:
+          id: category
+          type: attribute
+      - column: budget
+        dataType: INT
+        target:
+          id: budget
+          type: fact
+title: Campaign channels per category
+description: Campaign channels per categories
+attributes: []
+facts: []
+grain: []
+aggregatedFacts:
+  - id: budget_agg
+    sourceColumn: budget
+    sourceFactReference:
+      operation: SUM
+      reference:
+        id: budget
+        type: fact
+    sourceColumnDataType: INT
+    description: Budget Agg
+    tags:
+      - Campaign channels per category
+precedence: 1
+sql:
+  dataSourceId: demo-test-ds
+  statement: SELECT category, SUM(budget) FROM campaign_channels GROUP
+    BY category
+tags:
+  - Campaign channels per category
diff --git a/gooddata-sdk/tests/catalog/load/workspace_content/gooddata_layouts/default/workspaces/demo_testing/ldm/datasets/campaign_channels_per_category.yaml b/gooddata-sdk/tests/catalog/load/workspace_content/gooddata_layouts/default/workspaces/demo_testing/ldm/datasets/campaign_channels_per_category.yaml
@@ -0,0 +1,42 @@
+# (C) 2025 GoodData Corporation
+id: campaign_channels_per_category
+references:
+  - identifier:
+      id: campaign_channels
+      type: dataset
+    multivalue: false
+    sources:
+      - column: category
+        dataType: STRING
+        target:
+          id: category
+          type: attribute
+      - column: budget
+        dataType: INT
+        target:
+          id: budget
+          type: fact
+title: Campaign channels per category
+description: Campaign channels per categories
+attributes: []
+facts: []
+grain: []
+aggregatedFacts:
+  - id: budget_agg
+    sourceColumn: budget
+    sourceFactReference:
+      operation: SUM
+      reference:
+        id: budget
+        type: fact
+    sourceColumnDataType: INT
+    description: Budget Agg
+    tags:
+      - Campaign channels per category
+precedence: 1
+sql:
+  dataSourceId: demo-test-ds
+  statement: SELECT category, SUM(budget) FROM campaign_channels GROUP
+    BY category
+tags:
+  - Campaign channels per category
diff --git a/gooddata-sdk/tests/catalog/test_catalog_workspace.py b/gooddata-sdk/tests/catalog/test_catalog_workspace.py
@@ -701,8 +701,7 @@ def test_clone_workspace(test_config):
         default_cloned_decl_ws = sdk.catalog_workspace.get_declarative_workspace(default_cloned_ws_id)
         assert default_cloned_decl_ws.ldm.datasets[0].data_source_table_id.data_source_id == test_config["data_source2"]
         assert default_cloned_decl_ws.ldm.datasets[0].facts[0].source_column == "BUDGET"
-        # TODO: Add a nontrivial test for agg facts here
-        assert default_cloned_decl_ws.ldm.datasets[0].aggregated_facts == []
+        assert default_cloned_decl_ws.ldm.datasets[1].aggregated_facts[0].source_column == "BUDGET"
 
         sdk.catalog_workspace.clone_workspace(
             source_ws_id, target_workspace_id=custom_cloned_ws_id, target_workspace_name=custom_cloned_ws_name
@@ -756,7 +755,7 @@ def test_translate_workspace(test_config):
                 for fact in dataset.facts:
                     if fact.id == "budget":
                         assert fact.title == "Rozpočet"
-            # TODO: Add agg facts here for descriptions?
+            # TODO: Do for aggregated facts descriptions
 
             # Run second time without translation function. Previous execution created translation file, which is used.
             sdk.catalog_workspace.generate_localized_workspaces(
diff --git a/gooddata-sdk/tests/catalog/test_catalog_workspace_content.py b/gooddata-sdk/tests/catalog/test_catalog_workspace_content.py
@@ -63,8 +63,7 @@ def test_catalog_list_facts(test_config):
 def test_catalog_list_aggregated_facts(test_config):
     sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"])
     agg_facts_list = sdk.catalog_workspace_content.get_aggregated_facts_catalog(test_config["workspace"])
-    # TODO: Add a non-trivial test
-    assert len(agg_facts_list) == 0
+    assert len(agg_facts_list) == 1
 
 
 @gd_vcr.use_cassette(str(_fixtures_dir / "demo_catalog_list_attributes.yaml"))
@@ -129,7 +128,7 @@ def test_load_and_modify_ds_and_put_declarative_ldm(test_config):
     sdk.catalog_workspace.create_or_update(workspace)
 
     ldm_e = sdk.catalog_workspace_content.get_declarative_ldm(workspace_id)
-    ds_e = list(set([d.data_source_table_id.data_source_id for d in ldm_e.ldm.datasets]))
+    ds_e = list(set([d.data_source_table_id.data_source_id for d in ldm_e.ldm.datasets if d.data_source_table_id]))
     assert ds_e == [test_config["data_source"]]
 
     try:
@@ -147,7 +146,7 @@ def test_load_and_modify_ds_and_put_declarative_ldm(test_config):
         ldm_e.ldm.modify_mapped_data_source(data_source_mapping=reverse_data_source_mapping)
         sdk.catalog_workspace_content.put_declarative_ldm(identifier, ldm_e, validator, standalone_copy=True)
         ldm_o = sdk.catalog_workspace_content.get_declarative_ldm(identifier)
-        ds_o = list(set([d.data_source_table_id.data_source_id for d in ldm_o.ldm.datasets]))
+        ds_o = list(set([d.data_source_table_id.data_source_id for d in ldm_o.ldm.datasets if d.data_source_table_id]))
         assert ds_o == [test_config["data_source"]]
     finally:
         _refresh_workspaces(sdk)
@@ -166,7 +165,7 @@ def test_load_ldm_and_modify_tables_columns_case(test_config):
     assert ldm_e.ldm.datasets[0].data_source_table_id.id == table_id.upper()
     assert ldm_e.ldm.datasets[0].attributes[0].source_column == attribute_column.upper()
     assert ldm_e.ldm.datasets[0].facts[0].source_column == fact_column.upper()
-    # TODO: Add agg facts here
+    assert ldm_e.ldm.datasets[1].aggregated_facts[0].source_column == fact_column.upper()
     assert ldm_e.ldm.datasets[0].references[0].source_columns is None
     assert ldm_e.ldm.datasets[0].references[0].sources[0].column == reference_column.upper()
     # Test chaining approach as well
@@ -178,7 +177,7 @@ def test_load_ldm_and_modify_tables_columns_case(test_config):
     assert ldm_o.ldm.datasets[0].data_source_table_id.id == table_id
     assert ldm_o.ldm.datasets[0].attributes[0].source_column == attribute_column
     assert ldm_o.ldm.datasets[0].facts[0].source_column == fact_column
-    # TODO: Add agg facts here
+    assert ldm_o.ldm.datasets[1].aggregated_facts[0].source_column == fact_column
     assert ldm_o.ldm.datasets[0].references[0].source_columns is None
     assert ldm_e.ldm.datasets[0].references[0].sources[0].column == reference_column
 
@@ -313,7 +312,7 @@ def test_catalog_load(test_config):
 
     # rough initial smoke-test; just do a quick 'rub'
     assert len(catalog.metrics) == 24
-    assert len(catalog.datasets) == 6
+    assert len(catalog.datasets) == 7
 
     assert catalog.get_metric("order_amount") is not None
     assert catalog.get_metric("revenue") is not None
diff --git a/tests-support/fixtures/demo_declarative_hierarchy.json b/tests-support/fixtures/demo_declarative_hierarchy.json
@@ -2346,6 +2346,54 @@
               ],
               "title": "Campaign channels"
             },
+            {
+              "attributes": [],
+              "description": "Campaign channels per categories",
+              "facts": [],
+              "aggregatedFacts": [
+                {
+                  "description": "Budget Agg",
+                  "id": "budget_agg",
+                  "sourceColumn": "budget",
+                  "sourceColumnDataType": "INT",
+                  "tags": [
+                    "Campaign channels per category"
+                  ],
+                  "sourceFactReference": {
+                    "reference": {
+                      "id": "budget",
+                      "type": "fact"
+                    },
+                    "operation": "SUM"
+                  }
+                }
+              ],
+              "grain": [],
+              "id": "campaign_channels_per_category",
+              "precedence":  1,
+              "references": [
+                {
+                  "identifier": {
+                    "id": "campaign_channels",
+                    "type": "dataset"
+                  },
+                  "multivalue": false,
+                  "sourceColumns": [
+                    "category",
+                    "budget"
+                  ],
+                  "sourceColumnDataTypes": [ "STRING", "INT" ]
+                }
+              ],
+              "tags": [
+                "Campaign channels per category"
+              ],
+              "sql": {
+                "statement": "SELECT category, SUM(budget) FROM campaign_channels GROUP BY category",
+                "dataSourceId": "demo-test-ds"
+              },
+              "title": "Campaign channels per category"
+            },
             {
               "attributes": [
                 {
@@ -2672,11 +2720,21 @@
           "dateInstances": [
             {
               "granularities": [
+                "MINUTE",
+                "HOUR",
                 "DAY",
                 "MONTH",
                 "QUARTER",
                 "WEEK",
-                "YEAR"
+                "YEAR",
+                "MINUTE_OF_HOUR",
+                "HOUR_OF_DAY",
+                "DAY_OF_WEEK",
+                "DAY_OF_MONTH",
+                "DAY_OF_YEAR",
+                "WEEK_OF_YEAR",
+                "MONTH_OF_YEAR",
+                "QUARTER_OF_YEAR"
               ],
               "granularitiesFormatting": {
                 "titleBase": "",
diff --git a/tests-support/upload_demo_layout.py b/tests-support/upload_demo_layout.py
@@ -35,7 +35,7 @@ def rest_op(op, url_path, data=None, raise_ex=True):
 
     if response.status_code < 200 or response.status_code > 299:
         if raise_ex:
-            raise Exception(f"Call to {url} failed - {str(response)}")
+            raise Exception(f"Call to {url} failed - {str(response.text)}")
         else:
             return None
 

Original file line number	Diff line number	Diff line change
`@@ -145,6 +145,14 @@ def generate_facts_from_api(self) -> list[CatalogFact]:`
`145`	`145`	`default=attr.Factory(lambda self: self.generate_facts_from_api(), takes_self=True),`
`146`	`146`	`)`
`147`	`147`
	`148`	`+ # TODO: Doublecheck if we shouldn't do something like for facts`
	`149`	`+ aggregated_facts: Optional[list[CatalogAggregatedFact]] = attr.field(`
	`150`	`+ default=attr.Factory(lambda self: self.json_api_attributes.get("aggregatedFacts"), takes_self=True),`
	`151`	`+ )`
	`152`	`+ precedence: Optional[int] = attr.field(`
	`153`	`+ default=attr.Factory(lambda self: self.json_api_attributes.get("precedence"), takes_self=True)`
	`154`	`+ )`
	`155`	`+`
`148`	`156`	`grain: Optional[list] = attr.field(`
`149`	`157`	`default=attr.Factory(lambda self: self.json_api_attributes.get("grain"), takes_self=True)`
`150`	`158`	`)`