Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 36 additions & 43 deletions demos/reporting_pipeline.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -94,12 +94,9 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1781793606275,
"inputWidgets": {},
"nuid": "3a9e5503-3ba0-461a-a80b-116786825131",
"showTitle": false,
"startTime": 1781793599974,
"submitTime": 1781793599633,
"tableResultSettingsMap": {},
"title": ""
}
Expand Down Expand Up @@ -141,12 +138,9 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1781793606499,
"inputWidgets": {},
"nuid": "e6b082aa-cd90-4404-8d01-c4def6fba1d0",
"showTitle": false,
"startTime": 1781793606289,
"submitTime": 1781793599651,
"tableResultSettingsMap": {},
"title": ""
}
Expand All @@ -170,12 +164,9 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1781793606692,
"inputWidgets": {},
"nuid": "e10f7a71-bced-4bd8-944a-e849514810dc",
"showTitle": false,
"startTime": 1781793606538,
"submitTime": 1781793599653,
"tableResultSettingsMap": {},
"title": ""
}
Expand Down Expand Up @@ -251,12 +242,9 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1781793641047,
"inputWidgets": {},
"nuid": "852c8b37-8b00-416a-9caf-508562963ed6",
"showTitle": false,
"startTime": 1781793606709,
"submitTime": 1781793599671,
"tableResultSettingsMap": {},
"title": ""
}
Expand Down Expand Up @@ -295,12 +283,9 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1781793642921,
"inputWidgets": {},
"nuid": "d4ffcc59-1e8d-46bf-9f87-050786d77ae5",
"showTitle": false,
"startTime": 1781793641071,
"submitTime": 1781793599674,
"tableResultSettingsMap": {},
"title": ""
}
Expand All @@ -319,12 +304,9 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1781793644622,
"inputWidgets": {},
"nuid": "7659474f-f00e-4577-aa28-133387922e28",
"showTitle": false,
"startTime": 1781793642933,
"submitTime": 1781793599677,
"tableResultSettingsMap": {},
"title": ""
}
Expand Down Expand Up @@ -381,12 +363,9 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1781793647268,
"inputWidgets": {},
"nuid": "90785aab-526e-4afd-8156-cef0bb28b142",
"showTitle": false,
"startTime": 1781793644642,
"submitTime": 1781793599698,
"tableResultSettingsMap": {},
"title": ""
}
Expand Down Expand Up @@ -428,6 +407,11 @@
" \"container_id\", \"vehicle_key\",\n",
" \"start_ts\", \"stop_ts\",\n",
" ],\n",
" \"calculated_channels\": {\n",
" \"emit_channel_metrics\": True,\n",
" \"attribute_columns\": [],\n",
" \"kpis\": [\"duration\", \"min\", \"max\", \"mean\"],\n",
" },\n",
"}\n",
"\n",
"report = Report(\n",
Expand Down Expand Up @@ -470,12 +454,9 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1781793647472,
"inputWidgets": {},
"nuid": "8f9aebe1-97ec-417e-9072-e859eed8b86f",
"showTitle": false,
"startTime": 1781793647280,
"submitTime": 1781793599806,
"tableResultSettingsMap": {},
"title": ""
}
Expand Down Expand Up @@ -534,12 +515,9 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1781793647669,
"inputWidgets": {},
"nuid": "6e4fa193-a588-4bea-8849-abcc89aaac36",
"showTitle": false,
"startTime": 1781793647482,
"submitTime": 1781793599821,
"tableResultSettingsMap": {},
"title": ""
}
Expand Down Expand Up @@ -594,12 +572,9 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1781793647872,
"inputWidgets": {},
"nuid": "1bc57151-3988-4458-afef-fa258e8b74f8",
"showTitle": false,
"startTime": 1781793647688,
"submitTime": 1781793599839,
"tableResultSettingsMap": {},
"title": ""
}
Expand Down Expand Up @@ -668,12 +643,9 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1781793648182,
"inputWidgets": {},
"nuid": "4c31ea1c-64d0-4383-88bb-e363b2881d2c",
"showTitle": false,
"startTime": 1781793647893,
"submitTime": 1781793599855,
"tableResultSettingsMap": {},
"title": ""
}
Expand Down Expand Up @@ -751,6 +723,36 @@
"print(f\"{len(page.aggregations)} aggregations added\")"
]
},
{
"cell_type": "code",
"execution_count": 0,
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "41b9398e-6cc2-44e7-9320-0b47d55536a9",
"showTitle": true,
"tableResultSettingsMap": {},
"title": "Register Calculated Channel"
}
},
"outputs": [],
"source": [
"from impulse_reporting.channels.calculated_channel import CalculatedChannel\n",
"\n",
"avg_temp_channel = CalculatedChannel(\n",
" name=\"avg_temp\",\n",
" expr=avg_temp,\n",
" identity={\"channel_name\": \"avg_temp\", \"data_key\": \"CALC\"},\n",
" desc=\"Average of ambient and intake air temperature\",\n",
")\n",
"report.add_calculated_channel(avg_temp_channel)\n",
"print(\"Calculated channel 'avg_temp' registered\")"
]
},
{
"cell_type": "markdown",
"metadata": {
Expand Down Expand Up @@ -782,12 +784,9 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1781793726278,
"inputWidgets": {},
"nuid": "57dc4f62-2518-46b7-b418-7e3a9090860a",
"showTitle": false,
"startTime": 1781793648214,
"submitTime": 1781793599870,
"tableResultSettingsMap": {},
"title": ""
}
Expand Down Expand Up @@ -836,12 +835,9 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1781793733571,
"inputWidgets": {},
"nuid": "e8bc4484-f84a-45d7-b848-d9d17293e688",
"showTitle": true,
"startTime": 1781793726299,
"submitTime": 1781793599888,
"tableResultSettingsMap": {},
"title": "Visualize Gold Layer Results"
}
Expand Down Expand Up @@ -1016,12 +1012,9 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1781793733685,
"inputWidgets": {},
"nuid": "e71e2dd7-afdc-4468-a09f-ef0f94094a79",
"showTitle": false,
"startTime": 1781793733585,
"submitTime": 1781793599907,
"tableResultSettingsMap": {},
"title": ""
}
Expand Down Expand Up @@ -1144,7 +1137,7 @@
}
},
"schema": {
"currentValue": "default",
"currentValue": "thomas_bonfert",
"nuid": "6970e201-5950-4033-9f3c-54c0d3ba354f",
"typedWidgetInfo": {
"autoCreated": false,
Expand Down
34 changes: 34 additions & 0 deletions docs/impulse/docs/config/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,40 @@ mode-resolution rules and what counts as a definition change.

---

## calculated_channels (optional)

Controls the optional `calculated_channel_metrics` output. By default a report
writes calculated channels to `calculated_channel_fact` (the derived signal) and
`calculated_channel_dimension` (the definitions). Setting `emit_channel_metrics`
adds a third table, `calculated_channel_metrics`, shaped like the silver
`channel_metrics` table so the fact + metrics pair can serve as an Impulse silver
source. See the [Channels reference](../references/report/channel.md) for the
output schema.

| Field | Type | Default | Description |
|-----------------------|-------------|--------------------------------------|---------------------------------------------------------------------------------------------------|
| `emit_channel_metrics`| `bool` | `false` | Turns on the `calculated_channel_metrics` table. |
| `attribute_columns` | `list[str]` | `[]` | Calculated-channel `attributes` keys to surface as columns on the metrics table (e.g. `["unit"]`). |
| `kpis` | `list[str]` | `["duration", "min", "max", "mean"]` | KPIs computed per `(container_id, channel_id)`, one column each. Must be registered KPI names. |

When enabled, each row of `calculated_channel_metrics` is one
`(container_id, channel_id)` pair, carrying the selected `kpis` plus dynamic
identity columns (the union of `identity` keys across the report's channels) and
the configured `attribute_columns`. A channel that omits an identity or attribute
key gets `null` for that column; an identity key wins over an attribute key of the
same name.

The available `kpis` are `duration`, `min`, `max`, and `mean` (all
duration-weighted, matching the silver ingestion semantics). An unknown KPI name is
rejected at config validation with a `ValueError` naming the valid KPIs.

:::note Off by default
When `emit_channel_metrics` is `false` (the default), no metrics table is written
and `attribute_columns` / `kpis` have no effect.
:::

---

## measurement_dimensions (optional)

List of `container_metrics` column names to surface into the gold-layer
Expand Down
14 changes: 13 additions & 1 deletion docs/impulse/docs/data_model/gold_layer_event_normalized.md
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,17 @@ calculated_channel_fact {
timestamp _created_at
}

calculated_channel_metrics {
int container_id
long channel_id
string value_type
double duration
double min
double max
double mean
timestamp _created_at
}

histogram_fact }o--|| event_dimension: event_id
histogram2d_fact }o--|| event_dimension: event_id
stats_aggregator_fact }o--|| event_instance_fact: event_instance_id
Expand Down Expand Up @@ -221,7 +232,8 @@ guaranteed.
| `{prefix}_histogram2d_fact` | `container_id`, `visual_id`, `event_id`, `x_bin_id`, `y_bin_id` | 2D histogram bin values per container. |
| `{prefix}_stats_aggregator_fact` | `container_id`, `visual_id`, `event_instance_id`, `channel_name`, `aggregation_label` | Statistics values per signal, event instance, and container. |
| `{prefix}_event_instance_fact` | `container_id`, `event_id`, `event_instance_id` | Materialized event occurrences with start/end timestamps. |
| `{prefix}_calculated_channel_fact` | `container_id`, `channel_id`, `tstart` | Materialized derived signal — one row per sample interval, in the silver `channels` shape (`tstart`, `tend`, `value`). The channel's identity lives on `calculated_channel_dimension`, joined via `channel_id`. |
| `{prefix}_calculated_channel_fact` | `container_id`, `channel_id`, `tstart` | Materialized derived signal, one row per sample interval, in the silver `channels` shape (`tstart`, `tend`, `value`). The channel's identity lives on `calculated_channel_dimension`, joined via `channel_id`. |
| `{prefix}_calculated_channel_metrics` | `container_id`, `channel_id` | Optional per-channel metrics in the silver `channel_metrics` shape, so the fact + metrics pair can serve as an Impulse silver source. Written only when [`config.calculated_channels.emit_channel_metrics`](../config/configuration.md#calculated_channels-optional) is set. Carries the configured `kpis` plus dynamic identity/attribute columns. See [Channels](../references/report/channel.md). |

---

Expand Down
1 change: 1 addition & 0 deletions docs/impulse/docs/data_model/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ The Gold layer uses a **star schema** with fact and dimension tables. All table
| `histogram2d_fact` | One row per (x, y) bin per container | 2D histogram bin values, duration-weighted. |
| `stats_aggregator_fact` | One row per statistic label per signal per event instance | Descriptive statistics (built-in min/max/mean/median and any custom statistics). |
| `calculated_channel_fact` | One row per sample interval per container | Materialized derived signal (a *channel*, not a summary), in the silver `channels` shape. |
| `calculated_channel_metrics` | One row per calculated channel per container | Optional per-channel metrics in the silver `channel_metrics` shape. Written only when `config.calculated_channels.emit_channel_metrics` is set. |

### Dimension tables

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,6 @@ Enumeration of supported statistic types for aggregations.
- `MAX` (`str`): Maximum value statistic.
- `MEAN` (`str`): Mean (average) value statistic.
- `MEDIAN` (`str`): Median value statistic.
- `START` (`str`): First value in the interval.
- `END` (`str`): Last value in the interval.

Original file line number Diff line number Diff line change
Expand Up @@ -190,14 +190,16 @@ Only includes computation-affecting attributes:
- input_expressions
- statistics to be calculated
- event expression if there is any
- custom statistics (name, kind, declared input indices, and function
bytecode, so implementation or input-wiring changes invalidate cached
results; only appended when custom statistics are configured so
aggregators without them keep their previous hash)

Excludes: name, desc, signal_name, units, page_number, report_id, and the
cross-channel descriptors' channel_name (presentation metadata, like
channel_names).
- channel_names, and each cross-channel descriptor's channel_name. These
are the fact table's ``channel_name`` merge key, so a rename must force
a recompute (a changed definition recomputes and prunes all containers);
otherwise, in incremental mode, already-processed containers would keep
rows under the old name.
- custom statistics (labels, kind, declared input indices, params, and
function bytecode, so implementation or input-wiring changes invalidate
cached results; only appended when custom statistics are configured)

Excludes: name, desc, units, page_number, report_id.

**Returns**:

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -178,3 +178,50 @@ Create the dimension DataFrame for the given channels.
which ``createDataFrame`` builds directly from the plain dict returned by


#### determine\_channel\_metrics

```python
def determine_channel_metrics(
cls,
spark: SparkSession,
channels: list[CalculatedChannel],
fact_df: DataFrame | None,
*,
attribute_columns: list[str] | None = None,
kpis: list[str] | None = None) -> DataFrame | None
```

Derive a silver-shaped ``channel_metrics`` DataFrame from the fact rows.

The calculated-channel fact table already matches the silver ``channels``
table; this builds its companion ``channel_metrics`` so the pair can serve
as an Impulse silver source. Metrics are aggregated **directly from the
narrow fact rows** (``container_id, channel_id, tstart, tend, value``),
grouped by ``(container_id, channel_id)``.

The output schema is **dynamic**: fixed columns ``container_id,
channel_id, value_type`` plus one column per configured KPI (see ``kpis``),
one per identity key (the union across all ``channels``), and one per
configured attribute key. Identity/attribute values are pulled from
each channel's in-memory ``identity`` / ``attributes`` dicts (null where a
channel omits a key). On an identity/attribute key collision, identity wins
and the attribute is skipped.

**Arguments**:

- `spark` (`SparkSession`): Session used to build the per-channel metadata frame.
- `channels` (`list of CalculatedChannel`): The channels whose fact rows are in ``fact_df``; supply identity and
attributes.
- `fact_df` (`DataFrame or None`): Narrow fact DataFrame (output of :meth:`determine_calculated_channels`).
``None`` returns ``None``.
- `attribute_columns` (`list of str`): Attribute keys to surface as columns. Default/empty → no attribute
columns. A key no channel defines yields an all-null column.
- `kpis` (`list of str`): KPI names to compute (see ``calculated_channel_kpis.KPI_BUILDERS``); the
output carries one column per name, in order. ``None`` → the default
KPIs (``duration, min, max, mean``).

**Returns**:

`DataFrame or None`: The dynamic-schema metrics DataFrame, or ``None`` when ``fact_df`` is
``None``.

Loading
Loading