diff --git a/Makefile b/Makefile index 3d13e279..445f0aba 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,9 @@ coverage: build: uv build --require-hashes --build-constraints=.build-constraints.txt +update-api-docs: + cd docs/impulse && uv run pydoc-markdown + lock-dependencies: UV_LOCKED := 0 lock-dependencies: uv lock @@ -56,4 +59,4 @@ fork-sync: ./.github/scripts/fork-sync-pr.sh $(PR) .DEFAULT: all -.PHONY: all clean dev lint fmt test coverage build lock-dependencies fork-sync +.PHONY: all clean dev lint fmt test coverage build update-api-docs lock-dependencies fork-sync diff --git a/demos/data/reporting/channel_metrics.csv b/demos/data/reporting/channel_metrics.csv index 5637f583..46fe67c0 100644 --- a/demos/data/reporting/channel_metrics.csv +++ b/demos/data/reporting/channel_metrics.csv @@ -1,13 +1,19 @@ -container_id,channel_id,sample_count,min,max,mean,begin_ms,end_ms,duration_ms,sample_rate,value_type -1,4,56667,-11.0,-3.0,-8.35955670848995,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE -1,5,56667,0.0,3385.0,1572.765489614767,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE -1,7,56667,0.0,201.0,70.67870188998889,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE -1,9,56667,-33.0,130.0,2.3332627455132617,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE -2,2,47336,-8.0,6.0,-6.105247591684975,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE -2,5,47336,0.0,3177.0,1736.7744422849416,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE -2,7,47336,0.0,188.0,83.95660807841811,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE -2,8,47336,-26.0,117.0,0.8931046138245733,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE -3,2,54775,-32.0,134.0,9.632861706983112,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE -3,7,54775,0.0,2545.0,1308.4330990415335,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE -3,9,54775,0.0,125.0,44.4629849383843,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE -3,10,54775,-3.0,2.0,-2.157334550433592,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE +container_id,channel_id,sample_count,min,max,mean,begin_ms,end_ms,duration_ms,sample_rate,value_type +1,4,56667,-11.0,-3.0,-8.35955670848995,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE +1,5,56667,0.0,3385.0,1572.765489614767,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE +1,7,56667,0.0,201.0,70.67870188998889,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE +1,9,56667,-33.0,130.0,2.3332627455132617,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE +2,2,47336,-8.0,6.0,-6.105247591684975,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE +2,5,47336,0.0,3177.0,1736.7744422849416,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE +2,7,47336,0.0,188.0,83.95660807841811,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE +2,8,47336,-26.0,117.0,0.8931046138245733,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE +3,2,54775,-32.0,134.0,9.632861706983112,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE +3,7,54775,0.0,2545.0,1308.4330990415335,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE +3,9,54775,0.0,125.0,44.4629849383843,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE +3,10,54775,-3.0,2.0,-2.157334550433592,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE +1,90,3,,,,1519629856439000,1519633356439000,3500000,,STRING +1,91,3,1.0,3.0,2.0,1519629856439000,1519633356439000,3500000,,DOUBLE +2,90,2,,,,1519756824107000,1519758824107000,2000000,,STRING +2,91,2,1.0,2.0,1.5,1519756824107000,1519758824107000,2000000,,DOUBLE +3,90,1,,,,1519926478375000,1519926478375000,0,,STRING +3,91,1,1.0,1.0,1.0,1519926478375000,1519926478375000,0,,DOUBLE diff --git a/demos/data/reporting/channel_tags.csv b/demos/data/reporting/channel_tags.csv index a35cf010..59185950 100644 --- a/demos/data/reporting/channel_tags.csv +++ b/demos/data/reporting/channel_tags.csv @@ -119,3 +119,39 @@ container_id,channel_id,key,value 3,10,model,Leon 3,10,to_city,RT 3,10,unit,C +1,90,brand,Seat +1,90,channel_name,DTC +1,90,model,Leon +1,90,experiment_id,experiment_4 +1,90,ecu,Engine_ECU +1,90,bus,CAN1 +1,90,code_system,P +1,91,brand,Seat +1,91,channel_name,DTC_count +1,91,model,Leon +1,91,experiment_id,experiment_4 +1,91,ecu,Engine_ECU +2,90,brand,Seat +2,90,channel_name,DTC +2,90,model,Leon +2,90,experiment_id,experiment_4 +2,90,ecu,Engine_ECU +2,90,bus,CAN1 +2,90,code_system,P +2,91,brand,Seat +2,91,channel_name,DTC_count +2,91,model,Leon +2,91,experiment_id,experiment_4 +2,91,ecu,Engine_ECU +3,90,brand,Seat +3,90,channel_name,DTC +3,90,model,Leon +3,90,experiment_id,experiment_4 +3,90,ecu,Body_ECU +3,90,bus,CAN2 +3,90,code_system,U +3,91,brand,Seat +3,91,channel_name,DTC_count +3,91,model,Leon +3,91,experiment_id,experiment_4 +3,91,ecu,Body_ECU diff --git a/demos/data/reporting/poi_channels.csv b/demos/data/reporting/poi_channels.csv new file mode 100644 index 00000000..84357279 --- /dev/null +++ b/demos/data/reporting/poi_channels.csv @@ -0,0 +1,13 @@ +container_id,channel_id,timestamp,value_double,value_string,dtype +1,90,1519629856439000,,P0301,string +1,90,1519631856439000,,P0301,string +1,90,1519633356439000,,P0135,string +1,91,1519629856439000,1.0,,double +1,91,1519631856439000,2.0,,double +1,91,1519633356439000,3.0,,double +2,90,1519756824107000,,P0420,string +2,90,1519758824107000,,P0128,string +2,91,1519756824107000,1.0,,double +2,91,1519758824107000,2.0,,double +3,90,1519926478375000,,U0100,string +3,91,1519926478375000,1.0,,double diff --git a/demos/reporting_pipeline.ipynb b/demos/reporting_pipeline.ipynb index 8c3e5ebd..6dabbe94 100644 --- a/demos/reporting_pipeline.ipynb +++ b/demos/reporting_pipeline.ipynb @@ -16,21 +16,21 @@ } }, "source": [ - "# Impulse — Reporting Pipeline Demo\n", + "# Impulse \u2014 Reporting Pipeline Demo\n", "\n", "The **Impulse Framework** is a Python library that enables\n", "automotive and industrial engineers to process, aggregate,\n", "and analyze petabytes of time-series measurement data on\n", - "Databricks — without requiring Apache Spark expertise.\n", + "Databricks \u2014 without requiring Apache Spark expertise.\n", "\n", "It provides **TSAL** (Time Series Analytics Language),\n", "a Pythonic expression language for defining signals,\n", "events, and aggregations.\n", "\n", "**What this notebook builds:**\n", - "A complete reporting pipeline — RPM histograms,\n", + "A complete reporting pipeline \u2014 RPM histograms,\n", "RPM-vs-speed heatmaps, per-distance-bin statistics, and\n", - "channel values sampled at every 10 km milestone — across\n", + "channel values sampled at every 10 km milestone \u2014 across\n", "3 test drives, persisted as a Gold-layer star schema and\n", "visualized inline with matplotlib.\n", "\n", @@ -61,9 +61,9 @@ "\n", "Impulse sits between a governed silver layer and a gold-layer star schema in Unity Catalog and provides three components:\n", "\n", - "- **TSAL (Time Series Analytics Language)** — a declarative Python DSL for expressing signals, events, and aggregations in natural Python, without requiring Spark expertise.\n", - "- **Query Engine** — pluggable and distributed; compiles TSAL expressions into Spark execution plans and adapts to any silver-layer layout via interchangeable solvers.\n", - "- **Aggregations** — domain-aware physical aggregations, including duration- and distance-weighted 1D/2D histograms and event-scoped statistics." + "- **TSAL (Time Series Analytics Language)** \u2014 a declarative Python DSL for expressing signals, events, and aggregations in natural Python, without requiring Spark expertise.\n", + "- **Query Engine** \u2014 pluggable and distributed; compiles TSAL expressions into Spark execution plans and adapts to any silver-layer layout via interchangeable solvers.\n", + "- **Aggregations** \u2014 domain-aware physical aggregations, including duration- and distance-weighted 1D/2D histograms and event-scoped statistics." ] }, { @@ -94,9 +94,12 @@ "byteLimit": 2048000, "rowLimit": 10000 }, + "finishTime": 1781793606275, "inputWidgets": {}, "nuid": "3a9e5503-3ba0-461a-a80b-116786825131", "showTitle": false, + "startTime": 1781793599974, + "submitTime": 1781793599633, "tableResultSettingsMap": {}, "title": "" } @@ -138,9 +141,12 @@ "byteLimit": 2048000, "rowLimit": 10000 }, + "finishTime": 1781793606499, "inputWidgets": {}, "nuid": "e6b082aa-cd90-4404-8d01-c4def6fba1d0", "showTitle": false, + "startTime": 1781793606289, + "submitTime": 1781793599651, "tableResultSettingsMap": {}, "title": "" } @@ -164,9 +170,12 @@ "byteLimit": 2048000, "rowLimit": 10000 }, + "finishTime": 1781793606692, "inputWidgets": {}, "nuid": "e10f7a71-bced-4bd8-944a-e849514810dc", "showTitle": false, + "startTime": 1781793606538, + "submitTime": 1781793599653, "tableResultSettingsMap": {}, "title": "" } @@ -229,7 +238,7 @@ " (e.g., one test drive)\n", "- **Channel** = one sensor signal within a container\n", " (e.g., Engine RPM), stored as raw\n", - " `(timestamp, value)` samples — the framework\n", + " `(timestamp, value)` samples \u2014 the framework\n", " automatically converts these to intervals on the fly" ] }, @@ -242,9 +251,12 @@ "byteLimit": 2048000, "rowLimit": 10000 }, + "finishTime": 1781793641047, "inputWidgets": {}, "nuid": "852c8b37-8b00-416a-9caf-508562963ed6", "showTitle": false, + "startTime": 1781793606709, + "submitTime": 1781793599671, "tableResultSettingsMap": {}, "title": "" } @@ -263,7 +275,7 @@ "SILVER = [\n", " \"container_metrics\", \"container_tags\",\n", " \"channel_metrics\", \"channel_tags\",\n", - " \"channels\",\n", + " \"channels\", \"poi_channels\",\n", "]\n", "for t in SILVER:\n", " pdf = pd.read_csv(f\"{csv_dir}/{t}.csv\")\n", @@ -283,9 +295,12 @@ "byteLimit": 2048000, "rowLimit": 10000 }, + "finishTime": 1781793642921, "inputWidgets": {}, "nuid": "d4ffcc59-1e8d-46bf-9f87-050786d77ae5", "showTitle": false, + "startTime": 1781793641071, + "submitTime": 1781793599674, "tableResultSettingsMap": {}, "title": "" } @@ -304,9 +319,12 @@ "byteLimit": 2048000, "rowLimit": 10000 }, + "finishTime": 1781793644622, "inputWidgets": {}, "nuid": "7659474f-f00e-4577-aa28-133387922e28", "showTitle": false, + "startTime": 1781793642933, + "submitTime": 1781793599677, "tableResultSettingsMap": {}, "title": "" } @@ -344,13 +362,13 @@ "# 2. Initialize the Report\n", "\n", "The `Report` orchestrator takes a config specifying:\n", - "- **`source`** — Silver layer tables\n", - "- **`unity_sink`** — Gold layer output\n", - "- **`query_engine.solver`** — `DefaultSolver` for\n", + "- **`source`** \u2014 Silver layer tables\n", + "- **`unity_sink`** \u2014 Gold layer output\n", + "- **`query_engine.solver`** \u2014 `DefaultSolver` for\n", " parallel per-container execution\n", - "- **`query_engine.data_type`** — `RAW` for raw\n", + "- **`query_engine.data_type`** \u2014 `RAW` for raw\n", " timestamp data (auto-converted to intervals)\n", - "- **`measurement_dimensions`** — container metadata\n", + "- **`measurement_dimensions`** \u2014 container metadata\n", " to carry into Gold layer" ] }, @@ -363,9 +381,12 @@ "byteLimit": 2048000, "rowLimit": 10000 }, + "finishTime": 1781793647268, "inputWidgets": {}, "nuid": "90785aab-526e-4afd-8156-cef0bb28b142", "showTitle": false, + "startTime": 1781793644642, + "submitTime": 1781793599698, "tableResultSettingsMap": {}, "title": "" } @@ -391,6 +412,7 @@ " \"container_metrics_table\": f\"{pfx}_container_metrics\",\n", " \"channel_metrics_table\": f\"{pfx}_channel_metrics\",\n", " \"channels_uri\": f\"{pfx}_channels\",\n", + " \"poi_channels_uri\": f\"{pfx}_poi_channels\",\n", " \"container_tags_table\": f\"{pfx}_container_tags\",\n", " \"channel_tags_table\": f\"{pfx}_channel_tags\",\n", " },\n", @@ -407,11 +429,6 @@ " \"container_id\", \"vehicle_key\",\n", " \"start_ts\", \"stop_ts\",\n", " ],\n", - " \"calculated_channels\": {\n", - " \"emit_channel_metrics\": True,\n", - " \"attribute_columns\": [],\n", - " \"kpis\": [\"duration\", \"min\", \"max\", \"mean\"],\n", - " },\n", "}\n", "\n", "report = Report(\n", @@ -440,7 +457,7 @@ "source": [ "# 3. Select Physical Channels\n", "\n", - "Channels are selected by **metadata tags** —\n", + "Channels are selected by **metadata tags** \u2014\n", "no column names, no SQL, no joins.\n", "These are **lazy expressions**: no data is read yet." ] @@ -454,9 +471,12 @@ "byteLimit": 2048000, "rowLimit": 10000 }, + "finishTime": 1781793647472, "inputWidgets": {}, "nuid": "8f9aebe1-97ec-417e-9072-e859eed8b86f", "showTitle": false, + "startTime": 1781793647280, + "submitTime": 1781793599806, "tableResultSettingsMap": {}, "title": "" } @@ -481,6 +501,51 @@ ")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3a. Select POI Channels (Diagnostic Trouble Codes)\n", + "\n", + "Not every channel is a continuous signal. A **Points-in-Time (POI)** channel is\n", + "an *event stream*: each value exists **only at its timestamp**, with no validity\n", + "in between. The textbook example is **DTCs** (Diagnostic Trouble Codes) \u2014 the\n", + "fault codes an ECU emits at the instant it detects a problem (`P0301` = cylinder-1\n", + "misfire, \u2026).\n", + "\n", + "POI channels are selected with **`poi_channel(...)`** instead of `channel(...)`.\n", + "Identification is identical (same metadata tags); only the semantics differ \u2014\n", + "a POI channel solves to a `PointsInTimeSeries`, not a `SampleSeries`.\n", + "\n", + "| | `channel(...)` | `poi_channel(...)` |\n", + "|---|---|---|\n", + "| Shape | `[tstart, tend)` intervals | `(t\u1d62, v\u1d62)` points |\n", + "| Valid between points? | yes (interpolated) | **no** |\n", + "| Backed by | `SampleSeries` | `PointsInTimeSeries` |" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "# String POI channel: the DTC code emitted at each fault instant.\n", + "# dtype=\"string\" -> equality is the natural operation (\"when did P0301 occur?\"),\n", + "# never arithmetic or ordering on a code.\n", + "dtc = db.query.poi_channel(\n", + " channel_name=\"DTC\",\n", + " dtype=\"string\",\n", + " brand=\"Seat\", model=\"Leon\",\n", + ")\n", + "\n", + "# Numeric POI channel: a running fault-occurrence counter.\n", + "dtc_count = db.query.poi_channel(\n", + " channel_name=\"DTC_count\",\n", + " brand=\"Seat\", model=\"Leon\",\n", + ")" + ] + }, { "cell_type": "markdown", "metadata": { @@ -500,7 +565,7 @@ "# 4. Define Virtual Signals & Events\n", "\n", "**TSAL** uses Python operators to build lazy\n", - "expression trees — no Spark knowledge needed.\n", + "expression trees \u2014 no Spark knowledge needed.\n", "\n", "**Virtual signals** derive from physical channels.\n", "**Events** are time windows where a condition holds." @@ -515,9 +580,12 @@ "byteLimit": 2048000, "rowLimit": 10000 }, + "finishTime": 1781793647669, "inputWidgets": {}, "nuid": "6e4fa193-a588-4bea-8849-abcc89aaac36", "showTitle": false, + "startTime": 1781793647482, + "submitTime": 1781793599821, "tableResultSettingsMap": {}, "title": "" } @@ -536,8 +604,18 @@ ")\n", "\n", "# Instant the trip odometer crosses each additional\n", - "# 10 km — a set of points in time, not an interval.\n", - "distance_milestones = (distance_km % 10).falling_edges()" + "# 10 km \u2014 a set of points in time, not an interval.\n", + "distance_milestones = (distance_km % 10).falling_edges()\n", + "\n", + "# POI as a time-window anchor. A POI instant has no duration, but we often\n", + "# want the signal *around* it. `.expand(w)` turns each P0301 instant into a\n", + "# [t - w, t + w] interval (w in the data's time unit \u2014 microseconds here),\n", + "# so `\u00b110 s` is 10e6. Overlapping windows are merged.\n", + "WINDOW_US = 10e6 # \u00b110 seconds\n", + "p0301_window = (dtc == \"P0301\").expand(WINDOW_US)\n", + "\n", + "# All Engine RPM samples recorded within \u00b110 s of a misfire.\n", + "rpm_around_p0301 = eng_rpm.where(p0301_window)" ] }, { @@ -558,9 +636,11 @@ "source": [ "# 5. Register Events\n", "\n", - "- **BasicEvent** — from a TSAL boolean expression\n", - "- **ContainerEvent** — spans the entire recording\n", - "- **PointsInTimeEvent** — a set of instants (e.g. each 10 km milestone)" + "- **BasicEvent** \u2014 from a TSAL boolean expression\n", + "- **ContainerEvent** \u2014 spans the entire recording\n", + "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone)\n", + "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone, or each **P0301 misfire** from the DTC channel)\n", + "- **BasicEvent on a POI window** \u2014 `(dtc == \"P0301\").expand(\u00b110 s)` turns each fault instant into an interval, so you can aggregate the signal *around* each event" ] }, { @@ -572,9 +652,12 @@ "byteLimit": 2048000, "rowLimit": 10000 }, + "finishTime": 1781793647872, "inputWidgets": {}, "nuid": "1bc57151-3988-4458-afef-fa258e8b74f8", "showTitle": false, + "startTime": 1781793647688, + "submitTime": 1781793599839, "tableResultSettingsMap": {}, "title": "" } @@ -607,7 +690,26 @@ " expr=distance_milestones,\n", " desc=\"Each 10 km driven (instant)\",\n", ")\n", - "report.add_event(milestone_event)" + "report.add_event(milestone_event)\n", + "\n", + "# POI freeze-frame: the instants a P0301 misfire was logged.\n", + "# `dtc == \"P0301\"` is a PointsInTime \u2014 the set of timestamps where the\n", + "# string code equals P0301 \u2014 exactly what a PointsInTimeEvent wants.\n", + "p0301_event = PointsInTimeEvent(\n", + " name=\"p0301_misfires\",\n", + " expr=(dtc == \"P0301\"),\n", + " desc=\"Each instant a P0301 misfire code was set\",\n", + ")\n", + "report.add_event(p0301_event)\n", + "\n", + "# The \u00b110 s misfire windows as an interval event, so aggregations can run\n", + "# \"within 10 s of a P0301\" the same way they run within any other event.\n", + "p0301_window_event = BasicEvent(\n", + " name=\"p0301_window\",\n", + " expr=p0301_window,\n", + " desc=\"Within \u00b110 s of a P0301 misfire\",\n", + ")\n", + "report.add_event(p0301_window_event)" ] }, { @@ -628,10 +730,10 @@ "source": [ "# 6. Define Aggregations\n", "\n", - "- **Histogram** — 1D duration-weighted distribution\n", - "- **Histogram2D** — 2D heatmap of two signals\n", - "- **StatisticsAggregator** — min, median, mean, max per event\n", - "- **PointValueAggregator** — channel value sampled at each instant of a points-in-time event" + "- **Histogram** \u2014 1D duration-weighted distribution\n", + "- **Histogram2D** \u2014 2D heatmap of two signals\n", + "- **StatisticsAggregator** \u2014 min, median, mean, max per event\n", + "- **PointValueAggregator** \u2014 channel value sampled at each instant of a points-in-time event" ] }, { @@ -643,9 +745,12 @@ "byteLimit": 2048000, "rowLimit": 10000 }, + "finishTime": 1781793648182, "inputWidgets": {}, "nuid": "4c31ea1c-64d0-4383-88bb-e363b2881d2c", "showTitle": false, + "startTime": 1781793647893, + "submitTime": 1781793599855, "tableResultSettingsMap": {}, "title": "" } @@ -712,7 +817,7 @@ "))\n", "\n", "# Sample Vehicle Speed & Engine RPM at each 10 km\n", - "# milestone — one value per channel per instant.\n", + "# milestone \u2014 one value per channel per instant.\n", "page.add_aggregation(PointValueAggregator(\n", " name=\"values_at_distance_milestones\",\n", " input_expressions=[veh_spd, eng_rpm],\n", @@ -720,37 +825,63 @@ " event=milestone_event,\n", " desc=\"Speed & RPM at each 10 km milestone\",\n", "))\n", + "\n", + "# \u2500\u2500 POI aggregations \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n", + "# Freeze-frame: Engine RPM & Vehicle Speed at each P0301 misfire instant.\n", + "# The POI event supplies the timestamps; the sample channels supply the\n", + "# values valid there \u2014 both series types in one aggregation.\n", + "page.add_aggregation(PointValueAggregator(\n", + " name=\"values_at_p0301\",\n", + " input_expressions=[eng_rpm, veh_spd],\n", + " channel_names=[\"Engine RPM\", \"Vehicle Speed\"],\n", + " event=p0301_event,\n", + " desc=\"RPM & Speed at each P0301 misfire\",\n", + "))\n", + "\n", + "\n", + "# Derived-value-around-POI: the Engine RPM distribution within \u00b110 s of each\n", + "# misfire. The window event filters the (SampleSeries) channel to those\n", + "# intervals, so a duration-weighted histogram shows *what the engine was doing*\n", + "# around the fault \u2014 richer than a single min/mean/max.\n", + "page.add_aggregation(HistogramDuration(\n", + " name=\"rpm_around_p0301\",\n", + " base_expr=eng_rpm,\n", + " bins=[float(i) for i in range(0, 5000, 250)],\n", + " event=p0301_window_event,\n", + " desc=\"Engine RPM distribution within \u00b110 s of a P0301 misfire\",\n", + " channel_name=\"Engine RPM\",\n", + " bins_unit=\"RPM\", values_unit=\"s\",\n", + "))\n", + "\n", "print(f\"{len(page.aggregations)} aggregations added\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Numeric POI: fault counts per recording\n", + "\n", + "A numeric POI channel reduces like any signal \u2014 but the reductions are\n", + "**unweighted** (points have no duration). `count()` / `max()` on `dtc_count`\n", + "answer \"how many faults did each recording log?\" directly from the query engine.\n", + "\n", + "(The report-level `StatsAggregator` is designed for continuous `SampleSeries`\n", + "inputs, so a per-container POI count is shown here as a direct query instead.)" + ] + }, { "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "41b9398e-6cc2-44e7-9320-0b47d55536a9", - "showTitle": true, - "tableResultSettingsMap": {}, - "title": "Register Calculated Channel" - } - }, + "metadata": {}, + "execution_count": null, "outputs": [], "source": [ - "from impulse_reporting.channels.calculated_channel import CalculatedChannel\n", + "dtc_summary = db.query.select(\n", + " dtc_count.count().alias(\"n_faults\"),\n", + " dtc_count.max().alias(\"peak_count\"),\n", + ").solve(spark=spark, solver=report.get_solver())\n", "\n", - "avg_temp_channel = CalculatedChannel(\n", - " name=\"avg_temp\",\n", - " expr=avg_temp,\n", - " identity={\"channel_name\": \"avg_temp\", \"data_key\": \"CALC\"},\n", - " desc=\"Average of ambient and intake air temperature\",\n", - ")\n", - "report.add_calculated_channel(avg_temp_channel)\n", - "print(\"Calculated channel 'avg_temp' registered\")" + "display(dtc_summary.orderBy(\"container_id\"))" ] }, { @@ -771,8 +902,8 @@ "source": [ "# 7. Compute & Persist\n", "\n", - "- `determine_report()` — parallel execution\n", - "- `persist_results()` — writes star schema" + "- `determine_report()` \u2014 parallel execution\n", + "- `persist_results()` \u2014 writes star schema" ] }, { @@ -784,9 +915,12 @@ "byteLimit": 2048000, "rowLimit": 10000 }, + "finishTime": 1781793726278, "inputWidgets": {}, "nuid": "57dc4f62-2518-46b7-b418-7e3a9090860a", "showTitle": false, + "startTime": 1781793648214, + "submitTime": 1781793599870, "tableResultSettingsMap": {}, "title": "" } @@ -819,11 +953,13 @@ "Read the Gold-layer tables back and render the\n", "results inline with **matplotlib**:\n", "\n", - "- **Bar** — RPM histogram\n", - "- **Heatmap** — RPM vs Speed\n", - "- **Table** — per-container statistics\n", - "- **Scatter** — Speed & RPM at each 10 km milestone\n", - " (markers only — values exist only *at* each instant)" + "- **Bar** \u2014 RPM histogram\n", + "- **Heatmap** \u2014 RPM vs Speed\n", + "- **Table** \u2014 per-container statistics\n", + "- **Scatter** \u2014 Speed & RPM at each 10 km milestone\n", + " (markers only \u2014 values exist only *at* each instant)\n", + "\n", + "Includes two **POI** views: Engine RPM sampled at each P0301 misfire (freeze-frame), and fault-code counts per recording." ] }, { @@ -835,9 +971,12 @@ "byteLimit": 2048000, "rowLimit": 10000 }, + "finishTime": 1781793733571, "inputWidgets": {}, "nuid": "e8bc4484-f84a-45d7-b848-d9d17293e688", "showTitle": true, + "startTime": 1781793726299, + "submitTime": 1781793599888, "tableResultSettingsMap": {}, "title": "Visualize Gold Layer Results" } @@ -846,12 +985,12 @@ "source": [ "import matplotlib.pyplot as plt\n", "\n", - "# ─── Table prefix ───\n", + "# \u2500\u2500\u2500 Table prefix \u2500\u2500\u2500\n", "T = f\"{pfx}\"\n", "\n", - "# ════════════════════════════════════════════════════════════════════\n", - "# 1. BAR — RPM Histogram (aggregated across all containers)\n", - "# ════════════════════════════════════════════════════════════════════\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 1. BAR \u2014 RPM Histogram (aggregated across all containers)\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", "hist_df = (\n", " spark.read.table(f\"{T}_histogram_fact\")\n", " .join(\n", @@ -869,14 +1008,14 @@ "ax.bar(hist_df[\"bin_name\"], hist_df[\"duration_s\"], color=\"steelblue\", edgecolor=\"white\")\n", "ax.set_xlabel(\"Engine RPM bin\")\n", "ax.set_ylabel(\"Duration (s)\")\n", - "ax.set_title(\"RPM Histogram — Duration in Each RPM Band (all containers)\")\n", + "ax.set_title(\"RPM Histogram \u2014 Duration in Each RPM Band (all containers)\")\n", "plt.xticks(rotation=45, ha=\"right\", fontsize=8)\n", "plt.tight_layout()\n", "plt.show()\n", "\n", - "# ════════════════════════════════════════════════════════════════════\n", - "# 2. HEATMAP — RPM vs Speed\n", - "# ════════════════════════════════════════════════════════════════════\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 2. HEATMAP \u2014 RPM vs Speed\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", "heat_df = (\n", " spark.read.table(f\"{T}_histogram2d_fact\")\n", " .groupBy(\"x_bin_id\", \"y_bin_id\", \"x_bin_name\", \"y_bin_name\",\n", @@ -913,14 +1052,14 @@ "ax.set_yticklabels([lbl[1] for lbl in y_labels], fontsize=7)\n", "ax.set_xlabel(\"Engine RPM\")\n", "ax.set_ylabel(\"Vehicle Speed (km/h)\")\n", - "ax.set_title(\"RPM vs Speed Heatmap — Duration (s)\")\n", + "ax.set_title(\"RPM vs Speed Heatmap \u2014 Duration (s)\")\n", "plt.colorbar(im, ax=ax, label=\"Duration (s)\")\n", "plt.tight_layout()\n", "plt.show()\n", "\n", - "# ════════════════════════════════════════════════════════════════════\n", - "# 3. TABLE — Per-container Statistics (container_stats)\n", - "# ════════════════════════════════════════════════════════════════════\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 3. TABLE \u2014 Per-container Statistics (container_stats)\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", "stats_df = (\n", " spark.read.table(f\"{T}_stats_aggregator_fact\")\n", " .join(\n", @@ -948,9 +1087,9 @@ " ),\n", ")\n", "\n", - "# ════════════════════════════════════════════════════════════════════\n", - "# 4. SCATTER — Speed & RPM at Each 10 km Milestone\n", - "# ════════════════════════════════════════════════════════════════════\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 4. SCATTER \u2014 Speed & RPM at Each 10 km Milestone\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", "milestone_df = (\n", " spark.read.table(f\"{T}_stats_aggregator_fact\")\n", " .join(\n", @@ -979,7 +1118,70 @@ "ax.set_title(\"Speed & RPM at Each 10 km Milestone\")\n", "ax.legend()\n", "plt.tight_layout()\n", - "plt.show()" + "plt.show()\n", + "\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 5. POI \u2014 Engine RPM at Each P0301 Misfire (freeze-frame)\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# PointValueAggregator writes to the shared stats_aggregator_fact table;\n", + "# select its visual by name, like the milestone scatter above.\n", + "p0301_df = (\n", + " spark.read.table(f\"{T}_stats_aggregator_fact\")\n", + " .join(\n", + " spark.read.table(f\"{T}_stats_aggregator_dimension\")\n", + " .filter(\"name = 'values_at_p0301'\"),\n", + " on=\"visual_id\",\n", + " )\n", + " .select(\"container_id\", \"channel_name\", \"event_instance_id\", \"statistic_value\")\n", + " .toPandas()\n", + ")\n", + "\n", + "if not p0301_df.empty:\n", + " rpm_hits = p0301_df[p0301_df[\"channel_name\"] == \"Engine RPM\"]\n", + " fig, ax = plt.subplots(figsize=(10, 4))\n", + " for cid, grp in rpm_hits.groupby(\"container_id\"):\n", + " ax.scatter(\n", + " grp[\"event_instance_id\"], grp[\"statistic_value\"],\n", + " label=f\"Container {cid}\", s=90, alpha=0.85,\n", + " edgecolors=\"k\", linewidths=0.5,\n", + " )\n", + " ax.set_xlabel(\"P0301 occurrence #\")\n", + " ax.set_ylabel(\"Engine RPM at fault instant\")\n", + " ax.set_title(\"Freeze-frame: Engine RPM at each P0301 misfire\")\n", + " ax.legend()\n", + " plt.tight_layout()\n", + " plt.show()\n", + "else:\n", + " print(\"No P0301 misfires in the demo data.\")\n", + "\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 6. POI WINDOW \u2014 Engine RPM distribution within \u00b110 s of each P0301\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "win_hist = (\n", + " spark.read.table(f\"{T}_histogram_fact\")\n", + " .join(\n", + " spark.read.table(f\"{T}_histogram_dimension\")\n", + " .filter(\"name = 'rpm_around_p0301'\"),\n", + " on=\"visual_id\",\n", + " )\n", + " .groupBy(\"bin_id\", \"bin_name\")\n", + " .agg(F.sum(\"hist_value\").alias(\"total_us\"))\n", + " .orderBy(\"bin_id\")\n", + " .toPandas()\n", + ")\n", + "if not win_hist.empty:\n", + " win_hist[\"duration_s\"] = win_hist[\"total_us\"] / 1e6\n", + " fig, ax = plt.subplots(figsize=(10, 4))\n", + " ax.bar(win_hist[\"bin_name\"], win_hist[\"duration_s\"],\n", + " color=\"indianred\", edgecolor=\"white\")\n", + " ax.set_xlabel(\"Engine RPM bin\")\n", + " ax.set_ylabel(\"Duration (s)\")\n", + " ax.set_title(\"Engine RPM distribution within \u00b110 s of a P0301 misfire\")\n", + " plt.xticks(rotation=45, ha=\"right\", fontsize=8)\n", + " plt.tight_layout()\n", + " plt.show()\n", + "else:\n", + " print(\"No P0301 misfire windows in the demo data.\")" ] }, { @@ -1012,9 +1214,12 @@ "byteLimit": 2048000, "rowLimit": 10000 }, + "finishTime": 1781793733685, "inputWidgets": {}, "nuid": "e71e2dd7-afdc-4468-a09f-ef0f94094a79", "showTitle": false, + "startTime": 1781793733585, + "submitTime": 1781793599907, "tableResultSettingsMap": {}, "title": "" } @@ -1137,7 +1342,7 @@ } }, "schema": { - "currentValue": "thomas_bonfert", + "currentValue": "default", "nuid": "6970e201-5950-4033-9f3c-54c0d3ba354f", "typedWidgetInfo": { "autoCreated": false, @@ -1203,4 +1408,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} +} \ No newline at end of file diff --git a/docs/impulse/docs/config/configuration.md b/docs/impulse/docs/config/configuration.md index 8e7cdc55..310daf50 100644 --- a/docs/impulse/docs/config/configuration.md +++ b/docs/impulse/docs/config/configuration.md @@ -138,7 +138,7 @@ Two independent filter families: | `data_type` | `str` | `"RLE"` | `"RLE"` (intervals `[tstart, tend)`) or `"RAW"` (raw timestamps; converted to intervals at query time by the configured `raw_encoder`). | | `raw_encoder` | `str` | `null` (resolves to `"RLE"` when `data_type = "RAW"`) | How RAW point data is converted to intervals. `"RLE"` (the default) run-length encodes on the fly to reduce memory consumption: consecutive samples with the same value collapse into a single `[tstart, tend)` interval per run. `"INTERVAL"` keeps every original sample, only deriving `tend` from the next sample's timestamp and dropping exact duplicate points — use it when downstream analysis needs all original timestamps. Only takes effect with `data_type = "RAW"`; ignored for `"RLE"` input. See [How Impulse interprets intervals](../data_model/silver_layer_schema.md#raw-format) for validity semantics and what counts as a duplicate point. | | `drop_implausible_data` | `bool` | `false` | When `true`, drops `channels` rows where `is_plausible = false`. Requires `data_type = "RAW"`; combining with `"RLE"` raises a validation error. | -| `batch_size` | `int` | `500` | Maximum number of selectors solved per batch. Applies to events, aggregations, and calculated channels. | +| `batch_size` | `int` | `500` | Maximum number of selectors solved per batch. | | `solver_config` | `SolverConfig` | `null` | Per-table column mappings, per-table equality filters, and project scoping. Set `project_id` to scope reads by project — it is applied to `container_tags` (if configured), `container_metrics`, and `channel_mapping` (if configured), so it works in both narrow EAV and wide-only data models. Omit it when you don't need project scoping. See [Solver column mappings and filters](#solver-column-mappings-and-filters). | If `query_engine` is omitted, the default is `DefaultSolver` with @@ -353,40 +353,6 @@ mode-resolution rules and what counts as a definition change. --- -## calculated_channels (optional) - -Controls the optional `calculated_channel_metrics` output. By default a report -writes calculated channels to `calculated_channel_fact` (the derived signal) and -`calculated_channel_dimension` (the definitions). Setting `emit_channel_metrics` -adds a third table, `calculated_channel_metrics`, shaped like the silver -`channel_metrics` table so the fact + metrics pair can serve as an Impulse silver -source. See the [Channels reference](../references/report/channel.md) for the -output schema. - -| Field | Type | Default | Description | -|-----------------------|-------------|--------------------------------------|---------------------------------------------------------------------------------------------------| -| `emit_channel_metrics`| `bool` | `false` | Turns on the `calculated_channel_metrics` table. | -| `attribute_columns` | `list[str]` | `[]` | Calculated-channel `attributes` keys to surface as columns on the metrics table (e.g. `["unit"]`). | -| `kpis` | `list[str]` | `["duration", "min", "max", "mean"]` | KPIs computed per `(container_id, channel_id)`, one column each. Must be registered KPI names. | - -When enabled, each row of `calculated_channel_metrics` is one -`(container_id, channel_id)` pair, carrying the selected `kpis` plus dynamic -identity columns (the union of `identity` keys across the report's channels) and -the configured `attribute_columns`. A channel that omits an identity or attribute -key gets `null` for that column; an identity key wins over an attribute key of the -same name. - -The available `kpis` are `duration`, `min`, `max`, and `mean` (all -duration-weighted, matching the silver ingestion semantics). An unknown KPI name is -rejected at config validation with a `ValueError` naming the valid KPIs. - -:::note Off by default -When `emit_channel_metrics` is `false` (the default), no metrics table is written -and `attribute_columns` / `kpis` have no effect. -::: - ---- - ## measurement_dimensions (optional) List of `container_metrics` column names to surface into the gold-layer diff --git a/docs/impulse/docs/data_model/gold_layer_event_normalized.md b/docs/impulse/docs/data_model/gold_layer_event_normalized.md index 7e28f768..df45d8b7 100644 --- a/docs/impulse/docs/data_model/gold_layer_event_normalized.md +++ b/docs/impulse/docs/data_model/gold_layer_event_normalized.md @@ -186,17 +186,6 @@ calculated_channel_fact { timestamp _created_at } -calculated_channel_metrics { - int container_id - long channel_id - string value_type - double duration - double min - double max - double mean - timestamp _created_at -} - histogram_fact }o--|| event_dimension: event_id histogram2d_fact }o--|| event_dimension: event_id stats_aggregator_fact }o--|| event_instance_fact: event_instance_id @@ -232,8 +221,7 @@ guaranteed. | `{prefix}_histogram2d_fact` | `container_id`, `visual_id`, `event_id`, `x_bin_id`, `y_bin_id` | 2D histogram bin values per container. | | `{prefix}_stats_aggregator_fact` | `container_id`, `visual_id`, `event_instance_id`, `channel_name`, `aggregation_label` | Statistics values per signal, event instance, and container. | | `{prefix}_event_instance_fact` | `container_id`, `event_id`, `event_instance_id` | Materialized event occurrences with start/end timestamps. | -| `{prefix}_calculated_channel_fact` | `container_id`, `channel_id`, `tstart` | Materialized derived signal, one row per sample interval, in the silver `channels` shape (`tstart`, `tend`, `value`). The channel's identity lives on `calculated_channel_dimension`, joined via `channel_id`. | -| `{prefix}_calculated_channel_metrics` | `container_id`, `channel_id` | Optional per-channel metrics in the silver `channel_metrics` shape, so the fact + metrics pair can serve as an Impulse silver source. Written only when [`config.calculated_channels.emit_channel_metrics`](../config/configuration.md#calculated_channels-optional) is set. Carries the configured `kpis` plus dynamic identity/attribute columns. See [Channels](../references/report/channel.md). | +| `{prefix}_calculated_channel_fact` | `container_id`, `channel_id`, `tstart` | Materialized derived signal — one row per sample interval, in the silver `channels` shape (`tstart`, `tend`, `value`). The channel's identity lives on `calculated_channel_dimension`, joined via `channel_id`. | --- diff --git a/docs/impulse/docs/data_model/index.md b/docs/impulse/docs/data_model/index.md index 7dc4373d..fc123425 100644 --- a/docs/impulse/docs/data_model/index.md +++ b/docs/impulse/docs/data_model/index.md @@ -62,7 +62,6 @@ The Gold layer uses a **star schema** with fact and dimension tables. All table | `histogram2d_fact` | One row per (x, y) bin per container | 2D histogram bin values, duration-weighted. | | `stats_aggregator_fact` | One row per statistic label per signal per event instance | Descriptive statistics (built-in min/max/mean/median and any custom statistics). | | `calculated_channel_fact` | One row per sample interval per container | Materialized derived signal (a *channel*, not a summary), in the silver `channels` shape. | -| `calculated_channel_metrics` | One row per calculated channel per container | Optional per-channel metrics in the silver `channel_metrics` shape. Written only when `config.calculated_channels.emit_channel_metrics` is set. | ### Dimension tables diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md index 253c03d9..67c0c089 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md @@ -3,6 +3,39 @@ sidebar_label: time_series_expression title: impulse_query_engine.analyze.metadata.time_series_expression --- +## SeriesType + +```python +class SeriesType(StrEnum) +``` + +How a channel's samples are interpreted (mirrors :class:`RawEncoder`). + +``SAMPLE`` — the default; ``[tstart, tend)`` intervals over which the value is +*valid* (reconstructed by an interpolation method, zero-order hold today), +backed by :class:`SampleSeries`. + +``POINTS_IN_TIME`` — ``(tᵢ, vᵢ)`` points valid *only at* their timestamps, no +between-point validity, backed by :class:`PointsInTimeSeries`. + + +## PoiValueType + +```python +class PoiValueType(StrEnum) +``` + +The value data type of a POI channel — selects its ``poi_channels`` value + +column and which in-memory :class:`PointsInTimeSeries` variant is built. + +``DOUBLE`` — numeric points (``poi_channels.value_double``); the full +arithmetic / ordering / reduction operator set applies. + +``STRING`` — string points (``poi_channels.value_string``, e.g. DTC codes); +only sampling and equality apply (see :class:`PointsInTimeSeries`). + + ## TimeSeriesSelector ```python @@ -12,7 +45,10 @@ class TimeSeriesSelector(TimeSeriesExpression, RequiresDeserialization) #### \_\_init\_\_ ```python -def __init__(expr, uses_alias: bool = False) +def __init__(expr, + uses_alias: bool = False, + series_type: SeriesType = SeriesType.SAMPLE, + value_type: PoiValueType = PoiValueType.DOUBLE) ``` Initialize a TimeSeriesSelector. @@ -20,6 +56,18 @@ Initialize a TimeSeriesSelector. **Arguments**: - `expr` (`TagExpression`): Tag expression to select. +- `uses_alias` (`bool`): Whether the channel resolves via the channel-alias table. +- `series_type` (`SeriesType`): How the selected channel's samples are interpreted. ``SAMPLE`` +(default) builds a :class:`SampleSeries` — today's behavior, +unchanged. ``POINTS_IN_TIME`` builds a :class:`PointsInTimeSeries` +(values valid only at their timestamps); identification / matching is +identical, only the built object and its result dtype differ. This is +the plan-time source of truth for the series type (so ``dtype()`` is +correct for a bare POI selection with no per-channel metadata lookup). +- `value_type` (`PoiValueType`): For a ``POINTS_IN_TIME`` selection, the declared value data type +(``DOUBLE`` / ``STRING``). Ignored for ``SAMPLE``. Drives plan-time +typing and string-op gating; validated against the silver +``poi_channels.dtype`` at solve time (assertion contract). #### dtype @@ -31,7 +79,10 @@ Returns the Spark data type. **Returns**: -`pyspark.sql.types.DataType`: Data type (BinaryType). +`pyspark.sql.types.DataType`: ``BinaryType`` for a SAMPLE selection (serialized ``SampleSeries``), +or the value-type-aware ``PointsInTimeSeries.dtype()`` for a +POINTS_IN_TIME selection (``array>`` for numeric, +``array>`` for string). #### deserialize @@ -39,7 +90,11 @@ Returns the Spark data type. def deserialize(d) ``` -Deserialize sample series after collection/toPandas. +Deserialize a SAMPLE result after collection/toPandas. + +POINTS_IN_TIME results are serialized by ``get_data()`` (a plain +``[[t, v], ...]`` list) and need no deserialization, so they are returned +as-is; only a SAMPLE (binary) blob is decoded to a :class:`SampleSeries`. **Arguments**: @@ -47,23 +102,21 @@ Deserialize sample series after collection/toPandas. **Returns**: -`SampleSeries`: Deserialized sample series. +`SampleSeries or Any`: Deserialized sample series (SAMPLE), else *d* unchanged. #### build ```python -def build(cache: SeriesCache) -> SampleSeries +def build(cache: SeriesCache) ``` -Instantiate a SampleSeries from given cache data. +Instantiate the selected series from cache data. -**Arguments**: - -- `cache` (`SeriesCache`): Cache containing time series data. +Resolution is identical regardless of series type — resolve the matching +candidates, take the first ``(container_id, channel_id)``, and let the +cache build the right object. The **data** is authoritative for the built +type: :meth:`TimeSeriesCache.load_blob` returns a -**Returns**: - -`SampleSeries`: Built sample series. #### get\_required\_tag\_exprs diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md index 40fffeb0..1f5fb82e 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md @@ -119,6 +119,40 @@ Create a time series selector for the given channel tags. `TimeSeriesSelector`: Time series selector object. +#### poi\_channel + +```python +def poi_channel(dtype: PoiValueType = PoiValueType.DOUBLE, + **kwargs) -> TimeSeriesSelector +``` + +Create a Points-in-Time (POI) channel selector. + +Parallel to :meth:`channel` — it builds the **same** ``TimeSeriesSelector`` +from a tag/column match on ``**kwargs`` (e.g. +``poi_channel(channel_name="DTC")``), differing only in that it is stamped +``series_type=POINTS_IN_TIME`` (so it solves to a +:class:`~impulse_query_engine.model.series.points_in_time_series.PointsInTimeSeries` +— a value valid only *at* each timestamp — rather than a ``SampleSeries``) +and carries the declared value ``dtype``. + +Channel *identification* (tag/column match, ``get_selector_expr``, +``required_tags``, ``selector_id``) is identical to :meth:`channel`; only +the built object and its result dtype differ. + +**Arguments**: + +- `dtype` (`PoiValueType`): The POI channel's value data type: ``DOUBLE`` (default, numeric) or +``STRING`` (e.g. DTC codes — only sampling and equality apply). This +declared type drives plan-time result typing and string-op gating; it +is validated against the silver ``poi_channels.dtype`` at solve time +(an actual/declared mismatch raises). +- `**kwargs` (`dict`): Channel tag-value pairs, matched exactly like :meth:`channel`'s. + +**Returns**: + +`TimeSeriesSelector`: A selector stamped ``series_type=POINTS_IN_TIME`` with the given value type. + #### select ```python diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md index 8d65a48d..e16b99cb 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md @@ -233,6 +233,42 @@ def value_col() -> str Internal column name for the signal value on the channels table. +#### poi\_timestamp\_col + +```python +def poi_timestamp_col() -> str +``` + +Internal column name for the point timestamp on the poi_channels table. + + +#### poi\_value\_double\_col + +```python +def poi_value_double_col() -> str +``` + +Internal column name for the numeric value on the poi_channels table. + + +#### poi\_value\_string\_col + +```python +def poi_value_string_col() -> str +``` + +Internal column name for the string value on the poi_channels table. + + +#### poi\_dtype\_col + +```python +def poi_dtype_col() -> str +``` + +Internal column name for the per-row value-dtype discriminator on poi_channels. + + #### tag\_key\_col ```python diff --git a/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md b/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md index 50cbf77b..bdf55c39 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md @@ -24,6 +24,11 @@ A PointsInTimeSeries associates a value to each timestamp. Unlike a SampleSeries a value is only defined *at* its timestamp and is not considered valid in between consecutive timestamps. +The value type (numeric vs string) is inferred from *values*. An **empty** +series has no values to infer from and therefore defaults to numeric; use +:meth:`empty_string` when an explicitly string-typed empty series is needed +(e.g. plan-time result typing of a bare string-POI selection). + **Arguments**: - `tstarts` (`Sized`): Array-like of time points. @@ -37,9 +42,14 @@ def dtype() Returns the Spark data type for PointsInTimeSeries. +For numeric values the element is a homogeneous ``[tstart, value]`` double +pair (``ArrayType(ArrayType(DoubleType))``). String-valued series cannot use +that homogeneous nested array, so their element is a ``(tstart, value)`` +struct with a double timestamp and a string value. + **Returns**: -`pyspark.sql.types.ArrayType`: Spark ArrayType for points in time series: [[tstart_1, value_1], ...]. +`pyspark.sql.types.ArrayType`: Spark ArrayType matching ``get_data``'s shape for this series' value type. #### get\_data @@ -47,11 +57,16 @@ Returns the Spark data type for PointsInTimeSeries. def get_data() -> list ``` -Returns the series as a list of [tstart, value] lists. +Returns the series as a list of ``[tstart, value]`` pairs. + +For numeric values this is a list of two-element double lists. For string +values, ``column_stack`` would coerce the timestamps to strings, so the +pairs are built explicitly as ``[float(tstart), str(value)]`` — matching the +struct element type declared by :meth:`dtype`. **Returns**: -`list`: List of [tstart, value] pairs. +`list`: List of ``[tstart, value]`` pairs. #### \_\_len\_\_ @@ -410,9 +425,27 @@ Returns a string representation for debugging. def empty() -> PointsInTimeSeries ``` -Returns an empty PointsInTimeSeries. +Returns an empty (numeric) PointsInTimeSeries. + +**Returns**: + +`PointsInTimeSeries`: Empty numeric PointsInTimeSeries object. + +#### empty\_string + +```python +def empty_string() -> PointsInTimeSeries +``` + +Returns an empty **string-valued** PointsInTimeSeries. + +An empty series has no values to infer a type from, so the constructor +defaults to numeric; this factory forces the string value type. Used for +plan-time result typing of a bare string-POI selection, where the empty +series must report the string ``dtype()`` and reject numeric-only ops +(e.g. ``mean()``) before any data is read. **Returns**: -`PointsInTimeSeries`: Empty PointsInTimeSeries object. +`PointsInTimeSeries`: Empty string-valued PointsInTimeSeries object. diff --git a/docs/impulse/docs/references/api/impulse_reporting/channels/calculated_channel.md b/docs/impulse/docs/references/api/impulse_reporting/channels/calculated_channel.md index a80318fa..f2eadaf8 100644 --- a/docs/impulse/docs/references/api/impulse_reporting/channels/calculated_channel.md +++ b/docs/impulse/docs/references/api/impulse_reporting/channels/calculated_channel.md @@ -18,13 +18,10 @@ is driven by :class:`Report` to compute the channel across containers, persist the narrow result to a gold fact table, and update it incrementally. Structurally parallels :class:`BasicEvent` (holds an aliased expression, -name-derived id, SHA-256 definition hash). Like the other entity types, its -narrow solve is batched from :class:`Report` (see -``Report._solve_calculated_channels_batched``): the channels are partitioned by -``batch_size``, each batch solved via ``QueryBuilder.solve_calculated_channels`` -and persisted as a temp table, then the batches are unioned into a narrow -``solved_df``. :meth:`determine_calculated_channels` shapes that already-solved -df, mirroring ``determine_aggregations`` / ``determine_events``. +name-derived id, SHA-256 definition hash) but — like ``ContainerEvent`` — it +drives its own solve via ``QueryBuilder.solve_calculated_channels`` rather than +riding the centralized wide ``solved_df``. It is dispatched separately from +the batch solve (never passed to ``collect_solvable_expressions``). **Arguments**: @@ -143,30 +140,30 @@ def determine_calculated_channels( spark: SparkSession, channels: list[CalculatedChannel], *, - solved_df: DataFrame = None) -> DataFrame | None + query: QueryBuilder = None, + solver: QuerySolver = None, + pre_filtered_containers_df: DataFrame = None) -> DataFrame | None ``` -Shape the already-solved narrow rows into this type's fact rows. +Solve the given channels and shape the result into fact rows. -Mirrors ``determine_aggregations`` / ``determine_events``: the batched -narrow solve happens in the ``Report`` (see -``Report._solve_calculated_channels_batched``), and this only *shapes* the -resulting ``solved_df``: it selects the rows for these channels (by -``channel_id``) and projects to :data:`CALCULATED_CHANNEL_FACT_SCHEMA`. -Each channel's ``channel_id`` was fixed to its entity id at construction, -so the filter needs no join. +Drives ``QueryBuilder.solve_calculated_channels`` (the narrow, many-rows- +per-container endpoint) with the report's ``query`` + ``solver``, then +projects to :data:`CALCULATED_CHANNEL_FACT_SCHEMA`. Because each channel's +``channel_id`` was fixed to its entity id at construction, no id-join is +needed. **Arguments**: -- `spark` (`SparkSession`): Active Spark session (unused; kept for dispatcher symmetry). -- `channels` (`list of CalculatedChannel`): The channels whose rows to select from ``solved_df``. -- `solved_df` (`DataFrame`): Narrow batched solve output (``container_id, channel_id, tstart, tend, -value, identity``). ``None`` (no channels solved) returns ``None``. +- `spark` (`SparkSession`): Spark session, forwarded to ``QueryBuilder.solve_calculated_channels``. +- `channels` (`list of CalculatedChannel`): Channels to solve; identity keys may differ across channels. +- `query` (`QueryBuilder`): Query builder used to select and solve the channels. +- `solver` (`QuerySolver`): Solver implementing ``solve_calculated_channels`` (a ``DefaultSolver``). +- `pre_filtered_containers_df` (`DataFrame`): Incremental container subset; ``None`` processes all containers. **Returns**: -`DataFrame or None`: Narrow fact DataFrame, or ``None`` when there are no channels or no -solved rows. +`DataFrame or None`: Narrow fact DataFrame, or ``None`` when there are no channels. #### determine\_metadata\_df @@ -181,50 +178,3 @@ Create the dimension DataFrame for the given channels. which ``createDataFrame`` builds directly from the plain dict returned by -#### determine\_channel\_metrics - -```python -def determine_channel_metrics( - cls, - spark: SparkSession, - channels: list[CalculatedChannel], - fact_df: DataFrame | None, - *, - attribute_columns: list[str] | None = None, - kpis: list[str] | None = None) -> DataFrame | None -``` - -Derive a silver-shaped ``channel_metrics`` DataFrame from the fact rows. - -The calculated-channel fact table already matches the silver ``channels`` -table; this builds its companion ``channel_metrics`` so the pair can serve -as an Impulse silver source. Metrics are aggregated **directly from the -narrow fact rows** (``container_id, channel_id, tstart, tend, value``), -grouped by ``(container_id, channel_id)``. - -The output schema is **dynamic**: fixed columns ``container_id, -channel_id, value_type`` plus one column per configured KPI (see ``kpis``), -one per identity key (the union across all ``channels``), and one per -configured attribute key. Identity/attribute values are pulled from -each channel's in-memory ``identity`` / ``attributes`` dicts (null where a -channel omits a key). On an identity/attribute key collision, identity wins -and the attribute is skipped. - -**Arguments**: - -- `spark` (`SparkSession`): Session used to build the per-channel metadata frame. -- `channels` (`list of CalculatedChannel`): The channels whose fact rows are in ``fact_df``; supply identity and -attributes. -- `fact_df` (`DataFrame or None`): Narrow fact DataFrame (output of :meth:`determine_calculated_channels`). -``None`` returns ``None``. -- `attribute_columns` (`list of str`): Attribute keys to surface as columns. Default/empty → no attribute -columns. A key no channel defines yields an all-null column. -- `kpis` (`list of str`): KPI names to compute (see ``calculated_channel_kpis.KPI_BUILDERS``); the -output carries one column per name, in order. ``None`` → the default -KPIs (``duration, min, max, mean``). - -**Returns**: - -`DataFrame or None`: The dynamic-schema metrics DataFrame, or ``None`` when ``fact_df`` is -``None``. - diff --git a/docs/impulse/docs/references/api/impulse_reporting/channels/channel_types.md b/docs/impulse/docs/references/api/impulse_reporting/channels/channel_types.md index 1446ad39..46e7e05d 100644 --- a/docs/impulse/docs/references/api/impulse_reporting/channels/channel_types.md +++ b/docs/impulse/docs/references/api/impulse_reporting/channels/channel_types.md @@ -67,28 +67,6 @@ Get the dimension table name for the channel type. `str`: The name of the dimension table associated with this channel type. -#### get\_metrics\_table\_name - -```python -def get_metrics_table_name() -> str -``` - -Get the (optional) channel-metrics table name for the channel type. - -This table mirrors the silver-layer ``channel_metrics`` table so the -calculated-channel fact + metrics pair can serve as an Impulse silver -source. Unlike the fact/dimension tables it has **no** fixed schema -constant: identity/attribute columns are derived dynamically per report -(see :meth:`CalculatedChannel.determine_channel_metrics`). - -**Raises**: - -- `ValueError`: If the channel type is not supported. - -**Returns**: - -`str`: The name of the channel-metrics table associated with this channel type. - #### get\_dimension\_schema ```python @@ -125,26 +103,6 @@ Return the first ChannelType whose fact table name matches. `ChannelType`: -#### get\_any\_for\_metrics\_table - -```python -def get_any_for_metrics_table(cls, table_name: str) -> "ChannelType" -``` - -Return the first ChannelType whose metrics table name matches. - -**Arguments**: - -- `table_name` (`str`): Metrics table name to look up. - -**Raises**: - -- `ValueError`: If no ChannelType matches the given table name. - -**Returns**: - -`ChannelType`: - #### get\_any\_for\_dimension\_table ```python diff --git a/docs/impulse/docs/references/api/impulse_reporting/config/config_parser.md b/docs/impulse/docs/references/api/impulse_reporting/config/config_parser.md index 12099bc9..5201c851 100644 --- a/docs/impulse/docs/references/api/impulse_reporting/config/config_parser.md +++ b/docs/impulse/docs/references/api/impulse_reporting/config/config_parser.md @@ -241,27 +241,6 @@ Configuration for incremental processing behavior. - `silver_last_modified_column` (`str, default="timestamp"`): Column name in the silver layer used for freshness comparison. - `gold_last_modified_column` (`str, default="last_modified"`): Column name in the gold layer used for freshness comparison. -## CalculatedChannels - -```python -class CalculatedChannels(BaseModel) -``` - -Configuration for calculated-channel outputs. - -**Arguments**: - -- `emit_channel_metrics` (`bool, default=False`): When True, also emit a ``calculated_channel_metrics`` gold table (silver -``channel_metrics`` shape) alongside the calculated-channel fact table, so -the fact + metrics pair can serve as an Impulse silver source. -- `attribute_columns` (`list of str, default=[]`): Calculated-channel attribute keys to surface as columns on the metrics -table (e.g. ``["unit"]``). Empty (the default) → no attribute columns. -Identity keys are always surfaced dynamically and win over an -attribute key of the same name. -- `kpis` (`list of str, default=["duration", "min", "max", "mean"]`): KPIs computed on the metrics table, one column per name. Each must be a -registered KPI (see ``calculated_channel_kpis.KPI_BUILDERS``); an unknown -name is rejected at validation. Duplicates are removed (order preserved). - ## ImpulseConfig ```python @@ -282,9 +261,6 @@ Attributes Optional query engine configuration. Defaults to Solvers.DEFAULT_SOLVER. incremental : IncrementalConfig, optional Optional incremental processing configuration. Defaults to IncrementalConfig(). - calculated_channels : CalculatedChannels, optional - Optional calculated-channel output configuration (e.g. opting in to the - ``calculated_channel_metrics`` table). Defaults to CalculatedChannels(). measurement_dimensions : list of str, optional Column names to surface from ``container_metrics`` into the gold-layer ``measurement_dimension`` table. Names are matched diff --git a/docs/impulse/docs/references/report/channel.md b/docs/impulse/docs/references/report/channel.md index 7c035c94..fb2c6342 100644 --- a/docs/impulse/docs/references/report/channel.md +++ b/docs/impulse/docs/references/report/channel.md @@ -65,7 +65,7 @@ speed_kmh = CalculatedChannel( ### How it works 1. The `expr` is evaluated per measurement container by the solver, yielding a `SampleSeries` for each container. -2. The channels' narrow solve is **batched from the `Report`** (the same pattern events and aggregations use): the channels are partitioned by `query_engine.batch_size`, each batch is solved via `QueryBuilder.solve_calculated_channels` and persisted as a temporary table (`__impulse_temp_{run_id}_{batch_idx}`), and the batches are unioned into the final `calculated_channel_fact`. Multi-channel expressions (e.g. `rpm + speed`) are time-base synchronized automatically; emitted intervals are the intersection. +2. Calculated channels take their **own narrow solve path** (`QueryBuilder.solve_calculated_channels`), distinct from the wide batch solve used by events and aggregations. Multi-channel expressions (e.g. `rpm + speed`) are time-base synchronized automatically; emitted intervals are the intersection. 3. Each series is exploded into one row per sample interval: `container_id, channel_id, tstart, tend, value`. The `identity` is **not** on the fact — it lives on the dimension, joined via `channel_id`. 4. The `channel_id` is a deterministic hash of the sorted `identity` — the same value appears in both the fact and dimension tables, so they join directly. 5. The sample rows are written to the `calculated_channel_fact` table; the channel definition (name, description, expression, identity) is written to the `calculated_channel_dimension` table. @@ -138,32 +138,3 @@ run-length-encoded shape as the silver `channels` table. Both tables carry the configurable `table_prefix` (e.g. `{prefix}_calculated_channel_fact`). See the [gold layer schema](../../data_model/gold_layer_event_normalized.md) for how they fit the star schema, and [incremental processing](./index.md#incremental-processing) for how definition changes are reprocessed. - ---- - -## Optional channel metrics table - -Because `calculated_channel_fact` already matches the silver `channels` shape, a calculated channel needs -only a companion `channel_metrics` table to serve as an Impulse silver source in its own right. Set -[`config.calculated_channels.emit_channel_metrics`](../../config/configuration.md#calculated_channels-optional) -to also write a `calculated_channel_metrics` table, shaped like the silver `channel_metrics` table. - -### calculated_channel_metrics - -One row per `(container_id, channel_id)`, derived directly from the fact rows. The schema is **dynamic**: -fixed columns plus one column per configured KPI, one per identity key (the union of `identity` keys across -the report's channels), and one per configured attribute key. - -| Column | Type | Description | -|----------------|----------|-------------------------------------------------------------------------------------------------| -| `container_id` | `int` | Container identifier. Type is inherited from the silver source. | -| `channel_id` | `long` | Calculated-channel identifier (matches the fact and dimension). | -| *identity cols*| `str` | One column per identity key (e.g. `channel_name`, `data_key`); `null` where a channel omits it. | -| *attribute cols*| `str` | One column per `attribute_columns` entry (e.g. `unit`); `null` where a channel omits it. | -| `value_type` | `str` | `"double"`. | -| *kpi cols* | `double` | One column per configured KPI, in order (default `duration`, `min`, `max`, `mean`). | - -The KPIs are duration-weighted (matching the silver ingestion semantics): `duration` is the span -`max(tend) - min(tstart)`, `min` / `max` ignore NaN values, and `mean` is the duration-weighted average. -Select which KPIs to compute via `config.calculated_channels.kpis`; an identity key wins over an attribute -key of the same name. diff --git a/poi_series_integration.md b/poi_series_integration.md new file mode 100644 index 00000000..5c975be4 --- /dev/null +++ b/poi_series_integration.md @@ -0,0 +1,896 @@ +--- +sidebar_position: 1 +title: POI Series Integration +--- + +# Design: Integrating Points-in-Time (POI) Series into the Silver Layer + +**Status:** Proposed  ·  **Scope:** `impulse_query_engine` silver-layer +data model + `DefaultSolver` solve stage  ·  **Non-goal:** changing the +6-stage filter pipeline. + +## 1. Summary + +Impulse currently models every channel as a **sample series** — a sequence of +`[tstart, tend)` intervals over which the series is assumed to be **valid** (this +validity is what channel synchronization relies on). It does *not* intrinsically +assume the value is *held constant* over the interval: how a value is reconstructed +within `[tstart, tend)` is an **interpolation** choice. Today the only interpolation +used is **zero-order hold** (the value at `tstart` carries forward), but additional +interpolation methods could be added in the future without changing the underlying +validity model. We want to add a second kind of channel, a **Points-in-Time (POI) +Series**: a list of `(tᵢ, vᵢ)` pairs where each value is defined **only at its +timestamp** and **no assumption of validity (and hence no interpolation) is made +between two consecutive timestamps**. + +The backend model class already exists — +[`PointsInTimeSeries`](../references/api/impulse_query_engine/model/series/points_in_time_series.md) +— and already implements arithmetic, comparisons, `synchronized` / `synchronized_all`, +and the reducing aggregations (`count`, `sum`, `mean`, `min`, `max`). The +integration work is therefore **not** about series math; it is about: + +1. **Where POI samples live in silver** (a new `poi_channels` table), and +2. **How the solver knows a channel is POI** — table membership (data in + `poi_channels` ⇒ POI) plus the query author's `poi_channel(...)` selector; no + explicit `series_type` column is needed (see [§3.2](#32-discriminator-table-membership-which-table-holds-the-channels-data)), and +3. **How the solve step builds a `PointsInTimeSeries` instead of a `SampleSeries`** + for those channels. + +The central design observation is that the entire metadata **filter pipeline is +already series-type-agnostic**, so POI support drops into the *solve* stage only. + +:::note Terminology + +- **Sample series** — the existing channel type; `[tstart, tend)` intervals over + which the series is *valid*, with values reconstructed by an interpolation method + (zero-order hold today). Backed by `SampleSeries`. +- **POI series** — the new channel type; `(tᵢ, vᵢ)` points valid *only at* their + timestamps, with no between-point validity or interpolation. Backed by + `PointsInTimeSeries`. + +::: + +### 1.1 Motivating example: ECU defect / error codes (DTCs) + +The canonical real-world POI series in vehicle testing is the stream of **defect +codes** (a.k.a. error codes, or **Diagnostic Trouble Codes — DTCs**) emitted by a +vehicle's Electronic Control Units (ECUs). When an ECU's diagnostic monitor detects +a fault — a misfire, a sensor reading out of range, a lost CAN message — it emits a +code at the **instant the fault is registered**. In a test fleet these are captured +off the CAN/UDS bus (e.g. via the `ReadDTCInformation` service, UDS `0x19`) and +logged with the timestamp at which the ECU reported them. + +A DTC event stream is a **textbook POI series**, and specifically a **string-valued** +one: + +- **Event-driven, not continuous.** A code exists *at* the moment the ECU raised it + and says **nothing** about the time between two codes. Interpolating "the value + between two error codes" is meaningless — which is exactly the POI validity model + (no between-point validity), and exactly what the held-over-interval `SampleSeries` + model would get *wrong*. +- **String values.** The standardized code is a short alphanumeric string in the + `P0301` form (1 letter for the system — **P**owertrain / **C**hassis / **B**ody / + **U**network — a generic/OEM digit, a subsystem family digit, and a 2-digit fault + index; e.g. `P0301` = cylinder-1 misfire). This is why POI channels need the + string `value_type` from [§3.4](#34-per-channel-value-dtype-double-vs-string): the + natural analysis is *equality* ("when did `P0301` occur?"), never arithmetic or + ordering on the code — matching the equality-only operator set we implement for + string POI series. + +This use case also motivates **mix-and-match** ([§2](#2-background-why-this-fits-so-cleanly)), +because DTCs are almost always analyzed **together with the continuous signals** +recorded in the same container: + +- **"Freeze-frame"-style analysis.** ECUs snapshot continuous PIDs (engine RPM, + vehicle speed, coolant temperature, …) at the instant a code is set. In Impulse + this is the exact shape of `PointValueAggregator` / a `PointsInTimeEvent`: sample a + `SampleSeries` channel (`Engine_RPM`) **at the timestamps of** a POI channel + (`DTC == "P0301"`). The POI channel supplies the instants; the sample channel + supplies the values valid at those instants — one query, one container, both series + types in the same pandas UDF. +- **Counting / windowing.** "How many `P0301` events occurred while + `Engine_RPM > 4000`?" combines a string-POI equality filter with an interval + derived from a sample series — again both series types in one expression. + +Sketched in the query API this design proposes ([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)), +the string DTC channel is selected with the dedicated `poi_channel(...)` method +and its `dtype`, and mixes freely with an ordinary `channel(...)` sample selection: + +```python +dtc = query.poi_channel(channel_name="DTC", dtype="string") # string POI series +rpm = query.channel(channel_name="Engine_RPM") # sample series + +# "freeze-frame": RPM at the instants DTC == "P0301" +rpm.where(dtc == "P0301") + +# equality is the only comparator defined on a string POI series (§3.4) +``` + +:::note Timestamp caveat (informative) + +DTCs do not universally carry an absolute wall-clock timestamp in the ECU fault +memory — the reliable instant usually comes from the logger/gateway that timestamps +the event when it reads the code (GPS/NTP-synced), and any per-DTC snapshot/extended +records are OEM-dependent. For Impulse this is an **ingestion** concern: whatever +timestamp the silver pipeline lands on `poi_channels.timestamp` is the instant the +engine treats as the point's `tᵢ`. It does not affect the data-model or solver +design below. + +::: + +## 2. Background: why this fits so cleanly + +The [`DefaultSolver` filter pipeline](../references/query_engine/query_solvers.md) +runs six stages, but only ever passes **identity + selector metadata** between +them: + +``` +filter_container_tags → filter_container_metrics → filter_channel_tags → +filter_channel_metrics → (alias resolution) → solve +``` + +Every stage up to `solve` produces at most +`(container_id, channel_id, selector_ids)` (plus optional unit columns). **None of +these stages read `tstart` / `tend` / `value` or make any interval-validity or +interpolation assumption.** The validity-and-interpolation semantics enter the +system in exactly one place: + +- `DefaultSolver.solve` reads the `channels` table, joins it to the channel-match + frame, and runs a grouped-map UDF (`_solve_udf`). +- Inside the UDF, `TimeSeriesCache.load_blob(...)` constructs a **`SampleSeries`** + from the `(ts, te, val)` columns. +- `TimeSeriesSelector.build(cache)` calls `cache.load_blob(...)` and returns that + `SampleSeries` to the expression tree. + +So a channel becomes a `SampleSeries` at `TimeSeriesCache.load_blob`, and nowhere +else. If we can make that one call return a `PointsInTimeSeries` for POI channels, +the rest of the engine — expression evaluation, events, aggregations — already +works, because `PointsInTimeSeries` and `SampleSeries` share the operator and +synchronization protocol, and `SampleSeries.where(PointsInTime)` / +`PointsInTimeSeries.plane_sweep` already bridge the two representations. + +**Mixing is the common case, and it is a per-container concern.** POI and sample +channels live in the **same containers**, and users routinely combine them in one +expression (e.g. `poi_channel - sample_channel`). Cross-type alignment +(`synchronized`) happens **inside** the per-container pandas UDF, on the in-memory +series objects. That imposes a hard requirement: **both series types for a container +must be present in the same UDF invocation.** A design that solved sample and POI +channels in two *separate* UDFs and unioned the results would break mixing — each +UDF would see only half of a container's channels and could not evaluate a +cross-type expression. The design below therefore feeds **one unified pandas frame +per container** (sample + POI channel data together) into a **single** grouped-map +UDF, and a unified cache builds the correct series type per channel. + +```mermaid +flowchart TB + subgraph pipeline["Filter pipeline (UNCHANGED — series-type-agnostic)"] + direction LR + A[container tags] --> B[container metrics] --> C[channel tags] --> D[channel metrics] + end + D -->|"(container_id, channel_id, selector_ids, series_type)"| J + + subgraph solve["solve stage (the ONLY place that touches series semantics)"] + direction TB + RS["read channels
(SAMPLE rows)"] --> J + RP["read poi_channels
(POI rows)"] --> J + J["union sample + POI sample data
keyed by (container_id, channel_id)
→ ONE frame per container"] + J --> U["grouped-map UDF, grouped by container_id
(both series types in the same pandas frame)"] + U --> C2["unified cache builds per channel:
SampleSeries (valid over interval; ZOH today)
or PointsInTimeSeries (valid only at points)"] + C2 --> EV["evaluate expression tree
cross-type ops align via synchronized"] + end + EV --> OUT([one wide row per container]) +``` + +## 3. Chosen design + +### 3.1 Storage: a separate `poi_channels` table + +POI samples are stored in a **new silver table `poi_channels`**, parallel to +`channels` but carrying a single timestamp (no derived `tend`, because a POI point +has no notion of a validity interval) and **two typed value columns** plus a +per-row `dtype` discriminator, since a POI value may be numeric **or** a string: + +| Column | Type | Nullable | Description | +|----------------|----------|----------|-------------------------------------------------------------------| +| `container_id` | `long` | No | Parent container identifier (join key). | +| `channel_id` | `int` | No | Channel identifier. | +| `timestamp` | `long` | No | Point timestamp (microseconds). | +| `value_double` | `double` | Yes | Value at this timestamp **when `dtype = double`**; else null. | +| `value_string` | `string` | Yes | Value at this timestamp **when `dtype = string`**; else null. | +| `dtype` | `string` | No | Value data type: `double` or `string`. Selects the value column. | + +For any given row exactly one of `value_double` / `value_string` is populated, +chosen by `dtype`. `dtype` is expected to be **constant per +`(container_id, channel_id)`** — a channel is either a numeric POI channel or a +string POI channel, not a mix (see [§3.4](#34-per-channel-value-dtype-double-vs-string)). + +`container_id` follows the same +[type rules as the rest of the silver layer](../data_model/silver_layer_schema.md) +— it may be `long` / `int` / `string`, but must be **consistent across all silver +tables** since the engine joins on it. This matches the CLAUDE.md invariant that +`container_id` / `channel_id` types are derived dynamically and never hardcoded. + +**Why a separate table rather than reusing `channels`:** + +- **No semantic overloading of `tend`.** The `channels` RLE format treats a + trailing zero-duration `[t, t)` row as a *closed endpoint* of a sample series (an + interval of validity that has collapsed to a single instant). Reusing that row + shape for a *whole* POI channel would require every + reader (the solve UDF, the RLE/interval encoders, `SampleSeries` construction) to + disambiguate "closed endpoint of a sample series" from "a genuine point". A + dedicated table keeps the two data shapes physically and semantically distinct. +- **Cleaner ingestion contract.** Producers write POI points as `(timestamp, value)` + with no obligation to synthesize a `tend`, which they cannot do correctly for POI + data anyway. +- **Minimal disturbance to the sample-series path.** The existing `channels` read + and RLE/interval encoding are untouched, and a `SAMPLE` channel still builds the + identical `SampleSeries`. The cache does gain a per-channel series-type dispatch + (required so sample and POI channels can be mixed in one UDF — see + [§4.3](#43-one-unified-per-container-frame-one-udf-one-dispatching-cache)), but the + sample branch's behavior is unchanged. + +The cost is a new configured table + a new read path + a branch in the solve +prelude — all localized to `DefaultSolver.solve` / `MeasurementDB` (see §4). + +### 3.2 Discriminator: a `series_type` column on `channel_metrics` + +:::note Implemented differently — see [§9](#9-aspects-which-differ-from-the-design) +The `series_type` column described below was **not** added. Table membership +(`channels` vs `poi_channels`) is the discriminator instead. See [§9](#9-aspects-which-differ-from-the-design). +::: + +A channel is marked POI by a **new `series_type` column on `channel_metrics`**: + +| Column | Type | Nullable | Description | +|---------------|----------|----------|--------------------------------------------------------------------| +| `series_type` | `string` | Yes | `SAMPLE` (default) or `POINTS_IN_TIME`. Null/absent ⇒ `SAMPLE`. | + +Design points: + +- **Backward compatible.** Existing tables without the column, or with `NULL`, + resolve to `SAMPLE`, so every current deployment behaves exactly as today. +- **Rides the pipeline as pass-through metadata.** `channel_metrics` is already + read in `filter_channel_metrics`; `series_type` is just one more column carried + on the channel-match rows through to `solve`. It participates in **no** filtering + decision. +- **Not `value_type`.** `channel_metrics.value_type` already exists but describes + the *value's data type* (`double`, `int`, …). Overloading it to also encode + *series semantics* would conflate two orthogonal concepts and is rejected. A new, + purpose-specific column keeps the discriminator explicit and self-documenting. +- **Introduce a `SeriesType` enum** (mirroring `RawEncoder`) so the string literals + live in one place and are referenced by `SolverConfig.series_type_col` / + the solve branch rather than being sprinkled as bare strings. + +`series_type` is added to `SolverConfig` as an internal column name property +(`series_type_col`, default `"series_type"`), so a physical layout that names the +column differently maps it via `channel_metrics.column_name_mapping` exactly like +every other column. + +### 3.3 Data model after the change + +```mermaid +erDiagram + container_metrics { + long container_id PK + } + channel_metrics { + long container_id FK + int channel_id FK + string series_type "SAMPLE | POINTS_IN_TIME (null ⇒ SAMPLE)" + } + channels { + long container_id FK + int channel_id FK + long tstart + long tend + double value + } + poi_channels { + long container_id FK + int channel_id FK + long timestamp + double value_double "when dtype = double" + string value_string "when dtype = string" + string dtype "double | string" + } + + container_metrics ||--o{ channel_metrics : container_id + channel_metrics ||--o{ channels : "SAMPLE channels" + channel_metrics ||--o{ poi_channels : "POINTS_IN_TIME channels" +``` + +A given `(container_id, channel_id)` has its samples in **exactly one** of +`channels` or `poi_channels`, selected by its `series_type` row in +`channel_metrics`. + +### 3.4 Per-channel value dtype: double vs string + +The `poi_channels.dtype` column determines which value column +(`value_double` / `value_string`) carries the point value. We treat `dtype` as a +**per-channel** property: all rows of a `(container_id, channel_id)` share one +`dtype`. This keeps a channel's value type stable, matches how measurement channels +behave in practice, and lets the solve step pick the value column **once** per +channel rather than per row. + +The two dtypes are **not** symmetric, because the backend model represents them +differently. `PointsInTimeSeries` **cannot represent string values today** — its +constructor hardcodes `np.array(values, dtype=np.float64)`, which would coerce +strings to `NaN`. We close this gap by extending the **single** +[`PointsInTimeSeries`](../references/api/impulse_query_engine/model/series/points_in_time_series.md) +class to hold values of either kind, rather than adding a second class. + +**Chosen model change — dual value arrays + a `value_type` property:** + +- **Keep the existing `float64` value array** for numeric values (unchanged; today's + numeric behavior is preserved bit-for-bit). +- **Add a second value array of dtype `object`** to hold string values. + *(Implemented differently — a single value array whose type is inferred at + construction; see [§9](#9-aspects-which-differ-from-the-design).)* +- **Add a `value_type` property on the class** distinguishing a **numeric** from a + **string** POI series. This is the single source of truth for which value array is + populated and which operations are legal. (Constructors/factories set it; a + numeric series leaves the object array empty and vice-versa.) +- **Spark `dtype()` becomes `value_type`-aware:** `ArrayType(ArrayType(DoubleType))` + for numeric (unchanged), `ArrayType(ArrayType(StringType))` for string. + +**Operations on a string POI series (this iteration):** + +- **Only the equality comparator (`==`) is implemented.** It matches the numeric + behavior — synchronize on shared timestamps, compare values, return the + `PointsInTime` where values are equal — but over string values. + *(Implemented more permissively — both `==` and `!=` are supported for strings; + see [§9](#9-aspects-which-differ-from-the-design).)* +- **All other comparators (`<`, `<=`, `>`, `>=`) return a + `NotImplementedError`** for a string series, as do the numeric-only reductions and + arithmetic (`sum`, `mean`, `min`, `max`, `+`, `-`, `*`, `/`). These raise a clear, + explicit error rather than silently coercing to `NaN`. +- Value-type-independent operations remain valid regardless of `value_type`: + `count`, `start_time` / `end_time`, `to_points_in_time`, `plane_sweep`, and the + timestamp side of `synchronized`. + +:::note "series type" appears on three distinct axes — keep them straight + +| Where | Values | Meaning | +|-------|--------|---------| +| table membership (`channels` / `poi_channels`) | sample vs POI | Which table holds the channel's data — *this* is the sample-vs-POI discriminator (no `series_type` column; see [§9](#9-aspects-which-differ-from-the-design)). | +| `poi_channels.dtype` (silver column) | `double` / `string` | A POI channel's value type — selects `value_double` vs `value_string`. | +| `PointsInTimeSeries.value_type` (class property) | numeric / string | Which in-memory value array is active and which operations are legal. | + +The middle and bottom rows are the same distinction on two sides of the Arrow +boundary: `poi_channels.dtype` on a channel becomes `PointsInTimeSeries.value_type` +on the object the cache builds for it. + +::: + +The **selectable operations are gated by `value_type`** so that, e.g., +`string_poi.mean()` fails up front (via `evaluation_type()` — see [§4.4](#44-result-typing)) +rather than producing `NaN`. + +:::note Scope check + +String POI support is the one part of this design that requires touching the +backend model (`PointsInTimeSeries`). Everything else — storage, discriminator, +pipeline, solve branch — is additive. If string POI is not needed in the first +iteration, the numeric (`double`) path can ship alone: the solver simply routes +only `dtype = double` channels and rejects (or ignores, per config) `string` +channels until the model work lands. + +::: + +### 3.5 Example: tag & metric entries for DTC POI channels + +Concrete rows for the [DTC example](#11-motivating-example-ecu-defect--error-codes-dtcs), +on an existing recording `container_id = 1`. Two POI channels are added on +`channel_id`s not used by any sample channel in that container: a **string** DTC-code +channel (`channel_id = 90`) and a **numeric** fault-occurrence-count channel +(`channel_id = 91`). + +#### Channel level — where POI-specific entries naturally live + +**Channel selection metadata.** In the EAV layout these are `channel_tags` rows +(`container_id, channel_id, key, value`); in the wide layout the same facts are +columns on `channel_metrics`. A DTC channel is selected by its `channel_name` and +described by ECU/bus context: + +| container_id | channel_id | key | value | +|--------------|------------|----------------|--------------| +| 1 | 90 | `channel_name` | `DTC` | +| 1 | 90 | `ecu` | `Engine_ECU` | +| 1 | 90 | `bus` | `CAN1` | +| 1 | 90 | `code_system` | `P` (powertrain) | +| 1 | 91 | `channel_name` | `DTC_count` | +| 1 | 91 | `ecu` | `Engine_ECU` | + +**Channel metrics** (`channel_metrics`). The **new `series_type`** marks the channel +as POI; the **existing `value_type`** records the value data type. Crucially, the +numeric statistic columns behave differently by value type — they are **undefined +(null) for a string POI channel**, and meaningful (computed over the point values, +**unweighted** — there are no durations) for a numeric one: + +| Column | DTC string channel (90) | DTC count numeric channel (91) | Notes | +|----------------|-------------------------|--------------------------------|-------| +| `series_type` | `POINTS_IN_TIME` | `POINTS_IN_TIME` | new discriminator (§3.2) | +| `value_type` | `STRING` | `DOUBLE` | pre-existing data-type column | +| `channel_name` | `DTC` | `DTC_count` | selection key (wide layout) | +| `sample_count` | `3` (three events) | `3` | number of points | +| `begin_s`/`end_s` | first/last event time | first/last event time | point extent, not a validity span | +| `min`/`max`/`mean`/`std` | **null** | computed over point values | undefined for strings; unweighted for numeric POI | +| `pz1`/`pz10`/`pz90`/`pz99` | **null** | optional | percentiles undefined for strings | +| `nan_ratio` | **null** | **null** | duration-weighted → N/A for POI | + +The per-row **`dtype`** (`string` / `double`) lives on `poi_channels`, not here (§3.1); +`series_type` on `channel_metrics` is what routes the channel to `poi_channels`. + +#### Container level — optional summaries for pre-filtering + +A container is a whole recording and owns **both** sample and POI channels, so +container-level tags/metrics are **not** POI-specific — the usual `vehicle_key`, +`brand`, `model`, `project` entries are unchanged. What POI *optionally* adds here is +**summary metadata that lets you pre-filter containers** without scanning +`poi_channels` (the same role the percentile columns play for sample channels): + +EAV `container_tags` (`container_id, key, value`): + +| container_id | key | value | Purpose | +|--------------|------------------|-------------|---------| +| 1 | `vehicle_key` | `Seat_Leon` | existing — unchanged | +| 1 | `has_dtc` | `true` | optional — "recordings that logged any fault" | +| 1 | `ecu_sw_version` | `4.11.2` | optional — correlate faults with firmware | + +Wide `container_metrics` can carry the analogous optional column +`num_dtc_events = 3` for the same pre-filtering purpose. + +These container-level additions are **purely optional and additive**: omit them and +POI channels still work; add them only to enable "find recordings where a `P0301` +occurred"-style container filters before the channel stage. A query like +`query.havingTag(has_dtc="true")` then narrows containers exactly as any other +container tag does — no POI-specific pipeline behavior. + + +## 4. Implementation plan + +The change is localized. Nothing in stages 1–5 of the pipeline changes. + +### 4.1 Config & schema + +1. `SolverConfig`: add `poi_channels: TableConfig`, add the `series_type_col` + property (`"series_type"`), and add a `poi_channels_uri` slot to + `MeasurementDBConfig` (+ `for_unity_catalog` / `for_debug` wiring, mirroring + `channels_uri`). `poi_channels_uri = None` means "no POI channels configured". +2. `MeasurementDB.poi_channels(spark)` reader, mirroring `channels(...)`. +3. `schema.py`: add a reference `POI_CHANNELS_SCHEMA` (`container_id`, `channel_id`, + `timestamp`, `value_double`, `value_string`, `dtype`) and add `series_type` to + `CHANNEL_METRICS`. As documented in CLAUDE.md these are **reference** schemas, + not enforced on read. +4. Add a `SeriesType` StrEnum (`SAMPLE`, `POINTS_IN_TIME`) next to `RawEncoder`, and + a `PoiValueType` StrEnum (`double`, `string`) for the per-row `dtype`. +5. Add `SolverConfig` internal-name properties for the new POI columns + (`poi_timestamp_col`, `poi_value_double_col`, `poi_value_string_col`, + `poi_dtype_col`) so physical layouts remap them via + `poi_channels.column_name_mapping` like every other table. +6. Extend `SolverConfig.col_map` (the short-key → column-name map handed to the UDF + cache, today `cid/ch/ts/te/val/conv`) with `series_type`, `value_string`, and + `dtype` keys so the unified cache (§4.3) can locate them in the pandas frame. +7. Add two optional fields to `TimeSeriesSelector` (`series_type`, `value_type`), + defaulting to `SAMPLE` / numeric so existing `channel(...)` selectors are + unchanged, and add `QueryBuilder.poi_channel(*, dtype=PoiValueType.double, + **kwargs)` (see [§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)). + +### 4.2 Query API and carrying the discriminators to solve + +#### `QueryBuilder.poi_channel(...)` + +POI channels are selected through a dedicated **`poi_channel(...)` factory method** +on `QueryBuilder`, parallel to the existing `channel(...)` / `channel_with_alias(...)`: + +```python +def poi_channel(self, *, dtype: PoiValueType = PoiValueType.double, **kwargs) -> TimeSeriesSelector: + # same tag/column matching as channel(...) — builds the selector expr from **kwargs + return TimeSeriesSelector(expr, series_type=POINTS_IN_TIME, value_type=dtype) +``` + +Design points: + +- **No new selector class.** `poi_channel` returns the **same `TimeSeriesSelector`** + that `channel(...)` returns; channel *identification* (tag/column match, + `get_selector_expr`, `required_tags`, `selector_id`, the direct/aliased split) is + identical for POI and sample channels, so there is nothing to override. The method + is a **factory**, not a subclass — it just stamps the selector with its + `series_type` (`POINTS_IN_TIME`) and the caller-declared value `dtype`. +- **Explicit intent at the call site.** `query.poi_channel(channel_name="DTC")` + reads as "this is an event stream, not a signal," and gives POI-only knobs + (the `dtype`) a natural home. `dtype` defaults to `double`, so the common numeric + case stays terse; a string DTC channel is `poi_channel(channel_name="DTC", dtype=string)`. +- **The selector now carries `series_type` + `value_type`.** `TimeSeriesSelector` + gains two optional fields (defaulting to `SAMPLE` / numeric so `channel(...)` is + unchanged). This makes the selector the **plan-time** source of truth for the + series type — which is what simplifies result typing (see [§4.4](#44-result-typing)): + `evaluation_type()` / `dtype()` and the string-op gating work **without** any + pre-pipeline `channel_metrics` lookup, and `string_poi.mean()` can be rejected at + **build time** before Spark is involved. + +:::caution Declared `dtype` is validated against the data, not trusted over it + +The user-declared `dtype` and the silver data are **two sources that must agree**. +The contract is **assertion, not authority**: + +> The check validates against the **data itself**, not a `channel_metrics.series_type` +> column (which was dropped — see [§9](#9-aspects-which-differ-from-the-design)): a POI +> point row carries a null `tend`, so a `poi_channel(...)` that resolves to +> interval-shaped rows is a SAMPLE channel, and an all-null value column exposes a +> declared/actual `dtype` mismatch. + +- The declared `series_type` / `dtype` drive **plan-time** typing and op-gating. +- At **solve time** the data remains authoritative: if the resolved channel's actual + shape **disagrees** with what the selector declared, the solver **raises a clear + error** (mirroring the existing unit-conversion conflict check), rather than silently + reading the wrong value column or overriding the data. + +This keeps the ergonomic win (no plan-time lookup, early validation) without letting +a wrong declaration silently mis-read a channel (e.g. a `dtype=double` hint on a +string channel yielding all-null `value_double`). + +::: + +#### Carrying the discriminators through the pipeline + +`filter_channel_metrics` already reads and column-maps `channel_metrics`. Include +`series_type` in the projected channel-match columns (defaulting null → `SAMPLE` +via `F.coalesce`). It travels alongside `selector_ids` with no effect on any +filter, exactly like the existing per-channel metadata. This solve-time +`series_type` (and, for POI, `dtype`) is what the **assertion check above** +validates the selector's declared values against. + +### 4.3 One unified per-container frame, one UDF, one dispatching cache + +Because sample and POI channels share containers and are mixed in a single +expression, they **must be solved together in one grouped-map UDF per container** +(see the requirement established in [§2](#2-background-why-this-fits-so-cleanly)). +The design keeps the existing single-UDF shape and makes the *cache* series-type +aware, rather than forking the UDF. + +**Step 1 — normalize both sample sources into one Spark frame.** In +`_prepare_channels_join`, read and column-map **both** tables and project them into +a common superset schema keyed by `(container_id, channel_id)`, carrying a +`series_type` discriminator (and, for POI, `dtype`): + +| Column | SAMPLE row source | POI row source | +|----------------|-----------------------|---------------------------------------| +| `container_id` | `channels` | `poi_channels` | +| `channel_id` | `channels` | `poi_channels` | +| `series_type` | `SAMPLE` | `POINTS_IN_TIME` | +| `tstart` | `channels.tstart` | `poi_channels.timestamp` | +| `tend` | `channels.tend` | `null` (POI has no validity interval) | +| `value` | `channels.value` | `poi_channels.value_double` | +| `value_string` | `null` | `poi_channels.value_string` | +| `dtype` | `null` (⇒ numeric) | `poi_channels.dtype` | + +`unionByName` the two projections into a single DataFrame, join it to the +channel-match frame on `(container_id, channel_id)`, then — exactly as today — +`groupBy(container_id).apply(udf)`. Only channels that survived the filter pipeline +are shipped, so the union stays small. A container's sample and POI rows now land in +the **same** pandas frame. + +**Step 2 — a unified cache that dispatches per channel.** Generalize +`TimeSeriesCache` (or add a `UnifiedSeriesCache` that subsumes it) so `load_blob` +inspects the channel slice's `series_type` and builds the right object: + +- `series_type == SAMPLE` → `SampleSeries(tstart, tend, value)` (today's behavior, + unchanged). +- `series_type == POINTS_IN_TIME` and `dtype == double` → numeric + `PointsInTimeSeries(tstart, value)` (the POI timestamp lives in the `tstart` + column of the unified frame). +- `series_type == POINTS_IN_TIME` and `dtype == string` → the string point series + from [§3.4](#34-per-channel-value-dtype-double-vs-string), built from + `(tstart, value_string)`. + +The cache keeps the same `(cid, ch) → (start, stop)` range-index over the sorted +frame; the only change is which columns each slice reads and which class it +instantiates. Because `series_type` and `dtype` are constant per channel, the +dispatch is decided **once** per `(cid, ch)` slice, not per row. + +**Step 3 — expression evaluation is unchanged.** `TimeSeriesSelector.build(cache)` +still just calls `cache.load_blob(...)`; it now transparently gets a `SampleSeries` +or a point series. A mixed expression such as `poi_channel - sample_channel` is +evaluated on the two in-memory objects, and `PointsInTimeSeries._apply_basic_op` +already handles the cross-type case by aligning against the `SampleSeries` at the +POI timestamps via `synchronized`. **No new math and no second UDF.** + +The `series_type` / `dtype` discriminators are carried the same pass-through way as +the existing per-channel metadata (they originate on `channel_metrics` / +`poi_channels`; see [§8](#8-open-questions)), so both the cache and the result-typing +step (§4.4) know each channel's kind without scanning its data. + +:::note Why not two UDFs? + +Splitting SAMPLE and POI into two grouped-map UDFs and unioning their **outputs** +would be simpler to write but is **incorrect** for the common mix-and-match case: +each UDF would receive only a subset of a container's channels, so an expression +referencing one channel of each type could not be evaluated — one operand would +always be missing from that UDF's frame. Unifying the **input** frame and keeping a +single UDF is what makes cross-type expressions work. + +::: + +### 4.4 Result typing + +`QueryBuilder._determine_result_objects_dtypes` builds each selection against an +`EmptyTimeSeriesCache` to learn its result `dtype`. Today `EmptyTimeSeriesCache.load_blob` +always returns an empty `SampleSeries`, so a bare POI selection would be mistyped +as `BinaryType` (the `SampleSeries` serialization dtype) instead of +`PointsInTimeSeries.dtype()` (`ArrayType(ArrayType(DoubleType))`). + +**Because the selector now carries its own `series_type` / `value_type` +([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)), this resolves with +no plan-time metadata lookup.** `EmptyTimeSeriesCache.load_blob` simply consults the +calling selector and returns an empty series of the matching kind: + +- a `SAMPLE` selector → empty `SampleSeries` (today's behavior); +- a numeric POI selector → empty numeric `PointsInTimeSeries`; +- a string POI selector → empty `PointsInTimeSeries` with `value_type = string`. + +`evaluation_type()` / `dtype()` are then correct for bare POI selections and for +expressions whose output type depends on the input type — and the string-op gating +fires **at build time**: `string_poi.mean()` builds an empty string point series +whose `mean()` raises `NotImplementedError`, so the selection is rejected up front +rather than producing a silent `NaN`, before Spark is involved. + +This removes the earlier need to pre-resolve each selector's type from +`channel_metrics` and inject it into the empty cache: the declared type on the +selector *is* the plan-time source. (The silver metadata still has the final say at +solve time via the [§4.2 assertion check](#42-query-api-and-carrying-the-discriminators-to-solve).) +It also mirrors how `PointsInTimeEvent` and `PointValueAggregator` already validate +`evaluation_type()` up front, so the mechanism is consistent with existing code. + +### 4.5 `PointsInTimeSeries` model change + +The one backend-model change (see [§3.4](#34-per-channel-value-dtype-double-vs-string)): + +- Add a second value array (dtype `object`) alongside the existing `float64` array, + and a `value_type` property (numeric / string) selecting which is active. +- Constructors/factories set `value_type`: the numeric path keeps today's + `np.array(values, dtype=np.float64)`; the string path stores values as an `object` + array and leaves the numeric array empty. +- Make `dtype()` return `ArrayType(ArrayType(StringType))` when `value_type` is + string (numeric unchanged). +- Implement **`__eq__` for string series** (synchronize on timestamps → compare + string values → `PointsInTime`). Have `__ne__`, `__lt__`, `__le__`, `__gt__`, + `__ge__`, the arithmetic operators, and the numeric reductions (`sum`, `mean`, + `min`, `max`) **raise `NotImplementedError`** when `value_type` is string. +- Leave `count`, `start_time` / `end_time`, `to_points_in_time`, `plane_sweep`, and + the timestamp handling in `synchronized` value-type-independent (they already are). + +### 4.6 Extend the existing test dataset with DTC POI channels + +Rather than build a bespoke POI fixture, **extend the existing session-scoped silver +dataset** so POI channels live alongside the current sample channels in the **same +containers** — this is what exercises the mix-and-match path (§4.3) end to end and +mirrors the [DTC motivating example](#11-motivating-example-ecu-defect--error-codes-dtcs). +The guiding constraint is **additive, non-destructive**: every existing test must +keep passing untouched. + +The `setup_basic_db` fixture (autouse, session-scoped) loads +`tests/unit/data/basic_narrow_csv/` into `spark_catalog.silver.*`. Use the concrete +rows from [§3.5](#35-example-tag--metric-entries-for-dtc-poi-channels) (DTC string +channel `channel_id = 90`, numeric count channel `channel_id = 91` on +`container_id = 1`) as the fixture data. The plan: + +1. **New `poi_channels` data file.** Add + `basic_narrow_csv/poi_channels.csv` with + `container_id, channel_id, timestamp, value_double, value_string, dtype` and a + couple of **DTC channels** on **existing** `container_id`s (e.g. a `DTC` string + channel with points like `(t₁, "P0301")`, `(t₂, "P0420")`, and a numeric POI + channel such as a fault-occurrence counter). Choose `channel_id`s **not already + used** by that container in `channel_data.csv` so the two sample sources stay + disjoint per the design invariant (a channel lives in exactly one of + `channels` / `poi_channels`). +2. **Append POI rows to `channel_metrics.csv`.** Add one row per new POI channel + carrying the new `series_type = POINTS_IN_TIME` column. **Backfill existing rows + with `series_type = SAMPLE`** (or leave blank and rely on the null ⇒ `SAMPLE` + default — pick one and be consistent). Existing sample channels are unaffected. +3. **Load `poi_channels` in the fixture.** Extend `setup_basic_db` to read the new + CSV and write `spark_catalog.silver.poi_channels`, and add its slot to the + `MeasurementDBConfig` used by the basic-db fixtures (`poi_channels_uri`). Because + `poi_channels_uri` defaults to `None`, **any db config that does not opt in is + unchanged**, so unrelated fixtures/tests see no difference. +4. **EAV + wide tag/metric parity.** So POI channels are *selectable* the same way + in both channel-selection modes: + - **EAV fixtures** (`setup_narrow_db`, `unit_test_csv/`): append POI rows to + `1_channel_tags.csv` (e.g. `channel_name = "DTC"`) and `1_channel_metrics.csv`, + plus any container-level tags/metrics needed, so a + `query.poi_channel(channel_name="DTC", dtype="string")` resolves the POI channel + through the pivot path (identification is identical to `channel(...)`; only the + selector's declared `series_type` / `value_type` differ — [§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)). + - **Wide fixtures** (`basic_narrow_csv`): the `channel_name` column already on + `channel_metrics` covers direct selection; just ensure the appended POI rows + carry a distinct `channel_name` (e.g. `"DTC"`). + +:::caution Two different columns both once called "value type" + +`basic_narrow_csv/channel_metrics.csv` **already** has a `value_type` column holding +values like `DOUBLE` (and the EAV `1_channel_metrics.csv` has `numerical`). That is +the **pre-existing** per-channel data-type column and is **not** the discriminator +this design adds. Keep them separate: + +- **existing `channel_metrics.value_type`** — untouched; describes the value data + type and is not read by the solver for routing. +- **new `channel_metrics.series_type`** — `SAMPLE` / `POINTS_IN_TIME`; routes to + `channels` vs `poi_channels` (§3.2). +- **new `poi_channels.dtype`** — `double` / `string`; selects `value_double` / + `value_string` (§3.4). + +Do **not** overload the existing `value_type` column for either new purpose — the +column names in the fixtures must stay distinct, and existing tests that read +`value_type` must be left as-is. + +::: + +**Regression guard.** Run the full existing suite after extending the fixtures and +confirm it is green *before* adding POI-specific tests (§7). Because the changes are +purely additive — new file, appended rows with a defaulting column, an opt-in table +slot — no existing assertion (row counts, computed means, dimension contents) should +move. If any does, the extension was not additive and must be corrected. + +## 5. What explicitly does **not** change + +- **The 6-stage filter pipeline.** `filter_container_tags` → + `filter_container_metrics` → `filter_channel_tags` → `filter_channel_metrics` → + alias resolution are untouched. POI channels are identified by the *same* + `TimeSeriesSelector` class, tag/column matching, and tag/metric filters as sample + channels — `poi_channel(...)` is a factory over the same selector, not a new + selection path ([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)). +- **`channels` table and RLE/interval encoders.** The sample-series read and + raw→interval encoding are untouched. +- **The sample-series *behavior* in the cache.** `TimeSeriesCache` gains a + per-channel dispatch (§4.3), but for a `SAMPLE` channel it builds the exact same + `SampleSeries(tstart, tend, value)` as today — the sample path's semantics and + output are unchanged. (This is a behavior guarantee, not a "no code changed" + claim: the cache does gain POI-aware branching.) +- **The single grouped-map UDF per container.** The solve stage still groups by + `container_id` and applies one UDF; POI does **not** add a second UDF or a + post-hoc union of two result sets. The input frame is widened to carry both series + types, not the execution model. +- **Persistence / gold layer.** Aggregations over POI series already reduce to + scalars (`mean`, `sum`, `count`, …) or `PointsInTime` events, which the existing + fact/dimension tables already accept (`PointsInTimeEvent`, `PointValueAggregator`). +- **`PointsInTimeSeries` for numeric (`double`) channels.** No new methods needed; + it already implements the full operator/sync/aggregation protocol for `float64` + values. (String POI is the exception — it requires the model change in + [§3.4](#34-per-channel-value-dtype-double-vs-string).) +- **`SampleSeries` interpolation semantics.** This design does not change how + sample-series values are reconstructed within `[tstart, tend)`. Zero-order hold + remains the only interpolation today; adding further interpolation methods later + is an **orthogonal** effort. The distinction that matters for POI is *validity* + (does a value exist between two timestamps at all?), not *which* interpolation is + applied where validity holds. + +## 6. Alternatives considered + +| Alternative | Why not chosen | +|-------------|----------------| +| **Store POI in `channels` with `tend == tstart`** | Overloads the "closed endpoint" meaning of zero-duration rows; forces every reader/encoder to disambiguate a whole POI channel from a sample-series endpoint. | +| **Store POI in the RAW `channels` (timestamp, value) format + a skip-encoding flag** | Couples POI to RAW mode and to the raw→interval encoder; a channel's storage shape would depend on an unrelated `data_type` setting. | +| **Overload the existing `value_type` column as the discriminator** | Conflates value *data type* with *series semantics*; two orthogonal concerns in one column, harder to reason about and to validate. | +| **A new dedicated POI solver class** | Unnecessary — the filter pipeline is shared and identical; only `load_blob` differs. A per-channel branch inside `DefaultSolver.solve` is far less code than a parallel solver. | +| **Two grouped-map UDFs (one SAMPLE, one POI), union the outputs** | **Incorrect** for the common mix-and-match case: each UDF sees only a subset of a container's channels, so an expression combining a POI and a sample channel (e.g. `poi - sample`) has a missing operand. Cross-type `synchronized` must run on both in-memory series inside **one** UDF. | + +## 7. Testing strategy + +Following the repo's fixture-reuse convention (CLAUDE.md → *Testing patterns*). +The POI tests run against the **extended shared dataset from [§4.6](#46-extend-the-existing-test-dataset-with-dtc-poi-channels)** +(DTC channels added to the existing `spark_catalog.silver.*` fixtures) rather than a +throwaway db, so they cover the real read path and the mix-and-match case: + +- Assert on **real computed values**, not row counts: e.g. a numeric POI `mean()` + equals the unweighted mean of the point values (contrast with the duration-weighted + `SampleSeries.mean()`, whose weighting follows from interval validity), proving the + between-point validity is genuinely absent. +- A **string POI** test: `query.poi_channel(channel_name="DTC", dtype="string")` + builds a `PointsInTimeSeries` with `value_type = string`; the **equality comparator** + (`== "P0301"` → `PointsInTime` on matching timestamps) and value-type-independent ops + (`count`, `to_points_in_time`, point sampling) work, while every **other comparator** + (`!=`, `<`, `<=`, `>`, `>=`), the arithmetic operators, and the numeric reductions + (`mean`, `sum`, `min`, `max`) raise `NotImplementedError` — asserted both directly on + the series object and, for a reduction inside a selection, at `evaluation_type()` + **build time** (not as a silent `NaN`, and before Spark runs). +- A **mix-and-match test (the primary correctness case)**: a single container owning + both a SAMPLE channel and a numeric POI channel, selected with `query.channel(...)` + and `query.poi_channel(...)` respectively, with **one expression referencing both** + (`rpm.where(dtc == "P0301")`, and `poi - sample`). This asserts both series land in + the *same* per-container pandas frame, are built by the unified cache, and align via + `synchronized` — the behavior a two-UDF design would break. Assert the computed + values, not just that it runs. +- A **declared-vs-actual `dtype` assertion test** ([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)): + `query.poi_channel(channel_name="DTC", dtype="double")` on a channel whose silver + `dtype` is `string` (or a `poi_channel` on a `SAMPLE` channel) raises a clear error + at solve time — the data stays authoritative, the wrong declaration is not silently + honored. +- A backward-compat test: a `channel_metrics` with no `series_type` column still + solves as SAMPLE, and existing `channel(...)` selections are unaffected by the new + optional selector fields. + +## 8. Open questions + +- **Should `series_type` be validated against the presence of data in the matching + table?** (e.g. a POI-marked channel with rows only in `channels`.) Proposed: + no hard validation initially; document that the marker is authoritative and the + non-matching table is not read for that channel. +- **~~Where should the POI value `dtype` be resolved for planning?~~ (Resolved.)** + The user declares `dtype` on `query.poi_channel(...)` and the selector carries it + ([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)), so plan-time + result typing (§4.4) needs **no** pre-pipeline `poi_channels` / `channel_metrics` + scan. The silver `poi_channels.dtype` remains authoritative at solve time and is + validated against the declared value (assertion contract). *Remaining sub-question:* + should the engine also support **inferring** `dtype` when the user omits it (rather + than defaulting to `double`) — e.g. a cheap `distinct` on `channel_metrics` — for + callers who prefer not to declare it? Proposed: keep the explicit `double` default + for now; add inference only if a concrete need appears. +- **Enforcing the constant-`dtype` invariant.** A channel is entirely numeric or + entirely string — `dtype` is constant per `(container_id, channel_id)` by + contract. This is a settled invariant, not an open question; the only decision is + whether to *defend* it. Proposed: an optional validate-and-raise (like the + unit-conversion conflict check) that flags any channel carrying more than one + distinct `dtype`, so a malformed ingest fails loudly instead of picking an + arbitrary value column. +- **String value column when scaling.** If more non-numeric dtypes appear later + (e.g. `bool`, `int`), revisit whether a typed-column-per-dtype layout still scales + or whether a single `value` string column + cast is preferable. +- **Calculated channels producing POI output.** `solve_calculated_channels` emits a + narrow `[container_id, channel_id, tstart, tend, value]` frame. Emitting a POI + *calculated* channel would need a narrow POI shape (`timestamp, value`). Deferred — + out of scope for ingesting POI *input* series. + +## 9. Aspects which differ from the design + +A few things landed differently than sections 3–4 describe. The shipped code is the +source of truth; those sections are left as the original proposal, and each spot that +changed points here. None of these change what the feature does — they mostly remove +machinery the design added that turned out to be unnecessary once the selector became +the source of truth for a channel's series type. + +### 9.1 No `series_type` column on `channel_metrics` (§3.2) + +The design added a `series_type` marker to `channel_metrics` so the solver could tell a +POI channel from a sample channel. We dropped it. A channel's data lives in exactly one +of `channels` or `poi_channels`, so **which table it comes from already tells us the +series type** — the extra column was redundant, and nothing ever read it at solve time. +Today the only way to get a `PointsInTimeSeries` is to read from `poi_channels`, so the +table membership is a complete answer. + +### 9.2 One value array, type inferred at construction (§3.4, §4.5) + +The design proposed keeping the numeric `float64` array and adding a *second* `object` +array for strings, with a `value_type` property choosing between them. In practice +`PointsInTimeSeries` keeps a **single** value array and infers whether it's string or +numeric from the values at construction time (an `_is_string` flag). It's less +bookkeeping — there's no pair of arrays to keep in sync, one always empty — and it +reads more naturally: you build the series from whatever values you have and it figures +out its own type. An explicit `empty_string()` factory covers the one case inference +can't (an empty series has nothing to infer from). + +### 9.3 String POI also supports `!=` (§3.4, §4.5) + +The design limited string POI series to equality (`==`) and had `!=` raise alongside +the ordering and arithmetic operators. We kept `!=` too. + +### 9.4 The declared-vs-actual check reads the data shape, not a marker (§4.2) + +The design validated the selector's declared `series_type` / `dtype` against the +`channel_metrics.series_type` column. With that column gone (9.1), the solve-time check +instead looks at the **data it resolved to** + +### 9.5 Series-type dispatch is driven by the selector, not a per-row column (§4.3) + +The design's solve stage stamped `series_type` (and `dtype`) onto every channel-data +row so the cache could inspect each slice. Since the selector already knows its own +type, we pass that into `load_blob` instead and drop the per-row markers from the frame +that crosses into the pandas UDF. Only `value_string` still rides along, because that's +real data a string channel needs, not a discriminator. The result is the same object +per channel with a bit less shipped across the Arrow boundary. + +### 9.6 Enum placement (§4.1) + +Minor: the design suggested putting `SeriesType` next to `RawEncoder` in +`solver_config.py`. It lives in `time_series_expression.py` instead, next to +`TimeSeriesSelector` (which carries it) and alongside the new `PoiValueType` enum. That's +where the selector-as-source-of-truth logic reads most naturally. diff --git a/skills/impulse-channels/SKILL.md b/skills/impulse-channels/SKILL.md index e290cb86..b568c13b 100644 --- a/skills/impulse-channels/SKILL.md +++ b/skills/impulse-channels/SKILL.md @@ -5,8 +5,8 @@ description: > channels and materialized at the same per-sample grain. Use when the user wants to "add a calculated channel", derive/persist a signal (e.g. "speed in km/h", "power = rpm × torque"), materialize a virtual signal into a queryable table, or run `solve_calculated_channels`. Covers the reporting-layer - CalculatedChannel, the ad-hoc `QueryBuilder.solve_calculated_channels` endpoint, the - calculated_channel_fact/dimension gold output, and the optional calculated_channel_metrics table. + CalculatedChannel, the ad-hoc `QueryBuilder.solve_calculated_channels` endpoint, and the + calculated_channel_fact/dimension gold output. --- # Impulse — calculated channels @@ -87,17 +87,6 @@ df.show() # [container_id, channel_id, tstart, tend, value, identity] All selections in one call must be `CalculatedChannel`s; identity keys are arbitrary and need not match across selections. -## Batching (in a Report) - -Inside a `Report`, calculated channels are solved in **batches**, the same pattern events and -aggregations use (see `impulse-reporting`). The channels are partitioned by `query_engine.batch_size` -(max unique input selectors per batch), each batch is solved via `solve_calculated_channels` and -persisted as a temp table (`__impulse_temp_{run_id}_{batch_idx}` in the sink schema, or a Spark temp -view when sinkless), and the batches are unioned into the final `calculated_channel_fact`. The temp -tables share the `__impulse_temp_*` prefix, so they are cleaned up like the aggregation/event batches -(`unity_sink.cleanup_temp_tables`). This is internal orchestration; the `add_calculated_channel` API is -unchanged. - ## Output schema **calculated_channel_dimension** (one row per channel per report) — key columns: `channel_id`, @@ -109,30 +98,6 @@ unchanged. identity is **not** on the fact — it lives on the dimension; join on `channel_id` (and to `measurement_dimension` on `container_id`; see `impulse-data-model`). -## Optional channel metrics table - -`calculated_channel_fact` already matches the silver `channels` shape, so a calculated channel needs only a -companion `channel_metrics` table to become an Impulse silver source. Set -`config.calculated_channels.emit_channel_metrics = True` to also write `calculated_channel_metrics`, shaped -like silver `channel_metrics` (one row per `(container_id, channel_id)`, derived from the fact rows). - -```python -from impulse_reporting.config.config_parser import CalculatedChannels - -# in the ImpulseConfig: -calculated_channels = CalculatedChannels( - emit_channel_metrics=True, - attribute_columns=["unit"], # attribute keys to surface as columns; default [] - kpis=["duration", "min", "max", "mean"], # default; each must be a registered KPI -) -``` - -The schema is **dynamic**: fixed `container_id`, `channel_id`, `value_type` (`"double"`), -one column per configured KPI, one per identity key (union across the report's channels), and one per -`attribute_columns` entry. `null` fills a key a channel omits; an identity key wins over an attribute key of -the same name. KPIs are duration-weighted; an unknown KPI name is rejected at config validation. Adding a -new KPI is a one-line entry in `impulse_reporting.channels.calculated_channel_kpis.KPI_BUILDERS`. - ## Incremental Calculated channels reuse the report's incremental engine (see `impulse-reporting`). A definition change — diff --git a/skills/impulse-config/SKILL.md b/skills/impulse-config/SKILL.md index 006ec019..c8fff84d 100644 --- a/skills/impulse-config/SKILL.md +++ b/skills/impulse-config/SKILL.md @@ -6,7 +6,7 @@ description: > "configure an Impulse report", set the source/sink tables, filter which containers are processed, choose RLE vs RAW, turn on incremental processing, run without writing (sinkless), remap column names, or scope by project. Covers source, unity_sink, container_filters, query_engine, solver_config, - incremental, measurement_dimensions, and calculated_channels, all validated by Pydantic. + incremental, and measurement_dimensions, all validated by Pydantic. --- # Impulse — configuration @@ -40,7 +40,6 @@ config = { }, "incremental": {"enabled": True}, "measurement_dimensions": ["container_id", "vehicle_key", "start_ts", "stop_ts"], - "calculated_channels": {"emit_channel_metrics": True, "attribute_columns": ["unit"]}, # optional } ``` @@ -98,7 +97,7 @@ Omit entirely for `DefaultSolver` + `data_type="RLE"`. | `data_type` | `"RLE"` | `"RLE"` for interval-encoded `[tstart, tend)` samples; `"RAW"` for `(timestamp, value)` samples. | | `raw_encoder` | `null` (→ `"RLE"`) | How RAW point data becomes intervals. `"RLE"` collapses equal-valued runs (default); `"INTERVAL"` keeps every sample, only deriving `tend` + dropping exact duplicates. Only used when `data_type="RAW"`. | | `drop_implausible_data` | `false` | Drop `channels` rows where `is_plausible = false`. **Requires `data_type="RAW"`** (RLE raises). | -| `batch_size` | `500` | Max selectors solved per batch. Applies to events, aggregations, and calculated channels. | +| `batch_size` | `500` | Max selectors solved per batch. | | `solver_config` | `null` | Per-table column mappings, per-table filters, and project scoping (below). | ## solver_config (optional) — adapt to an existing layout @@ -174,19 +173,3 @@ List of `container_metrics` columns (post-mapping **internal** names) to surface Default: `["container_id", "start_ts", "stop_ts"]`. Keep `container_id` — it is the incremental upsert key and the join key to fact tables. Any column present in your post-mapping `container_metrics` DataFrame is valid; a missing one fails the run fast with a `ValueError` naming it. - -## calculated_channels (optional) - -Controls the optional `calculated_channel_metrics` table. Off by default; when on, it is written alongside -`calculated_channel_fact` / `calculated_channel_dimension` in the silver `channel_metrics` shape, so the -fact + metrics pair can serve as an Impulse silver source. See `impulse-channels`. - -| Field | Default | Description | -|------------------------|--------------------------------------|-----------------------------------------------------------------------------| -| `emit_channel_metrics` | `false` | Turn on the `calculated_channel_metrics` table. | -| `attribute_columns` | `[]` | Calculated-channel `attributes` keys to surface as columns (e.g. `["unit"]`). | -| `kpis` | `["duration", "min", "max", "mean"]` | KPIs computed per `(container_id, channel_id)`, one column each. | - -Each metrics row is one `(container_id, channel_id)` pair with the selected `kpis` (duration-weighted) plus -dynamic identity columns (union of `identity` keys) and the configured `attribute_columns`; identity wins -over an attribute of the same name. An unknown KPI name is rejected at config validation. diff --git a/skills/impulse-data-model/SKILL.md b/skills/impulse-data-model/SKILL.md index bdee84d8..e26bf674 100644 --- a/skills/impulse-data-model/SKILL.md +++ b/skills/impulse-data-model/SKILL.md @@ -96,7 +96,6 @@ A star schema. Every table is prefixed with your configured `table_prefix` (e.g. | `histogram2d_fact` | One row per (x, y) bin per container | | `stats_aggregator_fact` | One row per statistic label per signal per event instance | | `calculated_channel_fact` | One row per sample interval per container (a derived signal, silver `channels` shape) | -| `calculated_channel_metrics` | One row per calculated channel per container (optional; silver `channel_metrics` shape). Written only when `config.calculated_channels.emit_channel_metrics` is set. See `impulse-channels`. | **Dimension tables** diff --git a/src/impulse_query_engine/analyze/metadata/time_series_expression.py b/src/impulse_query_engine/analyze/metadata/time_series_expression.py index 0977dc32..87bd166a 100644 --- a/src/impulse_query_engine/analyze/metadata/time_series_expression.py +++ b/src/impulse_query_engine/analyze/metadata/time_series_expression.py @@ -4,18 +4,50 @@ import operator import zlib from collections.abc import Callable, Iterable +from enum import StrEnum from typing import TYPE_CHECKING, Any import pyspark.sql.types as T import impulse_query_engine.util as U from impulse_query_engine.analyze.metadata.tag_expression import TagExpression +from impulse_query_engine.model.series.points_in_time_series import PointsInTimeSeries from impulse_query_engine.model.series.sample_series import SampleSeries if TYPE_CHECKING: from impulse_query_engine.analyze.query.solvers.series_cache import SeriesCache +class SeriesType(StrEnum): + """How a channel's samples are interpreted (mirrors :class:`RawEncoder`). + + ``SAMPLE`` — the default; ``[tstart, tend)`` intervals over which the value is + *valid* (reconstructed by an interpolation method, zero-order hold today), + backed by :class:`SampleSeries`. + + ``POINTS_IN_TIME`` — ``(tᵢ, vᵢ)`` points valid *only at* their timestamps, no + between-point validity, backed by :class:`PointsInTimeSeries`. + """ + + SAMPLE = "SAMPLE" + POINTS_IN_TIME = "POINTS_IN_TIME" + + +class PoiValueType(StrEnum): + """The value data type of a POI channel — selects its ``poi_channels`` value + column and which in-memory :class:`PointsInTimeSeries` variant is built. + + ``DOUBLE`` — numeric points (``poi_channels.value_double``); the full + arithmetic / ordering / reduction operator set applies. + + ``STRING`` — string points (``poi_channels.value_string``, e.g. DTC codes); + only sampling and equality apply (see :class:`PointsInTimeSeries`). + """ + + DOUBLE = "double" + STRING = "string" + + class RequiresDeserialization: pass @@ -619,7 +651,13 @@ def from_dict(obj: dict) -> TimeSeriesExpression: class TimeSeriesSelector(TimeSeriesExpression, RequiresDeserialization): - def __init__(self, expr, uses_alias: bool = False): + def __init__( + self, + expr, + uses_alias: bool = False, + series_type: SeriesType = SeriesType.SAMPLE, + value_type: PoiValueType = PoiValueType.DOUBLE, + ): """ Initialize a TimeSeriesSelector. @@ -627,18 +665,48 @@ def __init__(self, expr, uses_alias: bool = False): ---------- expr : TagExpression Tag expression to select. + uses_alias : bool, optional + Whether the channel resolves via the channel-alias table. + series_type : SeriesType, optional + How the selected channel's samples are interpreted. ``SAMPLE`` + (default) builds a :class:`SampleSeries` — today's behavior, + unchanged. ``POINTS_IN_TIME`` builds a :class:`PointsInTimeSeries` + (values valid only at their timestamps); identification / matching is + identical, only the built object and its result dtype differ. This is + the plan-time source of truth for the series type (so ``dtype()`` is + correct for a bare POI selection with no per-channel metadata lookup). + value_type : PoiValueType, optional + For a ``POINTS_IN_TIME`` selection, the declared value data type + (``DOUBLE`` / ``STRING``). Ignored for ``SAMPLE``. Drives plan-time + typing and string-op gating; validated against the silver + ``poi_channels.dtype`` at solve time (assertion contract). """ self._expr = expr self._uses_alias = uses_alias + self._series_type = series_type + self._value_type = value_type TimeSeriesExpression.__init__(self, is_single_signal=True) @property def uses_alias(self) -> bool: return self._uses_alias + @property + def series_type(self) -> SeriesType: + return self._series_type + + @property + def value_type(self) -> PoiValueType: + return self._value_type + @property def selector_id(self) -> int: - return zlib.crc32(str(self._expr).encode()) + # Include series_type so a SAMPLE and a POINTS_IN_TIME selection of the + # same tag expression resolve as distinct channels. SAMPLE keeps the + # historical id (bare ``str(expr)`` hash) for backward compatibility. + if self._series_type is SeriesType.SAMPLE: + return zlib.crc32(str(self._expr).encode()) + return zlib.crc32(f"{self._series_type}|{self._expr}".encode()) def dtype(self): """ @@ -647,13 +715,33 @@ def dtype(self): Returns ------- pyspark.sql.types.DataType - Data type (BinaryType). + ``BinaryType`` for a SAMPLE selection (serialized ``SampleSeries``), + or the value-type-aware ``PointsInTimeSeries.dtype()`` for a + POINTS_IN_TIME selection (``array>`` for numeric, + ``array>`` for string). """ + if self._series_type is SeriesType.POINTS_IN_TIME: + return self._empty_points_in_time().dtype() return T.BinaryType() + def _empty_points_in_time(self) -> PointsInTimeSeries: + """Empty POI series carrying this selector's declared value type. + + A string selector must build a string-typed empty series so ``dtype()`` + and the string-op gating (e.g. ``.mean()`` raising) reflect the declared + type before any data is read. + """ + if self._value_type is PoiValueType.STRING: + return PointsInTimeSeries.empty_string() + return PointsInTimeSeries.empty() + def deserialize(self, d): """ - Deserialize sample series after collection/toPandas. + Deserialize a SAMPLE result after collection/toPandas. + + POINTS_IN_TIME results are serialized by ``get_data()`` (a plain + ``[[t, v], ...]`` list) and need no deserialization, so they are returned + as-is; only a SAMPLE (binary) blob is decoded to a :class:`SampleSeries`. Parameters ---------- @@ -662,14 +750,26 @@ def deserialize(self, d): Returns ------- - SampleSeries - Deserialized sample series. + SampleSeries or Any + Deserialized sample series (SAMPLE), else *d* unchanged. """ + if self._series_type is SeriesType.POINTS_IN_TIME: + return d return SampleSeries.deserialize(d) - def build(self, cache: SeriesCache) -> SampleSeries: + def build(self, cache: SeriesCache): """ - Instantiate a SampleSeries from given cache data. + Instantiate the selected series from cache data. + + Resolution is identical regardless of series type — resolve the matching + candidates, take the first ``(container_id, channel_id)``, and let the + cache build the right object. The **data** is authoritative for the built + type: :meth:`TimeSeriesCache.load_blob` returns a + :class:`PointsInTimeSeries` for a ``POINTS_IN_TIME`` slice and a + :class:`SampleSeries` otherwise. The selector's own :attr:`series_type` / + :attr:`value_type` are used only for **plan-time** typing (:meth:`dtype` + against an empty cache), so a bare POI selection types correctly and a + string-only op is rejected before Spark runs. Parameters ---------- @@ -678,16 +778,25 @@ def build(self, cache: SeriesCache) -> SampleSeries: Returns ------- - SampleSeries - Built sample series. + SampleSeries or PointsInTimeSeries """ candidates = cache.resolve(self) if len(candidates) == 0: + if self._series_type is SeriesType.POINTS_IN_TIME: + return self._empty_points_in_time() return SampleSeries.empty() # TODO: select candidate mid = candidates.container_id.iloc[0] cid = candidates.channel_id.iloc[0] - return cache.load_blob(mid, cid, uses_alias=self.uses_alias) + # The selector is the source of truth for the series type: pass it to the + # cache so load_blob builds the right object without a per-row discriminator. + return cache.load_blob( + mid, + cid, + uses_alias=self.uses_alias, + series_type=self._series_type, + value_type=self._value_type, + ) def get_required_tag_exprs(self) -> set[TagExpression]: """ @@ -765,6 +874,8 @@ def as_dict(self) -> dict[str, Any]: obj["type"] = U.name_of(TimeSeriesSelector) obj["expr"] = self._expr.as_dict() obj["uses_alias"] = self._uses_alias + obj["series_type"] = str(self._series_type) + obj["value_type"] = str(self._value_type) return obj @staticmethod @@ -783,7 +894,14 @@ def from_dict(obj: dict): Selector instance. """ expr = TimeSeriesExpression.from_dict(obj["expr"]) - m = TimeSeriesSelector(expr, uses_alias=obj.get("uses_alias", False)) + # Default to SAMPLE / DOUBLE so selectors serialized before POI support + # (no series_type / value_type keys) deserialize unchanged. + m = TimeSeriesSelector( + expr, + uses_alias=obj.get("uses_alias", False), + series_type=SeriesType(obj.get("series_type", SeriesType.SAMPLE)), + value_type=PoiValueType(obj.get("value_type", PoiValueType.DOUBLE)), + ) if "alias" in obj and obj["alias"] is not None: m.alias(obj["alias"]) return m diff --git a/src/impulse_query_engine/analyze/query/query_builder.py b/src/impulse_query_engine/analyze/query/query_builder.py index 26ac47e3..26469b8b 100644 --- a/src/impulse_query_engine/analyze/query/query_builder.py +++ b/src/impulse_query_engine/analyze/query/query_builder.py @@ -7,7 +7,9 @@ from impulse_query_engine.analyze.metadata.metric_expression import MetricSelector from impulse_query_engine.analyze.metadata.tag_expression import TagSelector from impulse_query_engine.analyze.metadata.time_series_expression import ( + PoiValueType, RequiresDeserialization, + SeriesType, TimeSeriesExpression, TimeSeriesSelector, ) @@ -161,6 +163,54 @@ def channel_with_alias(self, **kwargs) -> TimeSeriesSelector: expr = expr & (TagSelector(k) == str(arg)) return TimeSeriesSelector(expr, uses_alias=True) + def poi_channel( + self, dtype: PoiValueType = PoiValueType.DOUBLE, **kwargs + ) -> TimeSeriesSelector: + """ + Create a Points-in-Time (POI) channel selector. + + Parallel to :meth:`channel` — it builds the **same** ``TimeSeriesSelector`` + from a tag/column match on ``**kwargs`` (e.g. + ``poi_channel(channel_name="DTC")``), differing only in that it is stamped + ``series_type=POINTS_IN_TIME`` (so it solves to a + :class:`~impulse_query_engine.model.series.points_in_time_series.PointsInTimeSeries` + — a value valid only *at* each timestamp — rather than a ``SampleSeries``) + and carries the declared value ``dtype``. + + Channel *identification* (tag/column match, ``get_selector_expr``, + ``required_tags``, ``selector_id``) is identical to :meth:`channel`; only + the built object and its result dtype differ. + + Parameters + ---------- + dtype : PoiValueType or str, optional + The POI channel's value data type: ``DOUBLE`` (default, numeric) or + ``STRING`` (e.g. DTC codes — only sampling and equality apply). Accepts + either the enum or its string value (``"double"`` / ``"string"``). This + declared type drives plan-time result typing and string-op gating; it + is validated against the silver ``poi_channels.dtype`` at solve time + (an actual/declared mismatch raises). + **kwargs : dict + Channel tag-value pairs, matched exactly like :meth:`channel`'s. + + Returns + ------- + TimeSeriesSelector + A selector stamped ``series_type=POINTS_IN_TIME`` with the given value type. + """ + # Accept a plain string ("string" / "double") as well as the enum, so + # poi_channel(..., dtype="string") behaves identically to the enum form. + value_type = PoiValueType(dtype) + expr = None + for k, arg in kwargs.items(): + if not expr: + expr = TagSelector(k) == str(arg) + else: + expr = expr & (TagSelector(k) == str(arg)) + return TimeSeriesSelector( + expr, series_type=SeriesType.POINTS_IN_TIME, value_type=value_type + ) + def select(self, *args) -> Self: """ Set the selection expressions for the query. diff --git a/src/impulse_query_engine/analyze/query/solvers/blob_solver.py b/src/impulse_query_engine/analyze/query/solvers/blob_solver.py index 6e5bcedc..833f7499 100644 --- a/src/impulse_query_engine/analyze/query/solvers/blob_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/blob_solver.py @@ -49,10 +49,15 @@ def resolve(self, selection): idx = selection._expr.build_pandas(self.df) return self.df[idx] - def load_blob(self, container_id, channel_id, uses_alias: bool = False): + def load_blob( + self, container_id, channel_id, uses_alias: bool = False, series_type=None, value_type=None + ): """ Load a time series blob from disk. + ``series_type`` / ``value_type`` are accepted for interface compatibility + with :class:`SeriesCache`; this blob cache serves only SAMPLE series. + Parameters ---------- container_id : Any diff --git a/src/impulse_query_engine/analyze/query/solvers/default_solver.py b/src/impulse_query_engine/analyze/query/solvers/default_solver.py index 5e2e5d7d..ef0a5270 100644 --- a/src/impulse_query_engine/analyze/query/solvers/default_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/default_solver.py @@ -11,6 +11,11 @@ from impulse_query_engine.analyze.metadata.metric_expression import MetricExpression from impulse_query_engine.analyze.metadata.tag_expression import TagExpression +from impulse_query_engine.analyze.metadata.time_series_expression import ( + PoiValueType, + SeriesType, +) +from impulse_query_engine.model.series.points_in_time_series import PointsInTimeSeries from impulse_query_engine.model.series.sample_series import SampleSeries from .query_solver import QuerySolver @@ -43,7 +48,11 @@ def __init__(self, pdf, col_map: dict[str, str]): col_map : dict[str, str] Mapping with keys ``"cid"``, ``"ch"``, ``"ts"``, ``"te"``, ``"val"``, ``"conv"`` to the actual column names in *pdf*. The - ``"conv"`` column is optional in *pdf*. + ``"conv"`` column is optional in *pdf*. For a POI (``POINTS_IN_TIME``) + selector, :meth:`load_blob` builds a :class:`PointsInTimeSeries` — the + **selector** (not a per-row column) chooses the series type; the + ``"value_string"`` key names the string value column that a string POI + slice reads. """ self._cid_col = col_map["cid"] self._ch_col = col_map["ch"] @@ -52,6 +61,9 @@ def __init__(self, pdf, col_map: dict[str, str]): self._val_col = col_map["val"] self._conv_col = col_map.get("conv") self._has_conversion = self._conv_col is not None and self._conv_col in pdf.columns + # String POI slices read their value from this column; series-type dispatch + # is driven by the selector passed to load_blob, not a per-row marker. + self._value_string_col = col_map.get("value_string") # *pdf* holds channel data for a whole container, so avoid creating unnecessary copies of the data. meta_cols = [ @@ -98,13 +110,20 @@ def resolve(self, selection): idx = selection._expr.build_pandas(self.mdf) return self.mdf[idx] - def load_blob(self, mid, cid, uses_alias: bool = False): + def load_blob(self, mid, cid, uses_alias: bool = False, series_type=None, value_type=None): """ Load a time series blob from the DataFrame. + The **calling selector** chooses the series type (via *series_type* / + *value_type*), so no per-row discriminator column is needed: a + ``POINTS_IN_TIME`` selector yields a :class:`PointsInTimeSeries` (string- + valued when *value_type* is ``STRING``, else numeric), otherwise a + :class:`SampleSeries`. The declared type is validated against the silver + metadata in the solve prelude, so the data stays authoritative. + When the underlying *pdf* carries a conversion-factor column (the column named by ``col_map["conv"]``) **and** the caller is an - aliased selector (``uses_alias=True``), the returned values are + aliased selector (``uses_alias=True``), the returned SAMPLE values are multiplied by that factor. Direct selectors on the same physical channel always receive raw values — unit conversion is a property of the alias, not of the channel. @@ -118,14 +137,28 @@ def load_blob(self, mid, cid, uses_alias: bool = False): uses_alias : bool, optional ``True`` when the calling selector resolved via channel_mapping. Gates the per-channel conversion factor; defaults to ``False``. + series_type : SeriesType, optional + The calling selector's series type; ``POINTS_IN_TIME`` builds a + :class:`PointsInTimeSeries`. ``None`` (default) => SAMPLE. + value_type : PoiValueType, optional + For a POI selector, its declared value type; ``STRING`` reads the + string value column, otherwise the numeric one. Returns ------- - SampleSeries - The loaded sample series object. + SampleSeries or PointsInTimeSeries """ lo, hi = self._ranges.get((mid, cid), (0, 0)) s = self.pdf.iloc[lo:hi] + + if series_type == SeriesType.POINTS_IN_TIME: + self._assert_poi_data(s, value_type) + if value_type == PoiValueType.STRING: + # value_string is a populated string column, so the constructor + # infers the string value type from it. + return PointsInTimeSeries(s[self._ts_col], s[self._value_string_col]) + return PointsInTimeSeries(s[self._ts_col], s[self._val_col]) + values = s[self._val_col] if self._has_conversion and len(s) > 0 and uses_alias: factor = s[self._conv_col].iloc[0] @@ -133,6 +166,52 @@ def load_blob(self, mid, cid, uses_alias: bool = False): values = values * factor return SampleSeries(s[self._ts_col], s[self._te_col], values) + def _assert_poi_data(self, s, value_type) -> None: + """Validate a POI selector against the data it resolved to. + + The selector drives series-type dispatch, but the silver data stays + authoritative: a ``poi_channel(...)`` selector must land on genuine POI + rows. POI rows carry a null ``tend`` (a point has no validity interval), + whereas SAMPLE rows always carry a real ``tend`` (non-nullable in + ``channels``); so a non-null ``tend`` on a POI-declared slice means the + selector was pointed at a SAMPLE channel. A ``STRING`` declaration + additionally requires a populated ``value_string``. Either mismatch raises + rather than silently reading the wrong column (mirrors the unit-conversion + conflict check). + """ + if len(s) == 0: + return + if pd.notna(s[self._te_col].iloc[0]): + raise ValueError( + "POI channel series-type mismatch: poi_channel(...) resolved to a SAMPLE " + "channel (its rows carry a validity interval). Use channel(...) for SAMPLE " + "channels and poi_channel(...) for POINTS_IN_TIME channels." + ) + + has_string_col = self._value_string_col is not None and self._value_string_col in s.columns + string_all_null = has_string_col and s[self._value_string_col].isna().all() + double_all_null = s[self._val_col].isna().all() + + if value_type == PoiValueType.STRING: + # A string POI channel must carry string values; all-null means the + # channel is actually numeric (declared the wrong dtype). + if not has_string_col or string_all_null: + raise ValueError( + "POI channel dtype mismatch: poi_channel(dtype=string) resolved to a channel " + "with no string values (it is a numeric POI channel). Pass dtype=double to " + "poi_channel(...)." + ) + else: + # A numeric POI channel must carry numeric values; all-null numeric + # with populated string values means the channel is actually a string + # channel (declared the wrong dtype). + if double_all_null and has_string_col and not string_all_null: + raise ValueError( + "POI channel dtype mismatch: poi_channel(dtype=double) resolved to a channel " + "whose numeric values are all null (it is a string POI channel). Pass " + "dtype=string to poi_channel(...)." + ) + class DefaultSolver(QuerySolver): """ @@ -1026,6 +1105,15 @@ def _prepare_channels_join(self, query, channels_df) -> tuple[DataFrame, DataFra self.config.value_col, ) + # POI channel data is unioned in AFTER RLE encoding above, so its + # zero-duration points are never run-length merged. The inner join to + # channels_df below drops any POI rows whose channel was not selected, so + # unioning whenever a poi_channels table is configured is correct (a + # pure-SAMPLE query simply matches no POI channel_ids). Which object each + # channel builds is decided by the selector (passed to load_blob), not a + # per-row marker — SAMPLE rows just lack value_string. + q = self._union_poi_channel_data(query, q) + joined_df = q.join( F.broadcast(channels_df), on=[self.config.container_id_col, self.config.channel_id_col], @@ -1033,6 +1121,35 @@ def _prepare_channels_join(self, query, channels_df) -> tuple[DataFrame, DataFra container_count = channels_df.select(self.config.container_id_col).distinct().count() return q, joined_df, container_count + def _union_poi_channel_data(self, query, channels_q: DataFrame) -> DataFrame: + """Union POI channel-data rows into the (already-encoded) channel-data frame. + + Reads ``poi_channels``, column-maps it, and projects it into the SAMPLE + channel-data superset — POI ``timestamp`` becomes ``tstart`` (``tend`` + **null**, since a point has no validity interval, which is also the signal + the cache validates a POI selector against), ``value_double`` becomes the + numeric ``value`` column, and ``value_string`` rides alongside for a string + POI channel. No per-row ``series_type`` / ``dtype`` marker is shipped: the + selector drives series-type dispatch in :meth:`TimeSeriesCache.load_blob`. + Returns *channels_q* unchanged when no ``poi_channels`` table is configured. + """ + db = query.db + if not (hasattr(db, "has_poi_channels") and db.has_poi_channels()): + return channels_q + + cfg = self.config + poi = db.poi_channels(self.spark) + poi = self._apply_column_mapping(poi, cfg.poi_channels.column_name_mapping) + poi_proj = poi.select( + F.col(cfg.container_id_col), + F.col(cfg.channel_id_col), + F.col(cfg.poi_timestamp_col).alias(cfg.tstart_col), + F.lit(None).cast(T.LongType()).alias(cfg.tend_col), + F.col(cfg.poi_value_double_col).alias(cfg.value_col), + F.col(cfg.poi_value_string_col).alias(cfg.poi_value_string_col), + ) + return channels_q.unionByName(poi_proj, allowMissingColumns=True) + def _apply_grouped_map(self, joined_df, container_count, schema, solve_udf) -> DataFrame: """Run *solve_udf* per container, or return an empty frame when none match.""" if container_count == 0: diff --git a/src/impulse_query_engine/analyze/query/solvers/empty_cache.py b/src/impulse_query_engine/analyze/query/solvers/empty_cache.py index 32ca0d1d..96641263 100644 --- a/src/impulse_query_engine/analyze/query/solvers/empty_cache.py +++ b/src/impulse_query_engine/analyze/query/solvers/empty_cache.py @@ -25,7 +25,7 @@ def resolve(self, selection): """ return [] - def load_blob(self, mid, cid, uses_alias: bool = False): + def load_blob(self, mid, cid, uses_alias: bool = False, series_type=None, value_type=None): """ Return an empty SampleSeries for any container and channel ID. @@ -38,6 +38,11 @@ def load_blob(self, mid, cid, uses_alias: bool = False): uses_alias : bool, optional Unused by this cache; accepted for interface compatibility with :class:`SeriesCache`. + series_type, value_type : optional + Accepted for interface compatibility. The empty-series typing for a + POI selector is handled by ``TimeSeriesSelector.build`` (its length-0 + branch), which returns the correctly typed empty series without + reaching this method. Returns ------- diff --git a/src/impulse_query_engine/analyze/query/solvers/series_cache.py b/src/impulse_query_engine/analyze/query/solvers/series_cache.py index 081d53b3..8debf8d1 100644 --- a/src/impulse_query_engine/analyze/query/solvers/series_cache.py +++ b/src/impulse_query_engine/analyze/query/solvers/series_cache.py @@ -24,7 +24,14 @@ def resolve(self, selection) -> pd.DataFrame: pass @abstractmethod - def load_blob(self, mid, cid, uses_alias: bool = False) -> SampleSeries: + def load_blob( + self, + mid, + cid, + uses_alias: bool = False, + series_type=None, + value_type=None, + ) -> SampleSeries: """ Resolve given mid and cid to a series. @@ -41,10 +48,22 @@ def load_blob(self, mid, cid, uses_alias: bool = False) -> SampleSeries: conversion factor when this is ``True``, so a direct selector on the same physical channel always returns raw values. Defaults to ``False`` (direct / no-conversion semantics). + series_type : SeriesType, optional + The calling selector's series type. The selector — not a per-row + data column — is the source of truth for which object to build: + ``POINTS_IN_TIME`` builds a :class:`PointsInTimeSeries`, otherwise a + :class:`SampleSeries`. ``None`` (default) means SAMPLE, so callers + that predate POI are unchanged. + value_type : PoiValueType, optional + For a ``POINTS_IN_TIME`` selector, its declared value type + (``DOUBLE`` / ``STRING``) — selects the numeric vs string value + column. Ignored for SAMPLE. The declared type is validated against + the silver metadata in the solve prelude, so the data stays + authoritative. Returns ------- - SampleSeries - The loaded sample series object. + SampleSeries or PointsInTimeSeries + The loaded series object. """ pass diff --git a/src/impulse_query_engine/analyze/query/solvers/solver_config.py b/src/impulse_query_engine/analyze/query/solvers/solver_config.py index 0ca71138..0eb11227 100644 --- a/src/impulse_query_engine/analyze/query/solvers/solver_config.py +++ b/src/impulse_query_engine/analyze/query/solvers/solver_config.py @@ -143,6 +143,7 @@ class SolverConfig(BaseModel): channel_metrics: TableConfig = TableConfig() channel_mapping: ChannelMappingConfig = ChannelMappingConfig() channels: TableConfig = TableConfig() + poi_channels: TableConfig = TableConfig() unit_conversion: TableConfig = TableConfig() # ------------------------------------------------------------------ @@ -231,6 +232,26 @@ def value_col(self) -> str: """Internal column name for the signal value on the channels table.""" return "value" + @property + def poi_timestamp_col(self) -> str: + """Internal column name for the point timestamp on the poi_channels table.""" + return "timestamp" + + @property + def poi_value_double_col(self) -> str: + """Internal column name for the numeric value on the poi_channels table.""" + return "value_double" + + @property + def poi_value_string_col(self) -> str: + """Internal column name for the string value on the poi_channels table.""" + return "value_string" + + @property + def poi_dtype_col(self) -> str: + """Internal column name for the per-row value-dtype discriminator on poi_channels.""" + return "dtype" + @property def tag_key_col(self) -> str: """Internal column name for the attribute key on the container_tags (EAV) table.""" @@ -379,4 +400,8 @@ def col_map(self) -> dict[str, str]: "te": self.tend_col, "val": self.value_col, "conv": self.conversion_factor_col, + # String POI slices read their value from this column. Series-type + # dispatch is driven by the selector (passed to load_blob), so no + # per-row series_type / dtype marker column is needed in the frame. + "value_string": self.poi_value_string_col, } diff --git a/src/impulse_query_engine/measurement_db.py b/src/impulse_query_engine/measurement_db.py index c0ba27ca..3ebdcded 100644 --- a/src/impulse_query_engine/measurement_db.py +++ b/src/impulse_query_engine/measurement_db.py @@ -14,6 +14,7 @@ def __init__( channel_tags_table=None, channel_metrics_table=None, channels_uri=None, + poi_channels_uri=None, channel_mapping_table=None, unit_conversion_table=None, table_locations: str = "external_locations", @@ -23,6 +24,9 @@ def __init__( self.channel_tags_table = channel_tags_table self.channel_metrics_table = channel_metrics_table self.channels_uri = channels_uri + # Optional Points-in-Time (POI) channel-data table. ``None`` means no POI + # channels are configured, so POI-unaware deployments are unchanged. + self.poi_channels_uri = poi_channels_uri self.channel_mapping_table = channel_mapping_table self.unit_conversion_table = unit_conversion_table self.table_locations = table_locations @@ -34,6 +38,7 @@ def for_unity_catalog( core_schema_name: str = "core", channel_mapping_table: str | None = None, unit_conversion_table: str | None = None, + poi_channels_uri: str | None = None, ): return MeasurementDBConfig( container_tags_table=f"{catalog_name}.{core_schema_name}.container_tags", @@ -41,6 +46,7 @@ def for_unity_catalog( channel_tags_table=f"{catalog_name}.{core_schema_name}.channel_tags", channel_metrics_table=f"{catalog_name}.{core_schema_name}.channel_metrics", channels_uri=f"{catalog_name}.{core_schema_name}.channels", + poi_channels_uri=poi_channels_uri, channel_mapping_table=channel_mapping_table, unit_conversion_table=unit_conversion_table, table_locations="unity_catalog", @@ -58,6 +64,7 @@ def for_debug(debug_tables): "channel_metrics" if "channel_metrics" in debug_tables else None ), channels_uri="channels" if "channels" in debug_tables else None, + poi_channels_uri="poi_channels" if "poi_channels" in debug_tables else None, channel_mapping_table=( "channel_mapping" if "channel_mapping" in debug_tables else None ), @@ -103,6 +110,20 @@ def channel_metrics(self, spark) -> DataFrame: def channels(self, spark) -> DataFrame: return self._read_table(spark, self.config.channels_uri) + def has_poi_channels(self) -> bool: + """Whether a Points-in-Time (POI) channel-data table is configured.""" + return getattr(self.config, "poi_channels_uri", None) is not None + + def poi_channels(self, spark) -> DataFrame: + """Read the Points-in-Time (POI) channel-data table. + + Parallel to :meth:`channels`. Raises if no ``poi_channels_uri`` is + configured — callers should gate on :meth:`has_poi_channels` first. + """ + if not self.has_poi_channels(): + raise ValueError("poi_channels_uri is not configured") + return self._read_table(spark, self.config.poi_channels_uri) + def channel_mapping(self, spark) -> DataFrame: if self.config.channel_mapping_table is None: raise ValueError("channel_mapping_table is not configured") diff --git a/src/impulse_query_engine/model/series/points_in_time_series.py b/src/impulse_query_engine/model/series/points_in_time_series.py index ba46f831..44d8dcd0 100644 --- a/src/impulse_query_engine/model/series/points_in_time_series.py +++ b/src/impulse_query_engine/model/series/points_in_time_series.py @@ -2,7 +2,8 @@ from __future__ import annotations -from collections.abc import Sized +import functools +from collections.abc import Callable, Sized import numpy as np import numpy.typing as npt @@ -15,6 +16,34 @@ FloatOrNaN = float | np.float64 +def _numeric_only(method: Callable) -> Callable: + """Decorator that rejects the wrapped method on a string-valued series. + + String-valued :class:`PointsInTimeSeries` support only sampling and equality + (``==`` / ``!=``); arithmetic, ordering and numeric reductions have no meaning + for them. Numpy would either raise (``-``, ``/``, ``mean``) or — worse — + silently succeed with a nonsensical result (``+`` concatenates, ``*`` repeats, + ``sum`` concatenates), so guard those methods explicitly and fail loudly. + + Applied to arithmetic, ordering-comparison and reduction methods. + + Raises + ------ + TypeError + When the decorated method is called on a string-valued series. + """ + + @functools.wraps(method) + def wrapper(self: PointsInTimeSeries, *args, **kwargs): + if self._is_string: + raise TypeError( + f"{method.__name__} is not supported for string-valued PointsInTimeSeries" + ) + return method(self, *args, **kwargs) + + return wrapper + + class PointsInTimeSeries: def __init__(self, tstarts: Sized, values: Sized): """ @@ -24,6 +53,11 @@ def __init__(self, tstarts: Sized, values: Sized): a value is only defined *at* its timestamp and is not considered valid in between consecutive timestamps. + The value type (numeric vs string) is inferred from *values*. An **empty** + series has no values to infer from and therefore defaults to numeric; use + :meth:`empty_string` when an explicitly string-typed empty series is needed + (e.g. plan-time result typing of a bare string-POI selection). + Parameters ---------- tstarts : Sized @@ -32,31 +66,60 @@ def __init__(self, tstarts: Sized, values: Sized): Array-like of values, one per time point. """ assert len(tstarts) == len(values) + # Timestamps are always numeric. Values may be numeric or string: + # string-valued series support sampling (``synchronized`` / ``.where``) + # and equality comparisons (``==`` / ``!=``) only — arithmetic, ordering + # and numeric reductions are rejected (see the ``@_numeric_only`` methods). self.tstarts = np.array(tstarts, dtype=np.float64) - self.values = np.array(values, dtype=np.float64) + self._is_string = np.asarray(values).dtype.kind in ("U", "S", "O") + if self._is_string: + self.values = np.asarray(values, dtype=object) + else: + self.values = np.array(values, dtype=np.float64) def dtype(self): """ Returns the Spark data type for PointsInTimeSeries. + For numeric values the element is a homogeneous ``[tstart, value]`` double + pair (``ArrayType(ArrayType(DoubleType))``). String-valued series cannot use + that homogeneous nested array, so their element is a ``(tstart, value)`` + struct with a double timestamp and a string value. + Returns ------- pyspark.sql.types.ArrayType - Spark ArrayType for points in time series: [[tstart_1, value_1], ...]. - """ + Spark ArrayType matching ``get_data``'s shape for this series' value type. + """ + if self._is_string: + return T.ArrayType( + T.StructType( + [ + T.StructField("tstart", T.DoubleType()), + T.StructField("value", T.StringType()), + ] + ) + ) return T.ArrayType(T.ArrayType(T.DoubleType())) def get_data(self) -> list: """ - Returns the series as a list of [tstart, value] lists. + Returns the series as a list of ``[tstart, value]`` pairs. + + For numeric values this is a list of two-element double lists. For string + values, ``column_stack`` would coerce the timestamps to strings, so the + pairs are built explicitly as ``[float(tstart), str(value)]`` — matching the + struct element type declared by :meth:`dtype`. Returns ------- list - List of [tstart, value] pairs. + List of ``[tstart, value]`` pairs. """ if len(self) == 0: return [] + if self._is_string: + return [[float(t), str(v)] for t, v in zip(self.tstarts, self.values, strict=True)] return np.column_stack([self.tstarts, self.values]).tolist() def __len__(self) -> int: @@ -354,34 +417,42 @@ def _apply_basic_rop(self, operation, other: float | SampleSeries | PointsInTime return PointsInTimeSeries(s0.tstarts, operation(s1.values, s0.values)) return PointsInTimeSeries(self.tstarts, operation(other, self.values)) + @_numeric_only def __add__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Add another series or scalar to this series.""" return self._apply_basic_op(np.add, other) + @_numeric_only def __radd__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Add this series to another series or scalar (reversed operands).""" return self._apply_basic_rop(np.add, other) + @_numeric_only def __sub__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Subtract another series or scalar from this series.""" return self._apply_basic_op(np.subtract, other) + @_numeric_only def __rsub__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Subtract this series from another series or scalar (reversed operands).""" return self._apply_basic_rop(np.subtract, other) + @_numeric_only def __mul__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Multiply this series by another series or scalar.""" return self._apply_basic_op(np.multiply, other) + @_numeric_only def __rmul__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Multiply another series or scalar by this series (reversed operands).""" return self._apply_basic_rop(np.multiply, other) + @_numeric_only def __truediv__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Divide this series by another series or scalar.""" return self._apply_basic_op(np.true_divide, other) + @_numeric_only def __rtruediv__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Divide another series or scalar by this series (reversed operands).""" return self._apply_basic_rop(np.true_divide, other) @@ -411,18 +482,22 @@ def __apply_op( idx = operation(self.values, other) return PointsInTime(self.tstarts[idx]) + @_numeric_only def __gt__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTime: """Return points where this series is greater than another.""" return self.__apply_op(np.greater, other) + @_numeric_only def __ge__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTime: """Return points where this series is greater than or equal to another.""" return self.__apply_op(np.greater_equal, other) + @_numeric_only def __lt__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTime: """Return points where this series is less than another.""" return self.__apply_op(np.less, other) + @_numeric_only def __le__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTime: """Return points where this series is less than or equal to another.""" return self.__apply_op(np.less_equal, other) @@ -448,6 +523,7 @@ def count(self) -> int: """ return len(self) + @_numeric_only def sum(self) -> FloatOrNaN: """ Returns the sum of the values. @@ -461,6 +537,7 @@ def sum(self) -> FloatOrNaN: return np.nan return np.sum(self.values) + @_numeric_only def mean(self) -> FloatOrNaN: """ Returns the mean of the values. @@ -474,6 +551,7 @@ def mean(self) -> FloatOrNaN: return np.nan return np.mean(self.values) + @_numeric_only def min(self) -> FloatOrNaN: """ Returns the minimum value. @@ -487,6 +565,7 @@ def min(self) -> FloatOrNaN: return np.nan return np.min(self.values) + @_numeric_only def max(self) -> FloatOrNaN: """ Returns the maximum value. @@ -528,11 +607,34 @@ def __repr__(self) -> str: @staticmethod def empty() -> PointsInTimeSeries: """ - Returns an empty PointsInTimeSeries. + Returns an empty (numeric) PointsInTimeSeries. Returns ------- PointsInTimeSeries - Empty PointsInTimeSeries object. + Empty numeric PointsInTimeSeries object. """ return PointsInTimeSeries([], []) + + @staticmethod + def empty_string() -> PointsInTimeSeries: + """ + Returns an empty **string-valued** PointsInTimeSeries. + + An empty series has no values to infer a type from, so the constructor + defaults to numeric; this factory forces the string value type. Used for + plan-time result typing of a bare string-POI selection, where the empty + series must report the string ``dtype()`` and reject numeric-only ops + (e.g. ``mean()``) before any data is read. + + Returns + ------- + PointsInTimeSeries + Empty string-valued PointsInTimeSeries object. + """ + # A single-element object array makes the constructor infer string, then + # slice back to empty so no value is retained. + series = PointsInTimeSeries([], []) + series._is_string = True + series.values = np.asarray([], dtype=object) + return series diff --git a/src/impulse_query_engine/schema.py b/src/impulse_query_engine/schema.py index 8323182e..f64465bc 100644 --- a/src/impulse_query_engine/schema.py +++ b/src/impulse_query_engine/schema.py @@ -52,6 +52,26 @@ ] ) +# Points-in-Time (POI) channel samples: a value defined only *at* its timestamp +# (no derived tend / validity interval). Two typed value columns plus a per-row +# dtype discriminator, since a POI value may be numeric or a string; exactly one of +# value_double / value_string is populated per row, selected by dtype. +# +# A channel is a POI channel iff its data lives here rather than in ``channels`` — +# table membership *is* the series-type discriminator, so no ``series_type`` column +# is needed on ``channel_metrics``. A given (container_id, channel_id) lives in +# exactly one of ``channels`` / ``poi_channels``. +POI_CHANNELS_SCHEMA = T.StructType( + [ + T.StructField("container_id", T.LongType(), nullable=False), + T.StructField("channel_id", T.IntegerType(), nullable=False), + T.StructField("timestamp", T.LongType(), nullable=False), + T.StructField("value_double", T.DoubleType()), + T.StructField("value_string", T.StringType()), + T.StructField("dtype", T.StringType(), nullable=False), + ] +) + CHANNELS_SCHEMA = T.StructType( [ T.StructField("container_id", T.LongType(), nullable=False), diff --git a/src/impulse_reporting/channels/calculated_channel.py b/src/impulse_reporting/channels/calculated_channel.py index f7d857cb..4bba5319 100644 --- a/src/impulse_reporting/channels/calculated_channel.py +++ b/src/impulse_reporting/channels/calculated_channel.py @@ -3,8 +3,6 @@ import hashlib from collections.abc import Mapping -import pyspark.sql.functions as F -import pyspark.sql.types as T from pyspark.sql import DataFrame, Row, SparkSession from impulse_query_engine.analyze.metadata.time_series_expression import ( @@ -13,28 +11,13 @@ from impulse_query_engine.analyze.query.channels.calculated_channel import ( CalculatedChannel as QeCalculatedChannel, ) +from impulse_query_engine.analyze.query.query_builder import QueryBuilder +from impulse_query_engine.analyze.query.solvers.query_solver import QuerySolver from impulse_query_engine.model.series.sample_series import SampleSeries -from impulse_reporting.channels.calculated_channel_kpis import ( - DEFAULT_KPIS, - build_kpi_columns, -) from impulse_reporting.persist.dimension_schema import CALCULATED_CHANNEL_DIMENSION_SCHEMA from impulse_reporting.persist.fact_schema import CALCULATED_CHANNEL_FACT_SCHEMA -def _union_identity_keys(channels: list[CalculatedChannel]) -> list[str]: - """Return the sorted union of identity keys across ``channels``. - - Used to build the dynamic identity columns of the calculated-channel metrics - table: every key any channel declares becomes a column (null on channels that - omit it). Sorting gives a stable, order-independent column layout. - """ - keys: set[str] = set() - for channel in channels: - keys.update(channel.identity) - return sorted(keys) - - class CalculatedChannel: """A reporting-layer calculated (derived) channel. @@ -45,13 +28,10 @@ class CalculatedChannel: the narrow result to a gold fact table, and update it incrementally. Structurally parallels :class:`BasicEvent` (holds an aliased expression, - name-derived id, SHA-256 definition hash). Like the other entity types, its - narrow solve is batched from :class:`Report` (see - ``Report._solve_calculated_channels_batched``): the channels are partitioned by - ``batch_size``, each batch solved via ``QueryBuilder.solve_calculated_channels`` - and persisted as a temp table, then the batches are unioned into a narrow - ``solved_df``. :meth:`determine_calculated_channels` shapes that already-solved - df, mirroring ``determine_aggregations`` / ``determine_events``. + name-derived id, SHA-256 definition hash) but — like ``ContainerEvent`` — it + drives its own solve via ``QueryBuilder.solve_calculated_channels`` rather than + riding the centralized wide ``solved_df``. It is dispatched separately from + the batch solve (never passed to ``collect_solvable_expressions``). Parameters ---------- @@ -177,41 +157,44 @@ def determine_calculated_channels( spark: SparkSession, channels: list[CalculatedChannel], *, - solved_df: DataFrame = None, + query: QueryBuilder = None, + solver: QuerySolver = None, + pre_filtered_containers_df: DataFrame = None, ) -> DataFrame | None: - """Shape the already-solved narrow rows into this type's fact rows. + """Solve the given channels and shape the result into fact rows. - Mirrors ``determine_aggregations`` / ``determine_events``: the batched - narrow solve happens in the ``Report`` (see - ``Report._solve_calculated_channels_batched``), and this only *shapes* the - resulting ``solved_df``: it selects the rows for these channels (by - ``channel_id``) and projects to :data:`CALCULATED_CHANNEL_FACT_SCHEMA`. - Each channel's ``channel_id`` was fixed to its entity id at construction, - so the filter needs no join. + Drives ``QueryBuilder.solve_calculated_channels`` (the narrow, many-rows- + per-container endpoint) with the report's ``query`` + ``solver``, then + projects to :data:`CALCULATED_CHANNEL_FACT_SCHEMA`. Because each channel's + ``channel_id`` was fixed to its entity id at construction, no id-join is + needed. Parameters ---------- spark : SparkSession - Active Spark session (unused; kept for dispatcher symmetry). + Spark session, forwarded to ``QueryBuilder.solve_calculated_channels``. channels : list of CalculatedChannel - The channels whose rows to select from ``solved_df``. - solved_df : DataFrame, optional - Narrow batched solve output (``container_id, channel_id, tstart, tend, - value, identity``). ``None`` (no channels solved) returns ``None``. + Channels to solve; identity keys may differ across channels. + query : QueryBuilder + Query builder used to select and solve the channels. + solver : QuerySolver + Solver implementing ``solve_calculated_channels`` (a ``DefaultSolver``). + pre_filtered_containers_df : DataFrame, optional + Incremental container subset; ``None`` processes all containers. Returns ------- DataFrame or None - Narrow fact DataFrame, or ``None`` when there are no channels or no - solved rows. + Narrow fact DataFrame, or ``None`` when there are no channels. """ - if not channels or solved_df is None: + if not channels: return None - channel_ids = [channel.get_id() for channel in channels] - return solved_df.filter(F.col("channel_id").isin(channel_ids)).select( - *CALCULATED_CHANNEL_FACT_SCHEMA.fieldNames() + qe_channels = [channel.expression for channel in channels] + df = query.select(*qe_channels).solve_calculated_channels( + spark, solver, pre_filtered_containers_df ) + return df.select(*CALCULATED_CHANNEL_FACT_SCHEMA.fieldNames()) @classmethod def determine_metadata_df(cls, spark: SparkSession, channels: list[CalculatedChannel]): @@ -223,99 +206,3 @@ def determine_metadata_df(cls, spark: SparkSession, channels: list[CalculatedCha """ rows = [channel.as_spark_row() for channel in channels] return spark.createDataFrame(rows, schema=CALCULATED_CHANNEL_DIMENSION_SCHEMA) - - @classmethod - def determine_channel_metrics( - cls, - spark: SparkSession, - channels: list[CalculatedChannel], - fact_df: DataFrame | None, - *, - attribute_columns: list[str] | None = None, - kpis: list[str] | None = None, - ) -> DataFrame | None: - """Derive a silver-shaped ``channel_metrics`` DataFrame from the fact rows. - - The calculated-channel fact table already matches the silver ``channels`` - table; this builds its companion ``channel_metrics`` so the pair can serve - as an Impulse silver source. Metrics are aggregated **directly from the - narrow fact rows** (``container_id, channel_id, tstart, tend, value``), - grouped by ``(container_id, channel_id)``. - - The output schema is **dynamic**: fixed columns ``container_id, - channel_id, value_type`` plus one column per configured KPI (see ``kpis``), - one per identity key (the union across all ``channels``), and one per - configured attribute key. Identity/attribute values are pulled from - each channel's in-memory ``identity`` / ``attributes`` dicts (null where a - channel omits a key). On an identity/attribute key collision, identity wins - and the attribute is skipped. - - Parameters - ---------- - spark : SparkSession - Session used to build the per-channel metadata frame. - channels : list of CalculatedChannel - The channels whose fact rows are in ``fact_df``; supply identity and - attributes. - fact_df : DataFrame or None - Narrow fact DataFrame (output of :meth:`determine_calculated_channels`). - ``None`` returns ``None``. - attribute_columns : list of str, optional - Attribute keys to surface as columns. Default/empty → no attribute - columns. A key no channel defines yields an all-null column. - kpis : list of str, optional - KPI names to compute (see ``calculated_channel_kpis.KPI_BUILDERS``); the - output carries one column per name, in order. ``None`` → the default - KPIs (``duration, min, max, mean``). - - Returns - ------- - DataFrame or None - The dynamic-schema metrics DataFrame, or ``None`` when ``fact_df`` is - ``None``. - """ - if fact_df is None: - return None - - attribute_columns = list(attribute_columns or []) - kpis = list(kpis) if kpis is not None else list(DEFAULT_KPIS) - - # KPI aggregations are built from the registry (single extension point), so - # the computed columns and the projection below both follow ``kpis``. - agg_df = fact_df.groupBy("container_id", "channel_id").agg(*build_kpi_columns(kpis)) - - # Union of identity keys (stable order); attribute columns lose to identity - # on a key collision. - identity_keys = _union_identity_keys(channels) - effective_attribute_keys = [k for k in attribute_columns if k not in identity_keys] - - # Per-channel metadata frame. channel_id is cast to the fact's channel_id - # type so the join keys line up regardless of int/long width. - channel_id_type = fact_df.schema["channel_id"].dataType - meta_schema = T.StructType( - [T.StructField("channel_id", channel_id_type, False)] - + [T.StructField(k, T.StringType(), True) for k in identity_keys] - + [T.StructField(k, T.StringType(), True) for k in effective_attribute_keys] - ) - meta_rows = [] - for channel in channels: - row: dict = {"channel_id": channel.get_id()} - for key in identity_keys: - row[key] = channel.identity.get(key) - for key in effective_attribute_keys: - row[key] = channel.attributes.get(key) - meta_rows.append(Row(**row)) - meta_df = spark.createDataFrame(meta_rows, schema=meta_schema) - - result = agg_df.join(meta_df, on="channel_id", how="left").withColumn( - "value_type", F.lit("double") - ) - - ordered_columns = ( - ["container_id", "channel_id"] - + identity_keys - + effective_attribute_keys - + ["value_type"] - + kpis - ) - return result.select(*ordered_columns) diff --git a/src/impulse_reporting/channels/calculated_channel_kpis.py b/src/impulse_reporting/channels/calculated_channel_kpis.py deleted file mode 100644 index 4f1db7a4..00000000 --- a/src/impulse_reporting/channels/calculated_channel_kpis.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Registry of calculated-channel metric KPIs. - -Each KPI is a named builder that returns a Spark aggregation :class:`Column` -(already ``.alias(name)``-ed) computed over the narrow calculated-channel fact -rows grouped by ``(container_id, channel_id)``. The registry is the single -extension point for the ``calculated_channel_metrics`` table: **adding a KPI means -adding one entry to** :data:`KPI_BUILDERS` — it then becomes selectable via -``CalculatedChannels.kpis`` config with no other changes. - -Semantics match :class:`SampleSeries` (duration-weighted where relevant), with -``dur = tend - tstart``. NaN values are excluded from ``min``/``max``/``mean`` -(matching ``np.nanmin`` / ``nanmax`` / ``nansum``); the ``mean`` denominator keeps -every interval's duration and uses ``try_divide`` so a zero total duration (a group -of only zero-duration point-in-time samples) yields ``null`` instead of failing the -run under ANSI mode (Spark 4.0 default). -""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass - -import pyspark.sql.functions as F -from pyspark.sql import Column - - -@dataclass(frozen=True) -class KpiColumns: - """Reusable per-group column expressions shared by the KPI builders. - - Built once per aggregation from the fact columns so each builder does not - recompute the duration / NaN-masking expressions. - """ - - dur: Column - value: Column - non_nan_value: Column - weighted_value: Column - - @classmethod - def from_fact(cls) -> "KpiColumns": - """Build the shared expressions from the fixed fact column names.""" - dur = F.col("tend") - F.col("tstart") - value = F.col("value") - non_nan_value = F.when(~F.isnan(value), value) - weighted_value = F.when(~F.isnan(value), value * dur).otherwise(F.lit(0.0)) - return cls( - dur=dur, value=value, non_nan_value=non_nan_value, weighted_value=weighted_value - ) - - -def _duration(cols: KpiColumns) -> Column: - return (F.max("tend") - F.min("tstart")).alias("duration") - - -def _min(cols: KpiColumns) -> Column: - return F.min(cols.non_nan_value).alias("min") - - -def _max(cols: KpiColumns) -> Column: - return F.max(cols.non_nan_value).alias("max") - - -def _mean(cols: KpiColumns) -> Column: - # Duration-weighted; try_divide → null (not error) when total duration is zero. - return F.try_divide(F.sum(cols.weighted_value), F.sum(cols.dur)).alias("mean") - - -# Name → aggregation-Column builder. Adding a KPI = adding one entry here. -KPI_BUILDERS: dict[str, Callable[[KpiColumns], Column]] = { - "duration": _duration, - "min": _min, - "max": _max, - "mean": _mean, -} - -# Default KPIs computed when config does not narrow the selection. -DEFAULT_KPIS: list[str] = ["duration", "min", "max", "mean"] - - -def build_kpi_columns(names: list[str]) -> list[Column]: - """Return the aggregation columns for ``names`` in the given order. - - Parameters - ---------- - names : list of str - KPI names to compute; each must be a key of :data:`KPI_BUILDERS`. - - Returns - ------- - list of Column - Aliased aggregation columns, one per name, ready to splat into ``.agg``. - - Raises - ------ - ValueError - If any name is not a registered KPI. - """ - unknown = [n for n in names if n not in KPI_BUILDERS] - if unknown: - valid = ", ".join(sorted(KPI_BUILDERS)) - raise ValueError(f"Unknown calculated-channel KPI(s): {unknown}. Valid KPIs: {valid}.") - cols = KpiColumns.from_fact() - return [KPI_BUILDERS[name](cols) for name in names] diff --git a/src/impulse_reporting/channels/channel_types.py b/src/impulse_reporting/channels/channel_types.py index db89319d..bcb63330 100644 --- a/src/impulse_reporting/channels/channel_types.py +++ b/src/impulse_reporting/channels/channel_types.py @@ -83,32 +83,6 @@ def get_dimension_table_name(self) -> str: case _: raise ValueError(f"Unsupported channel type: {self}") - def get_metrics_table_name(self) -> str: - """ - Get the (optional) channel-metrics table name for the channel type. - - This table mirrors the silver-layer ``channel_metrics`` table so the - calculated-channel fact + metrics pair can serve as an Impulse silver - source. Unlike the fact/dimension tables it has **no** fixed schema - constant: identity/attribute columns are derived dynamically per report - (see :meth:`CalculatedChannel.determine_channel_metrics`). - - Returns - ------- - str - The name of the channel-metrics table associated with this channel type. - - Raises - ------ - ValueError - If the channel type is not supported. - """ - match self: - case ChannelType.CALCULATED_CHANNEL: - return "calculated_channel_metrics" - case _: - raise ValueError(f"Unsupported channel type: {self}") - def get_dimension_schema(self) -> StructType: """ Get the dimension schema for the channel type. @@ -152,29 +126,6 @@ def get_any_for_fact_table(cls, table_name: str) -> "ChannelType": return ct raise ValueError(f"No ChannelType found for fact table: {table_name}") - @classmethod - def get_any_for_metrics_table(cls, table_name: str) -> "ChannelType": - """Return the first ChannelType whose metrics table name matches. - - Parameters - ---------- - table_name : str - Metrics table name to look up. - - Returns - ------- - ChannelType - - Raises - ------ - ValueError - If no ChannelType matches the given table name. - """ - for ct in cls: - if ct.get_metrics_table_name() == table_name: - return ct - raise ValueError(f"No ChannelType found for metrics table: {table_name}") - @classmethod def get_any_for_dimension_table(cls, table_name: str) -> "ChannelType": """Return the first ChannelType whose dimension table name matches. diff --git a/src/impulse_reporting/config/config_parser.py b/src/impulse_reporting/config/config_parser.py index f8e3d0b8..3dc0f9a3 100644 --- a/src/impulse_reporting/config/config_parser.py +++ b/src/impulse_reporting/config/config_parser.py @@ -6,7 +6,6 @@ from pydantic import AfterValidator, BaseModel, field_validator, model_validator from impulse_query_engine.analyze.query.solvers.solver_config import RawEncoder, SolverConfig -from impulse_reporting.channels.calculated_channel_kpis import DEFAULT_KPIS, KPI_BUILDERS def is_valid_table_name(table_name: str) -> str: @@ -128,6 +127,10 @@ class Source(BaseModel): Full Unity Catalog path to the channel metrics table. channels_uri : str Full Unity Catalog path to the channels data table. + poi_channels_uri : str, optional + Full Unity Catalog path to the Points-in-Time (POI) channel data table. + Required only when the report selects POI channels via ``poi_channel()``; + omit it for sample-only data models. channel_mapping_table : str, optional Full Unity Catalog path to the channel mapping table. Required when using ``channel_with_alias()`` for logical alias resolution. @@ -148,6 +151,7 @@ class Source(BaseModel): container_metrics_table: Annotated[str, AfterValidator(is_valid_table_name)] channel_metrics_table: Annotated[str, AfterValidator(is_valid_table_name)] channels_uri: Annotated[str, AfterValidator(is_valid_table_name)] + poi_channels_uri: Annotated[str, AfterValidator(is_valid_table_name)] | None = None channel_mapping_table: Annotated[str, AfterValidator(is_valid_table_name)] | None = None unit_conversion_table: Annotated[str, AfterValidator(is_valid_table_name)] | None = None @@ -443,58 +447,6 @@ class IncrementalConfig(BaseModel): gold_last_modified_column: str = "_created_at" -class CalculatedChannels(BaseModel): - """ - Configuration for calculated-channel outputs. - - Attributes - ---------- - emit_channel_metrics : bool, default=False - When True, also emit a ``calculated_channel_metrics`` gold table (silver - ``channel_metrics`` shape) alongside the calculated-channel fact table, so - the fact + metrics pair can serve as an Impulse silver source. - attribute_columns : list of str, default=[] - Calculated-channel attribute keys to surface as columns on the metrics - table (e.g. ``["unit"]``). Empty (the default) → no attribute columns. - Identity keys are always surfaced dynamically and win over an - attribute key of the same name. - kpis : list of str, default=["duration", "min", "max", "mean"] - KPIs computed on the metrics table, one column per name. Each must be a - registered KPI (see ``calculated_channel_kpis.KPI_BUILDERS``); an unknown - name is rejected at validation. Duplicates are removed (order preserved). - """ - - emit_channel_metrics: bool = False - attribute_columns: list[str] = [] - kpis: list[str] = list(DEFAULT_KPIS) - - @field_validator("attribute_columns", mode="after") - @classmethod - def _normalize_attribute_columns(cls, value: list[str]) -> list[str]: - seen: set[str] = set() - normalized: list[str] = [] - for name in value: - is_valid_unity_entity_name(name) - if name not in seen: - seen.add(name) - normalized.append(name) - return normalized - - @field_validator("kpis", mode="after") - @classmethod - def _normalize_kpis(cls, value: list[str]) -> list[str]: - seen: set[str] = set() - normalized: list[str] = [] - for name in value: - if name not in KPI_BUILDERS: - valid = ", ".join(sorted(KPI_BUILDERS)) - raise ValueError(f"Unknown calculated-channel KPI: {name}. Valid KPIs: {valid}.") - if name not in seen: - seen.add(name) - normalized.append(name) - return normalized - - class ImpulseConfig(BaseModel): """ Main configuration model. @@ -511,9 +463,6 @@ class ImpulseConfig(BaseModel): Optional query engine configuration. Defaults to Solvers.DEFAULT_SOLVER. incremental : IncrementalConfig, optional Optional incremental processing configuration. Defaults to IncrementalConfig(). - calculated_channels : CalculatedChannels, optional - Optional calculated-channel output configuration (e.g. opting in to the - ``calculated_channel_metrics`` table). Defaults to CalculatedChannels(). measurement_dimensions : list of str, optional Column names to surface from ``container_metrics`` into the gold-layer ``measurement_dimension`` table. Names are matched @@ -588,7 +537,6 @@ class ImpulseConfig(BaseModel): container_filters: ContainerFilters | None = None query_engine: QueryEngine = QueryEngine(solver=Solvers.DEFAULT_SOLVER) incremental: IncrementalConfig | None = None - calculated_channels: CalculatedChannels = CalculatedChannels() measurement_dimensions: list[str] = list(DEFAULT_MEASUREMENT_DIMENSIONS) diff --git a/src/impulse_reporting/core/report.py b/src/impulse_reporting/core/report.py index 7eeba8a0..f0e9876d 100644 --- a/src/impulse_reporting/core/report.py +++ b/src/impulse_reporting/core/report.py @@ -25,17 +25,14 @@ cleanup_temp_tables, collect_solvable_expressions, dispatch_aggregations, - dispatch_calculated_channel_metrics, dispatch_calculated_channels, dispatch_events, group_selectables_by_type, merge_changed_unchanged, - persist_channel_metrics, persist_dimensions_full, persist_dimensions_incremental, persist_facts_full, persist_facts_incremental, - solve_calculated_channels_batched, solve_expressions_batched, split_by_hash_change, ) @@ -109,7 +106,6 @@ def __init__( self.aggregation_metadata_dfs = {} self.calculated_channel_dfs = {} self.calculated_channel_metadata_dfs = {} - self.calculated_channel_metrics_dfs = {} self.container_dimension_df = None self.channel_mapping_resolution_dimension_df = None self._is_incremental = None @@ -625,16 +621,6 @@ def _persist_full(self): persist_facts_full(self.calculated_channel_dfs, ChannelType, storage_factory) persist_dimensions_full(self.calculated_channel_metadata_dfs, ChannelType, storage_factory) - # optional calculated channel metrics table (dynamic schema — stored - # directly without the fixed-schema projecting writer) - persist_channel_metrics( - self.calculated_channel_metrics_dfs, - ChannelType, - self.sink, - ReportEntityTransformer(), - incremental=False, - ) - # persist measurement dimensions if self.container_dimension_df: writer = storage_factory.create_container_dimension_writer() @@ -745,17 +731,6 @@ def _transform(df, schema): merge_keys=["channel_id"], ) - # Optional calculated channel metrics table (dynamic schema — merged - # directly, scoping the delete-by-source to updated containers). - persist_channel_metrics( - self.calculated_channel_metrics_dfs, - ChannelType, - self.sink, - transformer, - incremental=True, - updated_container_ids=updated_container_ids, - ) - # Persist the measurement dimension LAST (as ``_persist_full`` does). It # holds the gold timestamp that container-update detection compares # against; the fact solves above are lazy, so writing it earlier would @@ -889,28 +864,6 @@ def _solve_expressions_batched( pre_filtered_containers_df=pre_filtered_containers_df, ) - def _solve_calculated_channels_batched( - self, - qe_channels: list, - pre_filtered_containers_df: DataFrame = None, - ) -> DataFrame | None: - """Solve calculated channels in configurable batches; return the unioned rows. - - Narrow counterpart to :meth:`_solve_expressions_batched`; delegates to - :func:`solve_calculated_channels_batched` in ``report_utils``. - """ - return solve_calculated_channels_batched( - spark=self.spark, - qe_channels=qe_channels, - query=self.query, - solver=self.solver, - batch_size=self.config.query_engine.batch_size, - has_sink=self._has_sink, - catalog=getattr(self.config, "unity_sink", None) and self.config.unity_sink.catalog, - schema=getattr(self.config, "unity_sink", None) and self.config.unity_sink.schema, - pre_filtered_containers_df=pre_filtered_containers_df, - ) - @telemetry_logger("report", "determine_report") def determine_report(self, is_incremental: bool = None): """ @@ -1054,10 +1007,9 @@ def determine_report(self, is_incremental: bool = None): aggs_by_type, AggregationType, self.spark ) - # Calculated channels: own narrow batched solve driven here (mirrors the - # wide expression solve above), producing a narrow ``solved_df`` that each - # channel type then shapes. Changed definitions recompute over all - # containers; unchanged ones over the incrementally-detected subset. + # Calculated channels: own narrow solve (not the wide solved_df). + # Changed definitions recompute over all containers; unchanged ones over + # the incrementally-detected subset. self._validate_unique_calculated_channels() channels_by_type = group_selectables_by_type(self.calculated_channels, ChannelType) changed_channels_by_type, unchanged_channels_by_type, self._changed_channel_ids = ( @@ -1070,31 +1022,21 @@ def determine_report(self, is_incremental: bool = None): kind="channel", ) ) - # Collect the query-engine channel expressions across types for the batched - # solve (mirrors collect_solvable_expressions for aggregations/events). - changed_channel_exprs = [ - c.expression for cs in changed_channels_by_type.values() for c in cs - ] - unchanged_channel_exprs = [ - c.expression for cs in unchanged_channels_by_type.values() for c in cs - ] - changed_channel_solved_df = self._solve_calculated_channels_batched( - changed_channel_exprs, pre_filtered_containers_df=None - ) - unchanged_channel_solved_df = self._solve_calculated_channels_batched( - unchanged_channel_exprs, pre_filtered_containers_df=pre_filtered_containers_df - ) changed_channel_dfs = dispatch_calculated_channels( self.spark, changed_channels_by_type, ChannelType, - changed_channel_solved_df, + self.query, + self.solver, + None, ) unchanged_channel_dfs = dispatch_calculated_channels( self.spark, unchanged_channels_by_type, ChannelType, - unchanged_channel_solved_df, + self.query, + self.solver, + pre_filtered_containers_df, ) self.calculated_channel_dfs = merge_changed_unchanged( changed_channel_dfs, unchanged_channel_dfs @@ -1103,37 +1045,6 @@ def determine_report(self, is_incremental: bool = None): channels_by_type, ChannelType, self.spark ) - # Optionally derive a silver-shaped channel_metrics table from the fact - # rows so the fact + metrics pair can serve as an Impulse silver source. - # Identity/attribute columns are derived dynamically; the full per-type - # channel list is passed to both buckets so changed/unchanged share a - # schema (extra channels are ignored by the fact-driven left join). - self.calculated_channel_metrics_dfs = {} - if self.config.calculated_channels.emit_channel_metrics: - cc = self.config.calculated_channels - # Pass the full per-type channel list to both buckets so changed and - # unchanged metrics share one schema (extra channels are dropped by the - # fact-driven left join in determine_channel_metrics). - changed_metrics = dispatch_calculated_channel_metrics( - self.spark, - channels_by_type, - changed_channel_dfs, - ChannelType, - attribute_columns=cc.attribute_columns, - kpis=cc.kpis, - ) - unchanged_metrics = dispatch_calculated_channel_metrics( - self.spark, - channels_by_type, - unchanged_channel_dfs, - ChannelType, - attribute_columns=cc.attribute_columns, - kpis=cc.kpis, - ) - self.calculated_channel_metrics_dfs = merge_changed_unchanged( - changed_metrics, unchanged_metrics - ) - # Determine container dimension self.container_dimension_df = ContainerDimension.get_dimension( spark=self.spark, diff --git a/src/impulse_reporting/core/report_utils.py b/src/impulse_reporting/core/report_utils.py index 795bdb5f..27337568 100644 --- a/src/impulse_reporting/core/report_utils.py +++ b/src/impulse_reporting/core/report_utils.py @@ -26,11 +26,7 @@ from impulse_reporting.incremental.definition_hash_comparator import ( DefinitionHashComparator, ) - from impulse_reporting.persist.report_storage import ( - ReportEntityTransformer, - Sink, - WriterFactory, - ) + from impulse_reporting.persist.report_storage import Sink, WriterFactory def build_batches( @@ -312,23 +308,25 @@ def dispatch_calculated_channels( spark: SparkSession, channels_by_type: dict[str, list], type_enum, - solved_df: DataFrame | None, + query: QueryBuilder, + solver: QuerySolver, + pre_filtered_containers_df: DataFrame | None, ) -> dict: """Dispatch ``determine_calculated_channels`` calls per type. - Mirrors :func:`dispatch_aggregations`: the batched narrow solve happens in the - ``Report`` (see ``Report._solve_calculated_channels_batched``), producing a - single ``solved_df`` of narrow calculated-channel rows; each type then shapes - its slice of that already-solved DataFrame. + Unlike events/aggregations, calculated channels never ride the wide + ``solved_df`` — each type drives its own narrow solve via + ``query.solve_calculated_channels`` (the ``ContainerEvent`` pattern), so this + always passes ``spark``/``query``/``solver``/``pre_filtered_containers_df``. Parameters ---------- spark : SparkSession channels_by_type : dict[str, list] type_enum : ChannelType enum - solved_df : DataFrame | None - The batched narrow solve output (``container_id, channel_id, tstart, tend, - value, identity``); ``None`` when there were no channels to solve. + query : QueryBuilder + solver : QuerySolver + pre_filtered_containers_df : DataFrame | None Returns ------- @@ -342,70 +340,14 @@ def dispatch_calculated_channels( continue cls = type_enum[type_name].value channel_dfs[type_name] = cls.determine_calculated_channels( - spark, channels, solved_df=solved_df - ) - - return channel_dfs - - -def dispatch_calculated_channel_metrics( - spark: SparkSession, - channels_by_type: dict[str, list], - fact_dfs_by_type: dict[str, DataFrame | None], - type_enum, - *, - attribute_columns: list[str], - kpis: list[str], -) -> dict: - """Dispatch ``determine_channel_metrics`` calls per type. - - Post-processing counterpart to :func:`dispatch_calculated_channels`: instead - of solving, it derives the silver-shaped ``channel_metrics`` DataFrame from the - already-solved fact df for each type (``fact_dfs_by_type`` as returned by - :func:`dispatch_calculated_channels`). - - Pass the **full** per-type channel list (not a changed/unchanged split): the - metrics aggregation is fact-driven (a left join from the fact-derived - aggregate), so channels absent from ``fact_df`` are dropped, while the identity - columns are the union across all channels — keeping the changed and unchanged - metrics dfs on an identical schema for the downstream ``unionByName`` MERGE. - - Parameters - ---------- - spark : SparkSession - channels_by_type : dict[str, list] - Full per-type channel list. - fact_dfs_by_type : dict[str, DataFrame | None] - Per-type calculated-channel fact df for this bucket (changed or unchanged). - type_enum : ChannelType enum - attribute_columns : list[str] - Attribute keys to surface as columns. - kpis : list[str] - KPI names to compute. - - Returns - ------- - dict - ``metrics_dfs`` keyed by type name (only types with a fact df). - """ - metrics_dfs: dict = {} - - for type_name, channels in channels_by_type.items(): - if not channels: - continue - fact_df = fact_dfs_by_type.get(type_name) - if fact_df is None: - continue - cls = type_enum[type_name].value - metrics_dfs[type_name] = cls.determine_channel_metrics( spark, channels, - fact_df, - attribute_columns=attribute_columns, - kpis=kpis, + query=query, + solver=solver, + pre_filtered_containers_df=pre_filtered_containers_df, ) - return metrics_dfs + return channel_dfs def solve_expressions_batched( @@ -490,92 +432,6 @@ def solve_expressions_batched( return result -def solve_calculated_channels_batched( - spark: SparkSession, - qe_channels: list, - query: QueryBuilder, - solver: QuerySolver, - batch_size: int, - *, - has_sink: bool = False, - catalog: str = None, - schema: str = None, - pre_filtered_containers_df: DataFrame = None, -) -> DataFrame | None: - """Solve calculated channels in configurable batches; return the unioned rows. - - The narrow, row-append counterpart to :func:`solve_expressions_batched`. Each - batch is solved independently via - ``query.select(*batch).solve_calculated_channels(...)`` and persisted as a - temporary Delta table (``__impulse_temp__``) when a sink is - configured, or a Spark temp view otherwise — the same convention (and shared - ``__impulse_temp_*`` prefix, so :func:`cleanup_temp_tables` covers it). - - Unlike ``solve_expressions_batched`` (wide, one row per container → batches - combined with a full-outer join on ``container_id``), calculated-channel output - is narrow (``container_id, channel_id, tstart, tend, value, identity``; many - rows per container, batches hold different ``channel_id``s), so batches are - combined with **``unionByName``** (row append). - - Parameters - ---------- - spark : SparkSession - Active Spark session. - qe_channels : list - Query-engine ``CalculatedChannel`` objects (each a ``TimeSeriesExpression`` - exposing ``get_selectors()``), i.e. ``[c.expression for c in channels]``. - query : QueryBuilder - Query builder instance. - solver : QuerySolver - Query solver implementing ``solve_calculated_channels``. - batch_size : int - Maximum number of unique selectors per batch (passed to ``build_batches``). - has_sink : bool - Whether a Unity Catalog sink is configured. - catalog : str, optional - Unity Catalog catalog name (required when *has_sink* is ``True``). - schema : str, optional - Unity Catalog schema name (required when *has_sink* is ``True``). - pre_filtered_containers_df : DataFrame, optional - Pre-filtered containers for incremental processing. - - Returns - ------- - DataFrame | None - The unioned narrow DataFrame (``container_id, channel_id, tstart, tend, - value, identity``), or ``None`` if *qe_channels* is empty. - """ - if not qe_channels: - return None - - run_id = uuid.uuid4().hex[:8] - batches = build_batches(qe_channels, batch_size) - - batch_names: list[str] = [] - for batch_idx, batch_channels in enumerate(batches): - batch_df = query.select(*batch_channels).solve_calculated_channels( - spark, solver, pre_filtered_containers_df - ) - - if has_sink: - table_name = f"__impulse_temp_{run_id}_{batch_idx}" - fq_name = f"`{catalog}`.`{schema}`.`{table_name}`" - batch_df.write.format("delta").mode("overwrite").saveAsTable(fq_name) - batch_names.append(fq_name) - else: - view_name = f"__impulse_temp_{run_id}_{batch_idx}" - batch_df.createOrReplaceTempView(view_name) - batch_names.append(view_name) - - dfs = [spark.table(name) for name in batch_names] - - result = dfs[0] - for df in dfs[1:]: - result = result.unionByName(df) - - return result - - def cleanup_temp_tables(spark: SparkSession, catalog: str, schema: str) -> None: """Drop leftover ``__impulse_temp_*`` Delta tables from previous runs. @@ -703,43 +559,6 @@ def _fact_dfs_for_table(dfs) -> list[DataFrame]: return [dfs] if dfs is not None else [] -def group_dfs_by_table( - dfs_by_type: dict, - table_name_getter: Callable[[str], str], -) -> dict[str, list[DataFrame]]: - """Group per-type DataFrames by output table name. - - Shared shaping step for the "write one table per output" persistence paths: - flattens each per-type value (a ``{"changed", "unchanged"}`` dict or a bare - DataFrame, via :func:`_fact_dfs_for_table`) and buckets the DataFrames by the - table name that ``table_name_getter`` returns for the type (so types sharing a - table land together). Types that contribute no DataFrame are skipped, so every - returned list is non-empty. Callers union each list themselves (e.g. via - ``ReportEntityTransformer.concat_dataframes`` / the entity writer). - - Parameters - ---------- - dfs_by_type : dict - ``{type_name: value}`` where value is a structured - ``{"changed", "unchanged"}`` dict or a bare DataFrame. - table_name_getter : Callable[[str], str] - Maps a type name to its output table name (e.g. - ``lambda t: ChannelType[t].get_metrics_table_name()``). - - Returns - ------- - dict[str, list[DataFrame]] - ``{table_name: [dfs]}`` for each table with at least one DataFrame. - """ - dfs_by_table: dict[str, list[DataFrame]] = {} - for type_name, dfs in dfs_by_type.items(): - table_dfs = _fact_dfs_for_table(dfs) - if not table_dfs: - continue - dfs_by_table.setdefault(table_name_getter(type_name), []).extend(table_dfs) - return dfs_by_table - - def persist_facts_full(dfs_by_type: dict, type_enum, writer_factory: WriterFactory) -> None: """Full-overwrite persist of fact DataFrames, grouped by output table. @@ -756,11 +575,14 @@ def persist_facts_full(dfs_by_type: dict, type_enum, writer_factory: WriterFacto writer_factory : WriterFactory Factory producing the entity writer. """ - dfs_by_table = group_dfs_by_table( - dfs_by_type, lambda type_name: type_enum[type_name].get_fact_table_name() - ) + dfs_by_table: dict[str, list[DataFrame]] = {} + for type_name, dfs in dfs_by_type.items(): + table_name = type_enum[type_name].get_fact_table_name() + dfs_by_table.setdefault(table_name, []).extend(_fact_dfs_for_table(dfs)) for table_name, dfs_list in dfs_by_table.items(): + if not dfs_list: + continue entity_type = type_enum.get_any_for_fact_table(table_name) writer = writer_factory.create_writer(entity_type) schema, uri = writer.extract_fact_schema_and_output_uri(entity_type) @@ -943,71 +765,3 @@ def persist_dimensions_incremental( writer = factory.create_writer(entity_type) schema, uri = writer.extract_metadata_schema_and_output_uri(entity_type) sink.upsert(transform_fn(metadata_df, schema), uri, merge_keys) - - -def persist_channel_metrics( - metrics_dfs_by_type: dict, - type_enum, - sink: Sink, - transformer: ReportEntityTransformer, - *, - incremental: bool, - updated_container_ids: list | None = None, -) -> None: - """Persist the optional calculated-channel metrics table(s). - - Unlike the fact/dimension writers, the metrics schema is **dynamic** - (identity/attribute/KPI columns vary per report), so this bypasses the - fixed-schema ``DefaultReportEntityWriter`` and writes the already-shaped - DataFrame directly (mirroring the ``container_dimension`` special-case). The - per-type dfs are grouped by output table via :func:`group_dfs_by_table` and - unioned with the same ``concat_dataframes`` the entity writer uses. - - Full mode overwrites; incremental mode upserts on ``(container_id, - channel_id)`` and prunes stale rows from updated containers via - ``merge_incremental``. - - Parameters - ---------- - metrics_dfs_by_type : dict - ``{type_name: value}`` where value is a structured - ``{"changed", "unchanged"}`` dict or a bare DataFrame. - type_enum : Enum - The channel type-enum (resolves the metrics table name/uri). - sink : Sink - Target sink exposing ``store`` / ``merge_incremental``. - transformer : ReportEntityTransformer - Supplies ``concat_dataframes`` (union) and ``add_meta_information``. - incremental : bool - Whether to merge (True) or overwrite (False). - updated_container_ids : list, optional - Ids of updated containers, scoping the incremental delete-by-source. - """ - if not metrics_dfs_by_type: - return - - updated_container_ids = updated_container_ids or [] - - dfs_by_table = group_dfs_by_table( - metrics_dfs_by_type, lambda type_name: type_enum[type_name].get_metrics_table_name() - ) - - for table_name, dfs_list in dfs_by_table.items(): - entity_type = type_enum.get_any_for_metrics_table(table_name) - # Resolve the metrics URI directly (no fixed-schema writer). - uri = sink.config.get_output_uri_channel_metrics_table(entity_type) - df_enriched = transformer.concat_dataframes(dfs_list).transform( - transformer.add_meta_information - ) - if incremental: - delete_conditions = [] - if updated_container_ids: - delete_conditions.append(F.col("target.container_id").isin(updated_container_ids)) - sink.merge_incremental( - df_enriched, - uri, - ["container_id", "channel_id"], - delete_conditions=delete_conditions, - ) - else: - sink.store(df_enriched, uri) diff --git a/src/impulse_reporting/persist/report_storage.py b/src/impulse_reporting/persist/report_storage.py index d09414b5..6e6c7169 100644 --- a/src/impulse_reporting/persist/report_storage.py +++ b/src/impulse_reporting/persist/report_storage.py @@ -86,23 +86,6 @@ def get_output_uri_channel_mapping_resolution_dimension_table(self) -> str: """ pass - @abstractmethod - def get_output_uri_channel_metrics_table(self, element: ChannelType) -> str: - """ - Get the output URI for the (optional) channel-metrics table. - - Parameters - ---------- - element : ChannelType - The channel type to get the URI for. - - Returns - ------- - str - The output URI for the channel-metrics table. - """ - pass - @dataclass() class UnitySinkConfig(SinkConfig): @@ -202,27 +185,6 @@ def get_output_uri_channel_mapping_resolution_dimension_table(self) -> str: uri = f"{self.catalog_name}.{self.schema_name}." "channel_mapping_resolution_dimension" return uri - def get_output_uri_channel_metrics_table(self, element: ChannelType) -> str: - """ - Get the output URI for the channel-metrics table in Unity Catalog format. - - Parameters - ---------- - element : ChannelType - The channel type to get the URI for. - - Returns - ------- - str - The Unity Catalog URI for the channel-metrics table. - """ - table_name = element.get_metrics_table_name() - if self.table_prefix: - uri = f"{self.catalog_name}.{self.schema_name}.{self.table_prefix}_{table_name}" - else: - uri = f"{self.catalog_name}.{self.schema_name}.{table_name}" - return uri - class Sink(ABC): """ diff --git a/tests/conftest.py b/tests/conftest.py index 0ea87313..d5713299 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,6 +47,7 @@ def basic_narrow_db(spark, mock_workspace_client) -> MeasurementDB: tables["container_metrics"] = spark.read.table("spark_catalog.silver.container_metrics") tables["channel_metrics"] = spark.read.table("spark_catalog.silver.channel_metrics") tables["channels"] = spark.read.table("spark_catalog.silver.channels") + tables["poi_channels"] = spark.read.table("spark_catalog.silver.poi_channels") cfg = MeasurementDBConfig.for_debug(tables) return MeasurementDB(cfg, ws=mock_workspace_client) @@ -96,6 +97,20 @@ def setup_narrow_db(spark): ), schema=S.CHANNELS_SCHEMA, ) + poi_channels = spark.createDataFrame( + pd.read_csv( + f"{base_path}/tests/unit/data/unit_test_csv/1_poi_channels.csv", + dtype={ + "container_id": np.int64, + "channel_id": np.int32, + "timestamp": np.longlong, + "value_double": np.float64, + "value_string": "object", + "dtype": "object", + }, + ), + schema=S.POI_CHANNELS_SCHEMA, + ) container_tags.write.format("delta").mode("overwrite").saveAsTable( "spark_catalog.silver_narrow_db.container_tags" @@ -112,6 +127,9 @@ def setup_narrow_db(spark): channels.write.format("delta").mode("overwrite").saveAsTable( "spark_catalog.silver_narrow_db.channels" ) + poi_channels.write.format("delta").mode("overwrite").saveAsTable( + "spark_catalog.silver_narrow_db.poi_channels" + ) @pytest.fixture(scope="session", autouse=True) @@ -131,11 +149,20 @@ def setup_basic_db(spark): container_metric_path = f"{base_path}/tests/unit/data/basic_narrow_csv/container_metrics.csv" channel_metric_path = f"{base_path}/tests/unit/data/basic_narrow_csv/channel_metrics.csv" channels_path = f"{base_path}/tests/unit/data/basic_narrow_csv/channel_data.csv" + poi_channels_path = f"{base_path}/tests/unit/data/basic_narrow_csv/poi_channels.csv" options = {"header": "True", "delimiter": ",", "inferSchema": "True"} container_metrics = spark.read.options(**options).csv(container_metric_path) channel_metrics = spark.read.options(**options).csv(channel_metric_path) channels = spark.read.options(**options).csv(channels_path) + # POI channel data: explicit schema so empty value columns keep their nullable + # typed shape (value_double double / value_string string) rather than being + # inferred as all-null strings. + poi_channels = ( + spark.read.schema(S.POI_CHANNELS_SCHEMA) + .options(header="True", delimiter=",") + .csv(poi_channels_path) + ) container_metrics.write.format("delta").mode("overwrite").saveAsTable( "spark_catalog.silver.container_metrics" @@ -150,6 +177,9 @@ def setup_basic_db(spark): "spark_catalog.silver.channel_metrics" ) channels.write.format("delta").mode("overwrite").saveAsTable("spark_catalog.silver.channels") + poi_channels.write.format("delta").mode("overwrite").saveAsTable( + "spark_catalog.silver.poi_channels" + ) @pytest.fixture(scope="session") @@ -236,6 +266,7 @@ def narrow_db(spark, setup_narrow_db, mock_workspace_client) -> MeasurementDB: "spark_catalog.silver_narrow_db.channel_metrics" ) debug_tables["channels"] = spark.read.table("spark_catalog.silver_narrow_db.channels") + debug_tables["poi_channels"] = spark.read.table("spark_catalog.silver_narrow_db.poi_channels") cfg = MeasurementDBConfig.for_debug(debug_tables) return MeasurementDB(cfg, ws=mock_workspace_client) diff --git a/tests/impulse_query_engine/integration/poi_channel_solve_test.py b/tests/impulse_query_engine/integration/poi_channel_solve_test.py new file mode 100644 index 00000000..8b032ffc --- /dev/null +++ b/tests/impulse_query_engine/integration/poi_channel_solve_test.py @@ -0,0 +1,243 @@ +"""End-to-end integration tests for Points-in-Time (POI) channels. + +Exercises the full solve pipeline for ``query.poi_channel(...)`` against the shared +``basic_narrow_db`` (wide) and ``narrow_db`` (EAV) fixtures, which carry POI channels +on ``container_id = 1`` alongside the existing sample channels (see conftest / +``poi_channels.csv``): + +- ``channel_id = 90`` — a **string** DTC-code channel (``P0301`` / ``P0420`` / ``P0301``) +- ``channel_id = 91`` — a **numeric** DTC-count channel (values ``1, 2, 3``) + +Covers: numeric POI unweighted reductions, string POI equality + op gating, the +mix-and-match case (a SAMPLE and a POI channel in one expression), the declared-vs-actual +dtype/series-type assertion, and SAMPLE backward-compatibility. +""" + +import math + +import pytest +import pyspark.sql.types as T +from pyspark.sql import SparkSession + +from impulse_query_engine.analyze.metadata.time_series_expression import PoiValueType +from impulse_query_engine.analyze.query.solvers.default_solver import DefaultSolver +from impulse_query_engine.measurement_db import MeasurementDB + + +class TestNumericPoi: + def test_numeric_poi_mean_is_unweighted(self, spark: SparkSession, basic_narrow_db): + """A numeric POI ``mean()`` is the plain (unweighted) mean of the point values — + POI points have no duration to weight by, unlike ``SampleSeries.mean()``.""" + solver = DefaultSolver(spark) + q = basic_narrow_db.query + dtc_count = q.poi_channel(channel_name="DTC_count") # values 1, 2, 3 + + result = q.select(dtc_count.mean().alias("m")).solve(spark=spark, solver=solver) + + rows = {r.container_id: r.m for r in result.collect()} + assert rows[1] == 2.0 # unweighted mean of (1, 2, 3) + + def test_numeric_poi_sum_and_count(self, spark: SparkSession, basic_narrow_db): + solver = DefaultSolver(spark) + q = basic_narrow_db.query + dtc_count = q.poi_channel(channel_name="DTC_count") + + result = q.select( + dtc_count.sum().alias("s"), + dtc_count.count().alias("c"), + ).solve(spark=spark, solver=solver) + + row = {r.container_id: r for r in result.collect()}[1] + assert row.s == 6.0 # 1 + 2 + 3 + assert row.c == 3 + + def test_bare_numeric_poi_selection_types_as_points_in_time( + self, spark: SparkSession, basic_narrow_db + ): + """A bare numeric POI selection serializes as ``array>`` + (PointsInTimeSeries), not the SAMPLE ``binary`` blob type.""" + solver = DefaultSolver(spark) + q = basic_narrow_db.query + dtc_count = q.poi_channel(channel_name="DTC_count").alias("pit") + + result = q.select(dtc_count).solve(spark=spark, solver=solver) + + assert result.schema["pit"].dataType == T.ArrayType(T.ArrayType(T.DoubleType())) + rows = {r.container_id: r.pit for r in result.collect()} + # three points [t, v], values 1..3 (unweighted, in timestamp order) + assert [pt[1] for pt in rows[1]] == [1.0, 2.0, 3.0] + + +class TestStringPoi: + def test_string_poi_equality_selects_matching_instants( + self, spark: SparkSession, basic_narrow_db + ): + """``string_poi == "P0301"`` yields the instants where the code equals P0301. + + Sampling the count channel at those instants (via ``.where``) picks out the two + P0301 occurrences, proving the string equality drove the point selection. + """ + solver = DefaultSolver(spark) + q = basic_narrow_db.query + dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + + # DTC == "P0301" is a PointsInTime; serialize it directly. + result = q.select((dtc == "P0301").alias("hits")).solve(spark=spark, solver=solver) + + rows = {r.container_id: r.hits for r in result.collect()} + # P0301 occurs at the 1st and 3rd of the three DTC timestamps. + assert len(rows[1]) == 2 + + def test_string_poi_count_and_sampling_allowed(self, spark: SparkSession, basic_narrow_db): + solver = DefaultSolver(spark) + q = basic_narrow_db.query + dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + + result = q.select(dtc.count().alias("c")).solve(spark=spark, solver=solver) + + assert {r.container_id: r.c for r in result.collect()}[1] == 3 + + @pytest.mark.parametrize("reduction", ["mean", "sum", "min", "max"]) + def test_string_poi_numeric_reduction_rejected_at_build( + self, spark: SparkSession, basic_narrow_db, reduction + ): + """A numeric reduction on a string POI selection is rejected at plan/build time + (before Spark runs), not as a silent NaN.""" + q = basic_narrow_db.query + dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + selection = getattr(dtc, reduction)().alias("bad") + with pytest.raises(TypeError, match="string-valued"): + q.select(selection)._determine_result_objects_dtypes() + + +class TestMixAndMatch: + """The primary correctness case: a SAMPLE and a POI channel in one expression, both + in the same per-container pandas frame, aligned via ``synchronized``.""" + + def test_sample_channel_sampled_at_poi_instants(self, spark: SparkSession, narrow_db): + """Sample the ``seed`` SAMPLE channel at the instants of the numeric POI channel. + + narrow_db container 1: ``seed`` sample channel has values 1..10 over t=0..10; the + numeric POI channel (91) has points at t = 2, 5, 8. Sampling seed at those instants + picks the seed values valid there. + """ + solver = DefaultSolver(spark) + q = narrow_db.query + seed = q.channel(seed="0") + dtc_count = q.poi_channel(channel_name="DTC_count") + + # Sample the sample-series at the POI points (cross-type synchronize). + result = q.select(seed.where(dtc_count.to_points_in_time()).alias("sampled")).solve( + spark=spark, solver=solver + ) + + rows = {r.container_id: r.sampled for r in result.collect()} + # three sampled points at the POI instants t = 2, 5, 8 + assert [pt[0] for pt in rows[1]] == [2.0, 5.0, 8.0] + for pt in rows[1]: + assert not math.isnan(pt[1]) + + def test_string_poi_and_sample_freeze_frame(self, spark: SparkSession, narrow_db): + """Freeze-frame: sample the seed channel at the instants where DTC == "P0301".""" + solver = DefaultSolver(spark) + q = narrow_db.query + seed = q.channel(seed="0") + dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + + result = q.select(seed.where(dtc == "P0301").alias("frozen")).solve( + spark=spark, solver=solver + ) + + rows = {r.container_id: r.frozen for r in result.collect()} + # P0301 at t = 2 and 8 (EAV fixture); seed sampled there. + assert [pt[0] for pt in rows[1]] == [2.0, 8.0] + + def test_sample_channel_sampled_at_poi_instants_wide( + self, spark: SparkSession, basic_narrow_db + ): + """Wide-mode counterpart of the mix-and-match case (``basic_narrow_db``). + + Uses a POI numeric channel to filter/sample a real sample channel. Only + "Ambient Air Temperature" (channel 6) spans all three POI instants in the + ``basic_narrow_csv`` fixture (the other channels end earlier), so it is the + channel sampled here. Proves POI-drives-channel-selection works through the + wide (columns-on-channel_metrics) path, not just the EAV pivot path. + """ + solver = DefaultSolver(spark) + q = basic_narrow_db.query + amb = q.channel(channel_name="Ambient Air Temperature") + dtc_count = q.poi_channel(channel_name="DTC_count") # points at 3 POI instants + + result = q.select( + amb.where(dtc_count.to_points_in_time()).alias("sampled") + ).solve(spark=spark, solver=solver) + + rows = {r.container_id: r.sampled for r in result.collect()} + # The three POI instants (microsecond epochs) from basic_narrow_csv/poi_channels.csv. + poi_instants = [1499929300000000.0, 1499931000000000.0, 1499933000000000.0] + assert [pt[0] for pt in rows[1]] == poi_instants + # Each instant sampled a real Ambient-Air-Temp value (not a miss / NaN). + for pt in rows[1]: + assert not math.isnan(pt[1]) + + def test_string_poi_freeze_frame_wide(self, spark: SparkSession, basic_narrow_db): + """Wide-mode freeze-frame: sample "Ambient Air Temperature" where DTC == "P0301". + + In ``basic_narrow_csv`` P0301 occurs at the 1st and 3rd DTC instants, so the + string-POI equality predicate selects exactly those two instants of the + sample channel — the freeze-frame case resolved via the wide channel path. + """ + solver = DefaultSolver(spark) + q = basic_narrow_db.query + amb = q.channel(channel_name="Ambient Air Temperature") + dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + + result = q.select(amb.where(dtc == "P0301").alias("frozen")).solve( + spark=spark, solver=solver + ) + + rows = {r.container_id: r.frozen for r in result.collect()} + # P0301 at the 1st and 3rd instants (basic_narrow_csv/poi_channels.csv). + assert [pt[0] for pt in rows[1]] == [1499929300000000.0, 1499933000000000.0] + for pt in rows[1]: + assert not math.isnan(pt[1]) + + +class TestDeclaredVsActual: + def test_poi_channel_declared_double_on_string_channel_raises( + self, spark: SparkSession, basic_narrow_db + ): + """Declaring ``dtype=double`` on a channel whose silver dtype is ``string`` raises + at solve time — the data stays authoritative.""" + solver = DefaultSolver(spark) + q = basic_narrow_db.query + # DTC is a numeric-less (string) channel; declaring double resolves rows + # whose value_double is all null → dtype mismatch raised in the solve UDF. + bad = q.poi_channel(channel_name="DTC", dtype=PoiValueType.DOUBLE) + with pytest.raises(Exception, match="dtype mismatch"): + q.select(bad.count().alias("c")).solve(spark=spark, solver=solver).collect() + + def test_poi_channel_on_sample_channel_raises(self, spark: SparkSession, basic_narrow_db): + """``poi_channel`` on a SAMPLE channel raises the series-type mismatch. + + The SAMPLE channel's rows carry a real (non-null) validity interval, which + is the signal load_blob validates a POI-declared selector against. + """ + solver = DefaultSolver(spark) + q = basic_narrow_db.query + bad = q.poi_channel(channel_name="Engine RPM") + with pytest.raises(Exception, match="series-type mismatch"): + q.select(bad.count().alias("c")).solve(spark=spark, solver=solver).collect() + + +class TestBackwardCompat: + def test_sample_channel_unaffected_by_poi(self, spark: SparkSession, basic_narrow_db): + """An ordinary SAMPLE ``channel(...)`` selection is unchanged by POI support.""" + solver = DefaultSolver(spark) + q = basic_narrow_db.query + rpm = q.channel(channel_name="Engine RPM") + + result = q.select(rpm.mean().alias("rpm_mean")).solve(spark=spark, solver=solver) + + rows = {r.container_id for r in result.collect()} + assert rows == {1, 2, 3} diff --git a/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py b/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py index 008b8117..ef361f92 100644 --- a/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py @@ -4,6 +4,8 @@ from impulse_query_engine.analyze.metadata.tag_expression import TagSelector from impulse_query_engine.analyze.metadata.time_series_expression import ( + PoiValueType, + SeriesType, TimeSeriesSelector, ) from impulse_query_engine.model.series import Intervals @@ -195,3 +197,56 @@ def test_timeseries_selector_dtype_matches_sample_series_dtype(): ts = TimeSeriesSelector(TagSelector("name") == "test") ss = SampleSeries.empty() assert ts.dtype() == ss.dtype() + + +# --------------------------------------------------------------------------- +# QueryBuilder.poi_channel — dtype accepts the enum OR its string value +# --------------------------------------------------------------------------- +class TestPoiChannelDtypeArg: + """``poi_channel(dtype=...)`` must accept both ``PoiValueType`` and the plain + string value (``"double"`` / ``"string"``). A regression guard: a plain + ``dtype="string"`` used to be stored verbatim (a ``str``, not the enum), so the + ``is PoiValueType.STRING`` identity checks silently fell through and a string + POI channel behaved as numeric — blowing up on the first string comparison. + """ + + def test_default_dtype_is_double(self, narrow_db): + sel = narrow_db.query.poi_channel(channel_name="DTC_count") + assert sel.series_type is SeriesType.POINTS_IN_TIME + assert sel.value_type is PoiValueType.DOUBLE + + def test_enum_string_dtype(self, narrow_db): + sel = narrow_db.query.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + assert sel.value_type is PoiValueType.STRING + + def test_plain_string_dtype_coerced_to_enum(self, narrow_db): + # the design-doc form: poi_channel(..., dtype="string") + sel = narrow_db.query.poi_channel(channel_name="DTC", dtype="string") + assert sel.value_type is PoiValueType.STRING + + def test_plain_string_double_dtype_coerced_to_enum(self, narrow_db): + sel = narrow_db.query.poi_channel(channel_name="DTC_count", dtype="double") + assert sel.value_type is PoiValueType.DOUBLE + + def test_invalid_dtype_raises(self, narrow_db): + with pytest.raises(ValueError): + narrow_db.query.poi_channel(channel_name="DTC", dtype="int") + + def test_string_poi_types_as_struct_regardless_of_arg_form(self, narrow_db): + # both arg forms must produce an identical string-typed result dtype + enum_sel = narrow_db.query.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + str_sel = narrow_db.query.poi_channel(channel_name="DTC", dtype="string") + assert enum_sel.dtype() == str_sel.dtype() + # string POI serializes as array>, not array> + assert isinstance(str_sel.dtype(), T.ArrayType) + assert isinstance(str_sel.dtype().elementType, T.StructType) + + def test_string_poi_equality_evaluates_to_points_in_time(self, narrow_db): + # dtype="string" must yield a string series so `== "code"` works at plan time + dtc = narrow_db.query.poi_channel(channel_name="DTC", dtype="string") + assert (dtc == "P0301").evaluation_type() is PointsInTime + + def test_string_poi_mean_rejected_at_build_time(self, narrow_db): + dtc = narrow_db.query.poi_channel(channel_name="DTC", dtype="string") + with pytest.raises(TypeError, match="string-valued"): + dtc.mean().evaluation_type() diff --git a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_column_mapping_test.py b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_column_mapping_test.py index 0e53a802..b236f0e5 100644 --- a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_column_mapping_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_column_mapping_test.py @@ -611,6 +611,7 @@ def test_col_map_always_returns_internal_names(self, spark): "te": "tend", "val": "value", "conv": "conversion_factor", + "value_string": "value_string", } def test_mapping_entries_stored_correctly(self, spark): diff --git a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_only_test.py b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_only_test.py index 9ab2b68f..b74b1cd2 100644 --- a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_only_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_only_test.py @@ -505,6 +505,7 @@ def test_col_map_always_returns_internal_names(self, spark: SparkSession): "te": "tend", "val": "value", "conv": "conversion_factor", + "value_string": "value_string", } def test_config_properties_return_internal_names(self, spark: SparkSession): diff --git a/tests/impulse_query_engine/unit/analyze/query/solvers/solver_config_test.py b/tests/impulse_query_engine/unit/analyze/query/solvers/solver_config_test.py index c692c2ee..524b97db 100644 --- a/tests/impulse_query_engine/unit/analyze/query/solvers/solver_config_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/solvers/solver_config_test.py @@ -38,6 +38,7 @@ "te": "tend", "val": "value", "conv": "conversion_factor", + "value_string": "value_string", } @@ -148,7 +149,15 @@ class TestColMap: def test_col_map_keys(self, cfg: SolverConfig): """col_map should contain exactly the expected short keys.""" - assert set(cfg.col_map.keys()) == {"cid", "ch", "ts", "te", "val", "conv"} + assert set(cfg.col_map.keys()) == { + "cid", + "ch", + "ts", + "te", + "val", + "conv", + "value_string", + } def test_col_map_default_config(self): """Default SolverConfig col_map should match hardcoded defaults.""" @@ -160,6 +169,7 @@ def test_col_map_default_config(self): "te": "tend", "val": "value", "conv": "conversion_factor", + "value_string": "value_string", } def test_col_map_consistent_with_properties(self, cfg: SolverConfig): diff --git a/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py b/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py index b957d965..77f8662b 100644 --- a/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py +++ b/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py @@ -152,6 +152,117 @@ def test_aggregations_empty(): assert pts.count() == 0 +# --- string values ------------------------------------------------------------------------------ +# String-valued series support sampling and equality only; arithmetic, ordering +# and numeric reductions are rejected. Timestamps stay numeric regardless. + + +def test_string_values_stored_as_object_with_numeric_timestamps(): + pts = PointsInTimeSeries([1, 2, 3], ["P108B", "U0046", "P108B"]) + assert pts._is_string is True + assert pts.values.dtype == object + assert pts.tstarts.dtype == np.float64 + nptest.assert_array_equal(pts.values, ["P108B", "U0046", "P108B"]) + + +def test_empty_series_defaults_to_numeric(): + # No observed value type -> numeric (backward-compatible default). + assert PointsInTimeSeries.empty()._is_string is False + + +def test_numeric_series_is_not_string(): + assert PointsInTimeSeries([0, 1], [10, 20])._is_string is False + + +def test_string_eq_scalar_returns_points_in_time(): + pts = PointsInTimeSeries([1, 2, 3], ["P108B", "U0046", "P108B"]) + result = pts == "P108B" + assert isinstance(result, PointsInTime) + nptest.assert_array_equal(result.tstarts, [1, 3]) + + +def test_string_ne_scalar_returns_points_in_time(): + pts = PointsInTimeSeries([1, 2, 3], ["P108B", "U0046", "P108B"]) + nptest.assert_array_equal((pts != "P108B").tstarts, [2]) + + +def test_string_eq_series_matches_on_value_and_timestamp(): + p1 = PointsInTimeSeries([1, 2, 3], ["A", "B", "C"]) + p2 = PointsInTimeSeries([2, 3, 4], ["X", "C", "C"]) + # Common timestamps {2,3}; values equal only at t=3 ("C" == "C"). + nptest.assert_array_equal((p1 == p2).tstarts, [3]) + + +def test_string_synchronized_with_sample_series_samples_values(): + pts = PointsInTimeSeries([5, 15, 25], ["a", "b", "c"]) + s = SampleSeries([0, 10, 20], [10, 20, 30], [1, 2, 3]) + a, b = pts.synchronized(s) + nptest.assert_array_equal(a.tstarts, [5, 15, 25]) + nptest.assert_array_equal(a.values, ["a", "b", "c"]) + nptest.assert_array_equal(b.values, [1, 2, 3]) + + +def test_string_get_data_pairs_double_timestamp_with_string_value(): + pts = PointsInTimeSeries([1, 2], ["P108B", "U0046"]) + assert pts.get_data() == [[1.0, "P108B"], [2.0, "U0046"]] + + +def test_string_dtype_is_struct_of_double_and_string(): + pts = PointsInTimeSeries([1, 2], ["P108B", "U0046"]) + assert pts.dtype() == T.ArrayType( + T.StructType( + [ + T.StructField("tstart", T.DoubleType()), + T.StructField("value", T.StringType()), + ] + ) + ) + + +@pytest.mark.parametrize( + "op", + [ + lambda p: p + "x", + lambda p: "x" + p, + lambda p: p - 1, + lambda p: 1 - p, + lambda p: p * 2, + lambda p: p / 2, + ], +) +def test_string_arithmetic_raises(op): + pts = PointsInTimeSeries([1, 2], ["A", "B"]) + with pytest.raises(TypeError, match="string-valued"): + op(pts) + + +@pytest.mark.parametrize( + "op", + [ + lambda p: p > "A", + lambda p: p >= "A", + lambda p: p < "Z", + lambda p: p <= "Z", + ], +) +def test_string_ordering_comparison_raises(op): + pts = PointsInTimeSeries([1, 2], ["A", "B"]) + with pytest.raises(TypeError, match="string-valued"): + op(pts) + + +@pytest.mark.parametrize("reduction", ["sum", "mean", "min", "max"]) +def test_string_reductions_raise(reduction): + pts = PointsInTimeSeries([1, 2], ["A", "B"]) + with pytest.raises(TypeError, match="string-valued"): + getattr(pts, reduction)() + + +def test_string_count_is_allowed(): + # count is structural (not value-dependent), so it works for strings. + assert PointsInTimeSeries([1, 2, 3], ["A", "B", "C"]).count() == 3 + + # --- plane_sweep -------------------------------------------------------------------------------- diff --git a/tests/impulse_reporting/integration/calculated_channel_test.py b/tests/impulse_reporting/integration/calculated_channel_test.py index 4b4d1457..1f57b205 100644 --- a/tests/impulse_reporting/integration/calculated_channel_test.py +++ b/tests/impulse_reporting/integration/calculated_channel_test.py @@ -14,10 +14,8 @@ import pytest from databricks.sdk import WorkspaceClient -from impulse_query_engine.measurement_db import MeasurementDB, MeasurementDBConfig from impulse_reporting.channels.calculated_channel import CalculatedChannel from impulse_reporting.config.config_parser import ( - CalculatedChannels, DataType, ImpulseConfig, IncrementalConfig, @@ -33,13 +31,10 @@ _FACT = "spark_catalog.gold.evaluation_calculated_channel_fact" _DIM = "spark_catalog.gold.evaluation_calculated_channel_dimension" -_METRICS = "spark_catalog.gold.evaluation_calculated_channel_metrics" -def _config( - silver_table="container_metrics", is_enabled=False, calculated_channels=None, batch_size=None -): - kwargs = dict( +def _config(silver_table="container_metrics", is_enabled=False): + return ImpulseConfig( source=Source( container_metrics_table=f"spark_catalog.silver.{silver_table}", channel_metrics_table="spark_catalog.silver.channel_metrics", @@ -56,11 +51,6 @@ def _config( gold_last_modified_column="_created_at", ), ) - if calculated_channels is not None: - kwargs["calculated_channels"] = calculated_channels - if batch_size is not None: - kwargs["query_engine"] = QueryEngine(batch_size=batch_size) - return ImpulseConfig(**kwargs) def _add_channel(report, factor=3.6, name="speed_kmh", identity=None): @@ -153,48 +143,6 @@ def test_persist_calculated_channel_full(spark): assert dim_row[0]["definition_hash"] is not None -def test_batched_calculated_channels_union(spark): - """batch_size=1 with two channels on DISTINCT input selectors splits them into - separate batches (each persisted as a temp table); the final fact is the union. - """ - report = Report( - name="calc_channel_report", - spark=spark, - workspace_client=create_autospec(WorkspaceClient), - config=dict(_config(is_enabled=False, batch_size=1)), - ) - q = report.get_db().query - ch_speed = CalculatedChannel( - name="speed_kmh", - expr=q.channel(channel_name="Vehicle Speed Sensor") * 3.6, - identity={"channel_name": "speed_kmh", "data_key": "CALC"}, - ) - ch_rpm = CalculatedChannel( - name="rpm_x2", - expr=q.channel(channel_name="Engine RPM") * 2, - identity={"channel_name": "rpm_x2", "data_key": "CALC"}, - ) - report.add_calculated_channel(ch_speed) - report.add_calculated_channel(ch_rpm) - - report.determine_report() - report.persist_results() - - # Union across batches is complete: both channels' rows land in the one fact. - fact = spark.read.table(_FACT) - ids = {r["channel_id"] for r in fact.select("channel_id").distinct().collect()} - assert ids == {ch_speed.get_id(), ch_rpm.get_id()} - assert fact.filter(F.col("channel_id") == ch_speed.get_id()).count() > 0 - assert fact.filter(F.col("channel_id") == ch_rpm.get_id()).count() > 0 - - # Each distinct-selector channel was solved in its own batch → two temp tables - # persisted (cleanup_temp_tables defaults to False, so they remain after the run). - temp_tables = spark.sql( - "SHOW TABLES IN `spark_catalog`.`gold` LIKE '__impulse_temp_*'" - ).count() - assert temp_tables >= 2 - - def test_incremental_unchanged_is_idempotent(spark): # Seed gold (full), then re-run incrementally with the same definition/data. r1 = Report( @@ -312,16 +260,12 @@ def test_incremental_modified_container_fewer_rows_deletes_stale(spark): def test_incremental_changed_definition_replaces(spark): - # Metrics enabled so this also covers a changed-definition refresh of the - # metrics table (no extra Report runs / no added suite time). - cc = CalculatedChannels(emit_channel_metrics=True) - # Seed gold with factor 3.6. r1 = Report( name="calc_channel_report", spark=spark, workspace_client=create_autospec(WorkspaceClient), - config=dict(_config(is_enabled=False, calculated_channels=cc)), + config=dict(_config(is_enabled=False)), ) ch1 = _add_channel(r1, factor=3.6) r1.determine_report() @@ -332,12 +276,6 @@ def test_incremental_changed_definition_replaces(spark): .select("definition_hash") .first()[0] ) - # Metric mean under the 3.6 definition (container 1), captured before the change. - mean_before = ( - spark.read.table(_METRICS) - .filter((F.col("channel_id") == ch1.get_id()) & (F.col("container_id") == 1)) - .first()["mean"] - ) # Re-run incrementally with a changed factor → the changed-definition path # rewrites the rows in the unified MERGE (scoped by channel_id). @@ -345,7 +283,7 @@ def test_incremental_changed_definition_replaces(spark): name="calc_channel_report", spark=spark, workspace_client=create_autospec(WorkspaceClient), - config=dict(_config(is_enabled=True, calculated_channels=cc)), + config=dict(_config(is_enabled=True)), ) ch2 = _add_channel(r2, factor=3.7) assert ch2.get_id() == ch1.get_id() # same identity → same entity id @@ -360,16 +298,6 @@ def test_incremental_changed_definition_replaces(spark): # A single dimension row per channel_id (upsert, not append). assert dim.filter(F.col("channel_id") == ch2.get_id()).count() == 1 - # The metrics row was recomputed for the changed definition: the signal is - # `Vehicle Speed Sensor * factor`, so its duration-weighted mean scales - # linearly with the factor (3.6 → 3.7). - mean_after = ( - spark.read.table(_METRICS) - .filter((F.col("channel_id") == ch2.get_id()) & (F.col("container_id") == 1)) - .first()["mean"] - ) - assert mean_after == pytest.approx(mean_before * 3.7 / 3.6) - def test_incremental_identity_reorder_does_not_reprocess(spark): # Seed gold with identity keys in one order. @@ -443,186 +371,12 @@ def test_calculated_channel_raw_mode(spark, setup_raw_channels_db): ) report.add_calculated_channel(ch) - # This is the branch that regressed: RAW-mode batched solve must not raise. - # Exercise the new path: Report-driven batched solve → shaping. - solved_df = report._solve_calculated_channels_batched([ch.expression]) - df = CalculatedChannel.determine_calculated_channels(spark, [ch], solved_df=solved_df) + # This is the branch that regressed: RAW-mode solve must not raise. + df = CalculatedChannel.determine_calculated_channels( + spark, [ch], query=q, solver=report.get_solver() + ) assert df.count() > 0 # Identity is dimension-only; the fact projection carries just the silver columns. assert "identity" not in df.columns ids = {r["channel_id"] for r in df.select("channel_id").distinct().collect()} assert ids == {ch.get_id()} - - -def test_channel_metrics_not_emitted_by_default(spark): - # Flag off (default) → no calculated_channel_metrics table is written even - # when channels carry attributes. - report = Report( - name="calc_channel_report", - spark=spark, - workspace_client=create_autospec(WorkspaceClient), - config=dict(_config(is_enabled=False)), - ) - _add_channel(report, factor=3.6) - report.determine_report() - report.persist_results() - - assert spark.catalog.tableExists(_FACT) - assert not spark.catalog.tableExists(_METRICS) - - -def test_channel_metrics_emitted_and_usable_as_impulse_source(spark): - # Opt in to the metrics table, with `unit` surfaced as an attribute column. - report = Report( - name="calc_channel_report", - spark=spark, - workspace_client=create_autospec(WorkspaceClient), - config=dict( - _config( - is_enabled=False, - calculated_channels=CalculatedChannels( - emit_channel_metrics=True, attribute_columns=["unit"] - ), - ) - ), - ) - q = report.get_db().query - ch = CalculatedChannel( - name="speed_kmh", - expr=q.channel(channel_name="Vehicle Speed Sensor") * 3.6, - identity={"channel_name": "speed_kmh", "data_key": "CALC"}, - attributes={"unit": "km/h"}, - ) - report.add_calculated_channel(ch) - - report.determine_report() - report.persist_results() - - assert spark.catalog.tableExists(_METRICS) - metrics = spark.read.table(_METRICS) - # Dynamic identity columns (channel_name, data_key) + configured attribute (unit) - # + fixed metric columns; identity keys always present, attribute opt-in. - for col in [ - "container_id", - "channel_id", - "channel_name", - "data_key", - "unit", - "value_type", - "duration", - "min", - "max", - "mean", - ]: - assert col in metrics.columns, col - - row = metrics.filter(F.col("channel_id") == ch.get_id()).first() - assert row["value_type"] == "double" - assert row["channel_name"] == "speed_kmh" - assert row["data_key"] == "CALC" - assert row["unit"] == "km/h" - # One metrics row per (container, channel). - fact = spark.read.table(_FACT) - n_pairs = fact.select("container_id", "channel_id").distinct().count() - assert metrics.count() == n_pairs - - # Numeric KPIs match a hand-computed, duration-weighted aggregate of the gold - # fact rows (container 1). Independently reproduces the production semantics so - # a regression in the KPI math is caught end to end. - fact_rows = ( - fact.filter((F.col("channel_id") == ch.get_id()) & (F.col("container_id") == 1)) - .select("tstart", "tend", "value") - .collect() - ) - durations = [r["tend"] - r["tstart"] for r in fact_rows] - values = [r["value"] for r in fact_rows] - exp_duration = max(r["tend"] for r in fact_rows) - min(r["tstart"] for r in fact_rows) - exp_mean = sum(v * d for v, d in zip(values, durations, strict=True)) / sum(durations) - m_row = metrics.filter( - (F.col("channel_id") == ch.get_id()) & (F.col("container_id") == 1) - ).first() - assert m_row["duration"] == exp_duration - assert m_row["min"] == pytest.approx(min(values)) - assert m_row["max"] == pytest.approx(max(values)) - assert m_row["mean"] == pytest.approx(exp_mean) - - # Round-trip: the fact + metrics pair is a valid Impulse silver source. Feed - # them back as `channels` + `channel_metrics` (wide model: channel_name is a - # column on channel_metrics) and resolve the channel by name. A minimal - # `container_metrics` (one row per container) satisfies the filter pipeline. - fact_as_channels = spark.read.table(_FACT).select( - "container_id", "channel_id", "tstart", "tend", "value" - ) - container_metrics = fact_as_channels.select("container_id").distinct() - db = MeasurementDB( - MeasurementDBConfig.for_debug( - { - "channels": fact_as_channels, - "channel_metrics": metrics, - "container_metrics": container_metrics, - } - ), - ws=report.ws, - ) - solved = ( - db.query.channel(channel_name="speed_kmh").alias("v").solve(spark, report.get_solver()) - ) - assert solved.count() > 0 - - -def test_channel_metrics_incremental_is_idempotent(spark): - # Seed gold (full) with the metrics table, then re-run incrementally with the - # same definition/data: the metrics upsert on (container_id, channel_id) must - # keep the row count stable (one row per container/channel). - cc = CalculatedChannels(emit_channel_metrics=True) - - r1 = Report( - name="calc_channel_report", - spark=spark, - workspace_client=create_autospec(WorkspaceClient), - config=dict(_config(is_enabled=False, calculated_channels=cc)), - ) - _add_channel(r1, factor=3.6) - r1.determine_report() - r1.persist_results() - count_before = spark.read.table(_METRICS).count() - assert count_before > 0 - - r2 = Report( - name="calc_channel_report", - spark=spark, - workspace_client=create_autospec(WorkspaceClient), - config=dict(_config(is_enabled=True, calculated_channels=cc)), - ) - _add_channel(r2, factor=3.6) - r2.determine_report() - r2.persist_results() - - assert spark.read.table(_METRICS).count() == count_before - - -def test_channel_metrics_custom_kpis(spark): - # A non-default KPI selection controls which KPI columns are emitted. - report = Report( - name="calc_channel_report", - spark=spark, - workspace_client=create_autospec(WorkspaceClient), - config=dict( - _config( - is_enabled=False, - calculated_channels=CalculatedChannels( - emit_channel_metrics=True, kpis=["min", "max"] - ), - ) - ), - ) - _add_channel(report, factor=3.6) - report.determine_report() - report.persist_results() - - metrics = spark.read.table(_METRICS) - # Only the selected KPIs are present; the dropped defaults are absent. - assert "min" in metrics.columns - assert "max" in metrics.columns - assert "mean" not in metrics.columns - assert "duration" not in metrics.columns diff --git a/tests/impulse_reporting/unit/channels/calculated_channel_test.py b/tests/impulse_reporting/unit/channels/calculated_channel_test.py index 4a933f89..8d5856e3 100644 --- a/tests/impulse_reporting/unit/channels/calculated_channel_test.py +++ b/tests/impulse_reporting/unit/channels/calculated_channel_test.py @@ -2,11 +2,11 @@ """Unit tests for the reporting-layer CalculatedChannel class.""" import pytest -import pyspark.sql.types as T from impulse_query_engine.analyze.metadata.time_series_expression import TimeSeriesSelector +from impulse_query_engine.analyze.query.solvers.default_solver import DefaultSolver from impulse_reporting.channels.calculated_channel import CalculatedChannel -from tests.conftest import spark # noqa: F401 (pytest fixtures) +from tests.conftest import basic_narrow_db, spark # noqa: F401 (pytest fixtures) _IDENTITY = {"channel_name": "speed_kmh", "data_key": "CALC"} @@ -137,239 +137,30 @@ def test_definition_hash_changes_with_identity_value(self): assert a.determine_definition_hash() != b.determine_definition_hash() -_SOLVED_SCHEMA = T.StructType( - [ - T.StructField("container_id", T.LongType()), - T.StructField("channel_id", T.LongType()), - T.StructField("tstart", T.LongType()), - T.StructField("tend", T.LongType()), - T.StructField("value", T.DoubleType()), - T.StructField("identity", T.MapType(T.StringType(), T.StringType())), - ] -) - - class TestDetermineCalculatedChannels: - """The solve now happens in Report; determine_calculated_channels only shapes - the already-solved narrow DataFrame (filter by channel_id + project).""" - - def test_returns_none_when_no_channels(self, spark): - assert CalculatedChannel.determine_calculated_channels(spark, [], solved_df=None) is None - - def test_returns_none_when_no_solved_df(self, spark): - ch = _channel() - assert CalculatedChannel.determine_calculated_channels(spark, [ch], solved_df=None) is None - - def test_filters_by_channel_id_and_projects_to_fact_schema(self, spark): - ch = CalculatedChannel( - name="rpm_x2", - expr=TimeSeriesSelector(None) * 2, - identity={"channel_name": "rpm_x2", "data_key": "CALC"}, - ) - other_id = (ch.get_id() + 1) & 0x7FFFFFFF - # solved_df carries this channel's rows AND an unrelated channel's, plus the - # identity map column that shaping must drop. - solved = spark.createDataFrame( - [ - (1, ch.get_id(), 0, 1, 10.0, {"channel_name": "rpm_x2"}), - (1, other_id, 0, 1, 99.0, {"channel_name": "other"}), - ], - schema=_SOLVED_SCHEMA, - ) - df = CalculatedChannel.determine_calculated_channels(spark, [ch], solved_df=solved) - # Identity dropped; only the fact columns remain. - assert df.columns == ["container_id", "channel_id", "tstart", "tend", "value"] - # Unrelated channel's rows are filtered out. - ids = {r["channel_id"] for r in df.collect()} - assert ids == {ch.get_id()} - - -_FACT_SCHEMA = T.StructType( - [ - T.StructField("container_id", T.LongType(), False), - T.StructField("channel_id", T.LongType(), False), - T.StructField("tstart", T.LongType(), False), - T.StructField("tend", T.LongType(), False), - T.StructField("value", T.DoubleType(), True), - ] -) - - -class TestDetermineChannelMetrics: - def test_returns_none_when_fact_none(self, spark): + def test_returns_none_when_empty(self, spark): assert ( - CalculatedChannel.determine_channel_metrics(spark, [], None, attribute_columns=[]) + CalculatedChannel.determine_calculated_channels(spark, [], query=None, solver=None) is None ) - def test_duration_weighted_values(self, spark): - ch = CalculatedChannel("a", TimeSeriesSelector(None) * 1.0, {"channel_name": "s"}) - cid = ch.get_id() - # Two intervals with different durations: [0,1) value 10, [1,3) value 20. - # duration-weighted mean = (10*1 + 20*2) / (1+2) = 50/3. - fact = spark.createDataFrame( - [(1, cid, 0, 1, 10.0), (1, cid, 1, 3, 20.0)], schema=_FACT_SCHEMA - ) - out = CalculatedChannel.determine_channel_metrics(spark, [ch], fact, attribute_columns=[]) - rows = out.collect() - assert len(rows) == 1 - r = rows[0] - assert r["container_id"] == 1 - assert r["channel_id"] == cid - assert r["value_type"] == "double" - assert r["duration"] == 3 # max(tend) - min(tstart) = 3 - 0 - assert r["min"] == 10.0 - assert r["max"] == 20.0 - assert r["mean"] == pytest.approx(50.0 / 3.0) - assert r["channel_name"] == "s" - - def test_nan_values_ignored_in_min_max_mean(self, spark): - ch = CalculatedChannel("a", TimeSeriesSelector(None) * 1.0, {"channel_name": "s"}) - cid = ch.get_id() - # [0,1) value 10, [1,2) NaN → NaN excluded from min/max/weighted-sum, but its - # duration still counts in the denominator (matches SampleSeries.mean). - fact = spark.createDataFrame( - [(1, cid, 0, 1, 10.0), (1, cid, 1, 2, float("nan"))], schema=_FACT_SCHEMA - ) - r = CalculatedChannel.determine_channel_metrics( - spark, [ch], fact, attribute_columns=[] - ).collect()[0] - assert r["min"] == 10.0 - assert r["max"] == 10.0 - # (10*1) / (1+1) = 5.0 - assert r["mean"] == pytest.approx(5.0) - - def test_zero_total_duration_mean_is_null_not_error(self, spark): - # A group of only zero-duration point-in-time samples (tstart == tend) has - # sum(dur) == 0. Under ANSI mode (Spark 4.0 default) plain division would - # raise DIVIDE_BY_ZERO; try_divide yields a null mean instead. - ch = CalculatedChannel("a", TimeSeriesSelector(None) * 1.0, {"channel_name": "s"}) - cid = ch.get_id() - fact = spark.createDataFrame( - [(1, cid, 5, 5, 10.0), (1, cid, 7, 7, 20.0)], schema=_FACT_SCHEMA - ) - r = CalculatedChannel.determine_channel_metrics( - spark, [ch], fact, attribute_columns=[] - ).collect()[0] - assert r["duration"] == 2 # max(tend) - min(tstart) = 7 - 5 - assert r["mean"] is None - # min/max still resolve from the values. - assert r["min"] == 10.0 - assert r["max"] == 20.0 - - def test_dynamic_identity_columns_union(self, spark): - # Two channels with DIFFERENT identity keys → output has the union, null - # where a channel omits a key. - ch1 = CalculatedChannel("a", TimeSeriesSelector(None) * 1.0, {"channel_name": "s1"}) - ch2 = CalculatedChannel( - "b", TimeSeriesSelector(None) * 1.0, {"channel_name": "s2", "data_key": "K"} - ) - fact = spark.createDataFrame( - [(1, ch1.get_id(), 0, 1, 1.0), (1, ch2.get_id(), 0, 1, 2.0)], - schema=_FACT_SCHEMA, - ) - out = CalculatedChannel.determine_channel_metrics( - spark, [ch1, ch2], fact, attribute_columns=[] - ) - assert "channel_name" in out.columns - assert "data_key" in out.columns - by_id = {r["channel_id"]: r for r in out.collect()} - assert by_id[ch1.get_id()]["channel_name"] == "s1" - assert by_id[ch1.get_id()]["data_key"] is None # ch1 has no data_key - assert by_id[ch2.get_id()]["data_key"] == "K" - - def test_attribute_columns_config_selected(self, spark): - # A configured attribute key surfaces as a column; unconfigured attributes - # do not. A channel omitting the key gets null. - ch1 = CalculatedChannel( - "a", TimeSeriesSelector(None) * 1.0, {"channel_name": "s1"}, attributes={"unit": "kmh"} - ) - ch2 = CalculatedChannel( - "b", TimeSeriesSelector(None) * 1.0, {"channel_name": "s2"}, attributes={"scale": "2"} - ) - fact = spark.createDataFrame( - [(1, ch1.get_id(), 0, 1, 1.0), (1, ch2.get_id(), 0, 1, 2.0)], - schema=_FACT_SCHEMA, - ) - out = CalculatedChannel.determine_channel_metrics( - spark, [ch1, ch2], fact, attribute_columns=["unit"] - ) - assert "unit" in out.columns - assert "scale" not in out.columns # not configured - by_id = {r["channel_id"]: r for r in out.collect()} - assert by_id[ch1.get_id()]["unit"] == "kmh" - assert by_id[ch2.get_id()]["unit"] is None - - def test_no_attribute_columns_by_default(self, spark): + def test_returns_fact_columns_with_matching_channel_id(self, spark, basic_narrow_db): + q = basic_narrow_db.query ch = CalculatedChannel( - "a", TimeSeriesSelector(None) * 1.0, {"channel_name": "s"}, attributes={"unit": "kmh"} - ) - fact = spark.createDataFrame([(1, ch.get_id(), 0, 1, 1.0)], schema=_FACT_SCHEMA) - out = CalculatedChannel.determine_channel_metrics(spark, [ch], fact, attribute_columns=[]) - assert out.columns == [ - "container_id", - "channel_id", - "channel_name", - "value_type", - "duration", - "min", - "max", - "mean", - ] - - def test_identity_wins_on_attribute_collision(self, spark): - # A key present in BOTH identity and attribute_columns yields the identity - # value, and only one column. - ch = CalculatedChannel( - "a", - TimeSeriesSelector(None) * 1.0, - {"channel_name": "s", "unit": "identity_unit"}, - attributes={"unit": "attr_unit"}, - ) - fact = spark.createDataFrame([(1, ch.get_id(), 0, 1, 1.0)], schema=_FACT_SCHEMA) - out = CalculatedChannel.determine_channel_metrics( - spark, [ch], fact, attribute_columns=["unit"] + name="rpm_x2", + expr=q.channel(channel_name="Engine RPM") * 2, + identity={"channel_name": "rpm_x2", "data_key": "CALC"}, ) - assert out.columns.count("unit") == 1 - assert out.collect()[0]["unit"] == "identity_unit" - - def test_kpis_subset_only_emits_selected(self, spark): - # Selecting a subset yields only those KPI columns (plus the fixed - # container/channel/identity/value_type columns). - ch = CalculatedChannel("a", TimeSeriesSelector(None) * 1.0, {"channel_name": "s"}) - fact = spark.createDataFrame([(1, ch.get_id(), 0, 2, 10.0)], schema=_FACT_SCHEMA) - out = CalculatedChannel.determine_channel_metrics( - spark, [ch], fact, attribute_columns=[], kpis=["mean"] + df = CalculatedChannel.determine_calculated_channels( + spark, [ch], query=q, solver=DefaultSolver(spark) ) - assert out.columns == [ + # Identity lives on the dimension (joined via channel_id), not the fact. + assert df.columns == [ "container_id", "channel_id", - "channel_name", - "value_type", - "mean", + "tstart", + "tend", + "value", ] - assert out.collect()[0]["mean"] == pytest.approx(10.0) - - def test_kpis_order_is_preserved(self, spark): - # The KPI columns appear in the configured order (tail of the schema). - ch = CalculatedChannel("a", TimeSeriesSelector(None) * 1.0, {"channel_name": "s"}) - fact = spark.createDataFrame([(1, ch.get_id(), 0, 2, 10.0)], schema=_FACT_SCHEMA) - out = CalculatedChannel.determine_channel_metrics( - spark, [ch], fact, attribute_columns=[], kpis=["max", "min", "duration"] - ) - assert out.columns[-3:] == ["max", "min", "duration"] - - def test_kpis_default_is_the_four(self, spark): - # kpis=None → default duration, min, max, mean (in order). - ch = CalculatedChannel("a", TimeSeriesSelector(None) * 1.0, {"channel_name": "s"}) - fact = spark.createDataFrame([(1, ch.get_id(), 0, 2, 10.0)], schema=_FACT_SCHEMA) - out = CalculatedChannel.determine_channel_metrics(spark, [ch], fact, attribute_columns=[]) - assert out.columns[-4:] == ["duration", "min", "max", "mean"] - - def test_unknown_kpi_raises(self, spark): - ch = CalculatedChannel("a", TimeSeriesSelector(None) * 1.0, {"channel_name": "s"}) - fact = spark.createDataFrame([(1, ch.get_id(), 0, 2, 10.0)], schema=_FACT_SCHEMA) - with pytest.raises(ValueError, match="Unknown calculated-channel KPI"): - CalculatedChannel.determine_channel_metrics( - spark, [ch], fact, attribute_columns=[], kpis=["bogus"] - ) + ids = {r["channel_id"] for r in df.select("channel_id").distinct().collect()} + assert ids == {ch.get_id()} diff --git a/tests/impulse_reporting/unit/config/config_parser_test.py b/tests/impulse_reporting/unit/config/config_parser_test.py index 64189af2..fbdfb545 100644 --- a/tests/impulse_reporting/unit/config/config_parser_test.py +++ b/tests/impulse_reporting/unit/config/config_parser_test.py @@ -2,7 +2,6 @@ from pydantic import ValidationError from impulse_reporting.config.config_parser import ( - CalculatedChannels, CastType, Comparator, ContainerFilters, @@ -113,6 +112,57 @@ def test_impulse_config_drop_implausible_data_enabled(): assert config.query_engine.drop_implausible_data is True +# --------------------------------------------------------------------------- +# Source.poi_channels_uri — must survive parsing AND reach the MeasurementDB. +# Regression: the field was missing from the Source model, so pydantic silently +# dropped it and the whole reporting-layer POI path was inert (has_poi_channels +# always False) even when the config supplied a poi_channels_uri. +# --------------------------------------------------------------------------- +def test_source_poi_channels_uri_parsed(): + config_json = { + **impulse_config_JSON, + "source": { + **impulse_config_JSON["source"], + "poi_channels_uri": "impulse_demo.silver.poi_channels", + }, + } + config = ImpulseConfig.model_validate(config_json) + assert config.source.poi_channels_uri == "impulse_demo.silver.poi_channels" + + +def test_source_poi_channels_uri_defaults_to_none(): + config = ImpulseConfig.model_validate(impulse_config_JSON) + assert config.source.poi_channels_uri is None + + +def test_poi_channels_uri_reaches_measurement_db(): + """End-to-end passthrough: a poi_channels_uri in the config makes the built + MeasurementDB POI-aware (this is what was silently broken).""" + from unittest.mock import create_autospec + + from databricks.sdk import WorkspaceClient + + from impulse_reporting.core.report import Report + + with_poi = ImpulseConfig.model_validate( + { + **impulse_config_JSON, + "source": { + **impulse_config_JSON["source"], + "poi_channels_uri": "impulse_demo.silver.poi_channels", + }, + } + ) + db = Report.create_measurement_db(with_poi, create_autospec(WorkspaceClient)) + assert db.has_poi_channels() + assert db.config.poi_channels_uri == "impulse_demo.silver.poi_channels" + + # ...and a config without it stays POI-unaware. + without_poi = ImpulseConfig.model_validate(impulse_config_JSON) + db2 = Report.create_measurement_db(without_poi, create_autospec(WorkspaceClient)) + assert not db2.has_poi_channels() + + def test_impulse_config_drop_implausible_data_rejects_rle(): """drop_implausible_data=True with RLE data must raise ValidationError. @@ -975,22 +1025,3 @@ def test_impulse_config_source_rejects_invalid_channel_mapping_table(): config_json["source"]["channel_mapping_table"] = "invalid_table_name" with pytest.raises(ValidationError): ImpulseConfig.model_validate(config_json) - - -def test_calculated_channels_default_kpis(): - """CalculatedChannels defaults to the four built-in KPIs.""" - config = CalculatedChannels() - assert config.emit_channel_metrics is False - assert config.kpis == ["duration", "min", "max", "mean"] - - -def test_calculated_channels_kpis_dedupe_preserves_order(): - """Duplicate KPI names are removed, insertion order preserved.""" - config = CalculatedChannels(kpis=["mean", "mean", "min"]) - assert config.kpis == ["mean", "min"] - - -def test_calculated_channels_unknown_kpi_rejected(): - """An unknown KPI name is rejected at validation with a helpful message.""" - with pytest.raises(ValidationError, match="Unknown calculated-channel KPI"): - CalculatedChannels(kpis=["bogus"]) diff --git a/tests/impulse_reporting/unit/core/report_utils_test.py b/tests/impulse_reporting/unit/core/report_utils_test.py index 497c1067..80928c3e 100644 --- a/tests/impulse_reporting/unit/core/report_utils_test.py +++ b/tests/impulse_reporting/unit/core/report_utils_test.py @@ -12,11 +12,8 @@ from impulse_reporting.core.report_utils import ( build_batches, build_metadata_dfs, - dispatch_calculated_channel_metrics, dispatch_events, - group_dfs_by_table, group_selectables_by_type, - persist_channel_metrics, merge_changed_unchanged, persist_dimensions_full, persist_dimensions_incremental, @@ -560,123 +557,6 @@ def determine_metadata_df(cls, spark, events): assert len(meta_calls) == 0 -class TestDispatchCalculatedChannelMetrics: - """Tests for dispatch_calculated_channel_metrics helper (routing only).""" - - def _make_enum(self, cls): - mock_type_enum = MagicMock() - mock_type_enum.__getitem__.return_value.value = cls - return mock_type_enum - - def test_empty_channels_by_type_returns_empty(self): - metrics = dispatch_calculated_channel_metrics( - spark=MagicMock(), - channels_by_type={}, - fact_dfs_by_type={}, - type_enum=MagicMock(), - attribute_columns=[], - kpis=["mean"], - ) - assert metrics == {} - - def test_routes_fact_df_channels_and_config(self): - """Each type's fact df + channel list + config are forwarded verbatim.""" - mock_result = MagicMock(spec=DataFrame) - mock_fact = MagicMock(spec=DataFrame) - channel = MagicMock() - received = {} - - class FakeCls: - @classmethod - def determine_channel_metrics(cls, spark, channels, fact_df, **kwargs): - received["channels"] = channels - received["fact_df"] = fact_df - received["kwargs"] = kwargs - return mock_result - - metrics = dispatch_calculated_channel_metrics( - spark=MagicMock(), - channels_by_type={"CALCULATED_CHANNEL": [channel]}, - fact_dfs_by_type={"CALCULATED_CHANNEL": mock_fact}, - type_enum=self._make_enum(FakeCls), - attribute_columns=["unit"], - kpis=["duration", "mean"], - ) - - assert metrics["CALCULATED_CHANNEL"] is mock_result - assert received["channels"] == [channel] - assert received["fact_df"] is mock_fact - assert received["kwargs"] == {"attribute_columns": ["unit"], "kpis": ["duration", "mean"]} - - def test_type_without_fact_df_is_skipped(self): - """A type present in channels but with no fact df (None) yields no entry.""" - calls = [] - - class FakeCls: - @classmethod - def determine_channel_metrics(cls, spark, channels, fact_df, **kwargs): - calls.append(fact_df) - return MagicMock(spec=DataFrame) - - metrics = dispatch_calculated_channel_metrics( - spark=MagicMock(), - channels_by_type={"CALCULATED_CHANNEL": [MagicMock()]}, - fact_dfs_by_type={"CALCULATED_CHANNEL": None}, - type_enum=self._make_enum(FakeCls), - attribute_columns=[], - kpis=["mean"], - ) - - assert metrics == {} - assert calls == [] # determine_channel_metrics never invoked - - def test_empty_channel_list_for_type_is_skipped(self): - metrics = dispatch_calculated_channel_metrics( - spark=MagicMock(), - channels_by_type={"CALCULATED_CHANNEL": []}, - fact_dfs_by_type={"CALCULATED_CHANNEL": MagicMock(spec=DataFrame)}, - type_enum=MagicMock(), - attribute_columns=[], - kpis=["mean"], - ) - assert metrics == {} - - -class TestGroupDfsByTable: - """Tests for the group-per-output-table shaping helper (returns lists).""" - - def test_empty_returns_empty(self): - assert group_dfs_by_table({}, lambda _t: "tbl") == {} - - def test_bare_df_grouped_into_list(self): - df = MagicMock(spec=DataFrame) - out = group_dfs_by_table({"A": df}, lambda _t: "tbl_a") - assert out == {"tbl_a": [df]} - - def test_changed_unchanged_dict_flattened_changed_then_unchanged(self): - changed = MagicMock(spec=DataFrame) - unchanged = MagicMock(spec=DataFrame) - out = group_dfs_by_table( - {"A": {"changed": changed, "unchanged": unchanged}}, lambda _t: "tbl_a" - ) - # Order is changed then unchanged (per _fact_dfs_for_table). - assert out == {"tbl_a": [changed, unchanged]} - - def test_types_sharing_a_table_are_combined(self): - df_a = MagicMock(spec=DataFrame) - df_b = MagicMock(spec=DataFrame) - out = group_dfs_by_table({"A": df_a, "B": df_b}, lambda _t: "shared") - assert out == {"shared": [df_a, df_b]} - - def test_none_and_empty_values_skipped(self): - df = MagicMock(spec=DataFrame) - out = group_dfs_by_table( - {"A": None, "B": {"changed": None, "unchanged": None}, "C": df}, - lambda t: t, - ) - assert out == {"C": [df]} - - # ============================================================================ # Tests: Report._solve_expressions_batched # ============================================================================ @@ -859,113 +739,6 @@ def test_query_select_called_with_batch_expressions(self, spark): report.query.select.return_value.solve.assert_called_once() -class TestSolveCalculatedChannelsBatched: - """Tests for Report._solve_calculated_channels_batched() (narrow / append).""" - - def test_empty_channels_returns_none(self, spark): - report = _build_report_for_solve(spark, _SINKLESS_CONFIG) - assert report._solve_calculated_channels_batched([]) is None - - def test_single_batch_sinkless_creates_temp_view(self, spark): - report = _build_report_for_solve(spark, _SINKLESS_CONFIG) - - mock_batch_df = MagicMock(spec=DataFrame) - mock_table_df = MagicMock(spec=DataFrame) - report.query = MagicMock() - report.query.select.return_value.solve_calculated_channels.return_value = mock_batch_df - report.spark = MagicMock() - report.spark.table.return_value = mock_table_df - - ch = MagicMock() - ch.get_selectors.return_value = [MagicMock()] - - result = report._solve_calculated_channels_batched([ch]) - - mock_batch_df.createOrReplaceTempView.assert_called_once() - view_name = mock_batch_df.createOrReplaceTempView.call_args[0][0] - assert view_name.startswith("__impulse_temp_") - report.spark.table.assert_called_once_with(view_name) - assert result is mock_table_df - - def test_single_batch_with_sink_writes_delta_table(self, spark): - report = _build_report_for_solve(spark, _SINK_CONFIG) - - mock_batch_df = MagicMock(spec=DataFrame) - mock_table_df = MagicMock(spec=DataFrame) - report.query = MagicMock() - report.query.select.return_value.solve_calculated_channels.return_value = mock_batch_df - report.spark = MagicMock() - report.spark.table.return_value = mock_table_df - - ch = MagicMock() - ch.get_selectors.return_value = [MagicMock()] - - result = report._solve_calculated_channels_batched([ch]) - - write_chain = mock_batch_df.write.format.return_value.mode.return_value - write_chain.saveAsTable.assert_called_once() - fq_name = write_chain.saveAsTable.call_args[0][0] - assert "__impulse_temp_" in fq_name - assert "spark_catalog" in fq_name - assert "gold" in fq_name - report.spark.table.assert_called_once_with(fq_name) - assert result is mock_table_df - - def test_multiple_batches_combined_with_union_by_name(self, spark): - """Narrow batches are row-appended with unionByName, NOT joined.""" - report = _build_report_for_solve(spark, _SINKLESS_CONFIG) - - mock_df1 = MagicMock(spec=DataFrame) - mock_df2 = MagicMock(spec=DataFrame) - mock_unioned = MagicMock(spec=DataFrame) - mock_df1.unionByName.return_value = mock_unioned - - table_returns = [mock_df1, mock_df2] - call_idx = [0] - - def fake_table(name): - df = table_returns[call_idx[0]] - call_idx[0] += 1 - return df - - report.query = MagicMock() - report.query.select.return_value.solve_calculated_channels.return_value = MagicMock( - spec=DataFrame - ) - report.spark = MagicMock() - report.spark.table.side_effect = fake_table - - ch1, ch2 = MagicMock(), MagicMock() - ch1.get_selectors.return_value = [MagicMock()] - ch2.get_selectors.return_value = [MagicMock()] - - with patch( - "impulse_reporting.core.report_utils.build_batches", return_value=[[ch1], [ch2]] - ): - result = report._solve_calculated_channels_batched([ch1, ch2]) - - mock_df1.unionByName.assert_called_once_with(mock_df2) - mock_df1.join.assert_not_called() - assert result is mock_unioned - - def test_query_select_called_with_batch_channels(self, spark): - report = _build_report_for_solve(spark, _SINKLESS_CONFIG) - - mock_batch_df = MagicMock(spec=DataFrame) - report.query = MagicMock() - report.query.select.return_value.solve_calculated_channels.return_value = mock_batch_df - report.spark = MagicMock() - report.spark.table.return_value = MagicMock(spec=DataFrame) - - ch = MagicMock() - ch.get_selectors.return_value = [MagicMock()] - - report._solve_calculated_channels_batched([ch]) - - report.query.select.assert_called_once_with(ch) - report.query.select.return_value.solve_calculated_channels.assert_called_once() - - # ============================================================================ # Fixtures for the generic entity orchestration/persistence helpers # ============================================================================ @@ -1435,68 +1208,3 @@ def test_upserts_each_type_with_merge_keys(self): ) sink.upsert.assert_called_once_with(foo_meta, "uri", ["channel_id"]) - - -class TestPersistChannelMetrics: - """Tests for the persist_channel_metrics free function (mock-based).""" - - def _mocks(self): - combined = MagicMock(spec=DataFrame) - combined.transform.return_value = combined # add_meta_information passthrough - transformer = MagicMock() - transformer.concat_dataframes.return_value = combined - type_enum = MagicMock() - type_enum.__getitem__.return_value.get_metrics_table_name.return_value = "metrics" - type_enum.get_any_for_metrics_table.return_value = "entity" - sink = MagicMock() - sink.config.get_output_uri_channel_metrics_table.return_value = "cat.sch.metrics" - return combined, transformer, type_enum, sink - - def test_empty_is_noop(self): - sink = MagicMock() - persist_channel_metrics({}, MagicMock(), sink, MagicMock(), incremental=False) - sink.store.assert_not_called() - sink.merge_incremental.assert_not_called() - - def test_full_mode_overwrites(self): - combined, transformer, type_enum, sink = self._mocks() - df = MagicMock(spec=DataFrame) - - persist_channel_metrics( - {"CALCULATED_CHANNEL": df}, type_enum, sink, transformer, incremental=False - ) - - sink.store.assert_called_once_with(combined, "cat.sch.metrics") - sink.merge_incremental.assert_not_called() - - def test_incremental_merges_with_keys_and_delete_scope(self): - combined, transformer, type_enum, sink = self._mocks() - df = MagicMock(spec=DataFrame) - - persist_channel_metrics( - {"CALCULATED_CHANNEL": df}, - type_enum, - sink, - transformer, - incremental=True, - updated_container_ids=[1, 2], - ) - - sink.store.assert_not_called() - args, kwargs = sink.merge_incremental.call_args - assert args[0] is combined - assert args[1] == "cat.sch.metrics" - assert args[2] == ["container_id", "channel_id"] - # updated containers → one delete-by-source predicate. - assert len(kwargs["delete_conditions"]) == 1 - - def test_incremental_without_updated_containers_has_no_delete_scope(self): - combined, transformer, type_enum, sink = self._mocks() - df = MagicMock(spec=DataFrame) - - persist_channel_metrics( - {"CALCULATED_CHANNEL": df}, type_enum, sink, transformer, incremental=True - ) - - _args, kwargs = sink.merge_incremental.call_args - assert kwargs["delete_conditions"] == [] diff --git a/tests/unit/data/basic_narrow_csv/channel_metrics.csv b/tests/unit/data/basic_narrow_csv/channel_metrics.csv index d43f4322..887cb034 100644 --- a/tests/unit/data/basic_narrow_csv/channel_metrics.csv +++ b/tests/unit/data/basic_narrow_csv/channel_metrics.csv @@ -11,3 +11,5 @@ container_id,channel_id,channel_name,group_idx,channel_idx,unit,sample_count,min 2,6,Ambient Air Temperature,2,2,C,57240,21,33,28.793081761006288,1499367269349000,1499372240481000,4971132000,11.514480001738036,DOUBLE 1,7,Vehicle Speed Sensor,3,1,km/h,59625,0,217,68.22906498951782,1499929242072000,1499934640063000,5397991000,11.045776104480352,DOUBLE 1,5,Engine RPM,2,1,RPM,59625,0,3658,1490.707790356394,1499929242072000,1499934640063000,5397991000,11.045776104480352,DOUBLE +1,90,DTC,0,0,,3,,,,1499929300000000,1499933000000000,3700000,,STRING +1,91,DTC_count,0,0,,3,1,3,2.0,1499929300000000,1499933000000000,3700000,,DOUBLE diff --git a/tests/unit/data/unit_test_csv/1_channel_metrics.csv b/tests/unit/data/unit_test_csv/1_channel_metrics.csv index 46dc9750..59749d53 100644 --- a/tests/unit/data/unit_test_csv/1_channel_metrics.csv +++ b/tests/unit/data/unit_test_csv/1_channel_metrics.csv @@ -1,2 +1,4 @@ container_id,channel_id,value_type,sample_count,nan_ratio,begin_s,end_s,duration_ms,original_sample_count,original_sr,min,max,mean,std,pz1,pz10,pz90,pz99 1,1,numerical,1,1.0,0.0,100.0,1,1,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +1,90,string,3,,2.0,8.0,0,3,,,,,,,,, +1,91,numerical,3,,2.0,8.0,0,3,,1.0,3.0,2.0,,,,, diff --git a/tests/unit/data/unit_test_csv/1_channel_tags.csv b/tests/unit/data/unit_test_csv/1_channel_tags.csv index b5724656..41208776 100644 --- a/tests/unit/data/unit_test_csv/1_channel_tags.csv +++ b/tests/unit/data/unit_test_csv/1_channel_tags.csv @@ -1,2 +1,4 @@ container_id,channel_id,key,value 1,1,seed,0 +1,90,channel_name,DTC +1,91,channel_name,DTC_count