From bde6b1e7c2894df98a29777eb2fb8fba5751fca0 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Mon, 15 Jun 2026 14:40:10 +0200 Subject: [PATCH 01/17] draft solution for fork pr handling --- .github/workflows/acceptance.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 96819e2..218c38b 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -37,6 +37,10 @@ jobs: labels: linux-ubuntu-latest permissions: id-token: write + # Fork PRs get no OIDC token / secrets from GitHub, so JFrog auth (and therefore + # dependency installation) cannot run. Skip CI for them; fork PRs are to be tested + # by the reviewer(s) / maintainer(s) before merging. + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -63,6 +67,8 @@ jobs: needs: [ not-a-fork, lint ] permissions: id-token: write + # See the note on `lint`: fork PRs cannot authenticate to JFrog, so skip CI for them. + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} strategy: fail-fast: false matrix: From 74504ee552eb5e2df9a44e1e38f37bc8b9e4677b Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 06:57:41 +0200 Subject: [PATCH 02/17] Added functionality for PointsInTimeSeries to support string as well --- .../model/series/points_in_time_series.py | 88 +++++++++++++- .../series/points_in_time_series_test.py | 111 ++++++++++++++++++ 2 files changed, 193 insertions(+), 6 deletions(-) diff --git a/src/impulse_query_engine/model/series/points_in_time_series.py b/src/impulse_query_engine/model/series/points_in_time_series.py index ba46f83..192a73d 100644 --- a/src/impulse_query_engine/model/series/points_in_time_series.py +++ b/src/impulse_query_engine/model/series/points_in_time_series.py @@ -2,7 +2,8 @@ from __future__ import annotations -from collections.abc import Sized +import functools +from collections.abc import Callable, Sized import numpy as np import numpy.typing as npt @@ -15,6 +16,34 @@ FloatOrNaN = float | np.float64 +def _numeric_only(method: Callable) -> Callable: + """Decorator that rejects the wrapped method on a string-valued series. + + String-valued :class:`PointsInTimeSeries` support only sampling and equality + (``==`` / ``!=``); arithmetic, ordering and numeric reductions have no meaning + for them. Numpy would either raise (``-``, ``/``, ``mean``) or — worse — + silently succeed with a nonsensical result (``+`` concatenates, ``*`` repeats, + ``sum`` concatenates), so guard those methods explicitly and fail loudly. + + Applied to arithmetic, ordering-comparison and reduction methods. + + Raises + ------ + TypeError + When the decorated method is called on a string-valued series. + """ + + @functools.wraps(method) + def wrapper(self: PointsInTimeSeries, *args, **kwargs): + if self._is_string: + raise TypeError( + f"{method.__name__} is not supported for string-valued PointsInTimeSeries" + ) + return method(self, *args, **kwargs) + + return wrapper + + class PointsInTimeSeries: def __init__(self, tstarts: Sized, values: Sized): """ @@ -32,31 +61,62 @@ def __init__(self, tstarts: Sized, values: Sized): Array-like of values, one per time point. """ assert len(tstarts) == len(values) + # Timestamps are always numeric. Values may be numeric or string: + # string-valued series support sampling (``synchronized`` / ``.where``) + # and equality comparisons (``==`` / ``!=``) only — arithmetic, ordering + # and numeric reductions are rejected (see the ``@_numeric_only`` methods). + # An empty series has no observed value type, so it defaults to numeric + # (the safe, backward-compatible case). self.tstarts = np.array(tstarts, dtype=np.float64) - self.values = np.array(values, dtype=np.float64) + self._is_string = np.asarray(values).dtype.kind in ("U", "S", "O") + if self._is_string: + self.values = np.asarray(values, dtype=object) + else: + self.values = np.array(values, dtype=np.float64) def dtype(self): """ Returns the Spark data type for PointsInTimeSeries. + For numeric values the element is a homogeneous ``[tstart, value]`` double + pair (``ArrayType(ArrayType(DoubleType))``). String-valued series cannot use + that homogeneous nested array, so their element is a ``(tstart, value)`` + struct with a double timestamp and a string value. + Returns ------- pyspark.sql.types.ArrayType - Spark ArrayType for points in time series: [[tstart_1, value_1], ...]. - """ + Spark ArrayType matching ``get_data``'s shape for this series' value type. + """ + if self._is_string: + return T.ArrayType( + T.StructType( + [ + T.StructField("tstart", T.DoubleType()), + T.StructField("value", T.StringType()), + ] + ) + ) return T.ArrayType(T.ArrayType(T.DoubleType())) def get_data(self) -> list: """ - Returns the series as a list of [tstart, value] lists. + Returns the series as a list of ``[tstart, value]`` pairs. + + For numeric values this is a list of two-element double lists. For string + values, ``column_stack`` would coerce the timestamps to strings, so the + pairs are built explicitly as ``[float(tstart), str(value)]`` — matching the + struct element type declared by :meth:`dtype`. Returns ------- list - List of [tstart, value] pairs. + List of ``[tstart, value]`` pairs. """ if len(self) == 0: return [] + if self._is_string: + return [[float(t), str(v)] for t, v in zip(self.tstarts, self.values, strict=True)] return np.column_stack([self.tstarts, self.values]).tolist() def __len__(self) -> int: @@ -354,34 +414,42 @@ def _apply_basic_rop(self, operation, other: float | SampleSeries | PointsInTime return PointsInTimeSeries(s0.tstarts, operation(s1.values, s0.values)) return PointsInTimeSeries(self.tstarts, operation(other, self.values)) + @_numeric_only def __add__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Add another series or scalar to this series.""" return self._apply_basic_op(np.add, other) + @_numeric_only def __radd__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Add this series to another series or scalar (reversed operands).""" return self._apply_basic_rop(np.add, other) + @_numeric_only def __sub__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Subtract another series or scalar from this series.""" return self._apply_basic_op(np.subtract, other) + @_numeric_only def __rsub__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Subtract this series from another series or scalar (reversed operands).""" return self._apply_basic_rop(np.subtract, other) + @_numeric_only def __mul__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Multiply this series by another series or scalar.""" return self._apply_basic_op(np.multiply, other) + @_numeric_only def __rmul__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Multiply another series or scalar by this series (reversed operands).""" return self._apply_basic_rop(np.multiply, other) + @_numeric_only def __truediv__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Divide this series by another series or scalar.""" return self._apply_basic_op(np.true_divide, other) + @_numeric_only def __rtruediv__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Divide another series or scalar by this series (reversed operands).""" return self._apply_basic_rop(np.true_divide, other) @@ -411,18 +479,22 @@ def __apply_op( idx = operation(self.values, other) return PointsInTime(self.tstarts[idx]) + @_numeric_only def __gt__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTime: """Return points where this series is greater than another.""" return self.__apply_op(np.greater, other) + @_numeric_only def __ge__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTime: """Return points where this series is greater than or equal to another.""" return self.__apply_op(np.greater_equal, other) + @_numeric_only def __lt__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTime: """Return points where this series is less than another.""" return self.__apply_op(np.less, other) + @_numeric_only def __le__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTime: """Return points where this series is less than or equal to another.""" return self.__apply_op(np.less_equal, other) @@ -448,6 +520,7 @@ def count(self) -> int: """ return len(self) + @_numeric_only def sum(self) -> FloatOrNaN: """ Returns the sum of the values. @@ -461,6 +534,7 @@ def sum(self) -> FloatOrNaN: return np.nan return np.sum(self.values) + @_numeric_only def mean(self) -> FloatOrNaN: """ Returns the mean of the values. @@ -474,6 +548,7 @@ def mean(self) -> FloatOrNaN: return np.nan return np.mean(self.values) + @_numeric_only def min(self) -> FloatOrNaN: """ Returns the minimum value. @@ -487,6 +562,7 @@ def min(self) -> FloatOrNaN: return np.nan return np.min(self.values) + @_numeric_only def max(self) -> FloatOrNaN: """ Returns the maximum value. diff --git a/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py b/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py index b957d96..77f8662 100644 --- a/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py +++ b/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py @@ -152,6 +152,117 @@ def test_aggregations_empty(): assert pts.count() == 0 +# --- string values ------------------------------------------------------------------------------ +# String-valued series support sampling and equality only; arithmetic, ordering +# and numeric reductions are rejected. Timestamps stay numeric regardless. + + +def test_string_values_stored_as_object_with_numeric_timestamps(): + pts = PointsInTimeSeries([1, 2, 3], ["P108B", "U0046", "P108B"]) + assert pts._is_string is True + assert pts.values.dtype == object + assert pts.tstarts.dtype == np.float64 + nptest.assert_array_equal(pts.values, ["P108B", "U0046", "P108B"]) + + +def test_empty_series_defaults_to_numeric(): + # No observed value type -> numeric (backward-compatible default). + assert PointsInTimeSeries.empty()._is_string is False + + +def test_numeric_series_is_not_string(): + assert PointsInTimeSeries([0, 1], [10, 20])._is_string is False + + +def test_string_eq_scalar_returns_points_in_time(): + pts = PointsInTimeSeries([1, 2, 3], ["P108B", "U0046", "P108B"]) + result = pts == "P108B" + assert isinstance(result, PointsInTime) + nptest.assert_array_equal(result.tstarts, [1, 3]) + + +def test_string_ne_scalar_returns_points_in_time(): + pts = PointsInTimeSeries([1, 2, 3], ["P108B", "U0046", "P108B"]) + nptest.assert_array_equal((pts != "P108B").tstarts, [2]) + + +def test_string_eq_series_matches_on_value_and_timestamp(): + p1 = PointsInTimeSeries([1, 2, 3], ["A", "B", "C"]) + p2 = PointsInTimeSeries([2, 3, 4], ["X", "C", "C"]) + # Common timestamps {2,3}; values equal only at t=3 ("C" == "C"). + nptest.assert_array_equal((p1 == p2).tstarts, [3]) + + +def test_string_synchronized_with_sample_series_samples_values(): + pts = PointsInTimeSeries([5, 15, 25], ["a", "b", "c"]) + s = SampleSeries([0, 10, 20], [10, 20, 30], [1, 2, 3]) + a, b = pts.synchronized(s) + nptest.assert_array_equal(a.tstarts, [5, 15, 25]) + nptest.assert_array_equal(a.values, ["a", "b", "c"]) + nptest.assert_array_equal(b.values, [1, 2, 3]) + + +def test_string_get_data_pairs_double_timestamp_with_string_value(): + pts = PointsInTimeSeries([1, 2], ["P108B", "U0046"]) + assert pts.get_data() == [[1.0, "P108B"], [2.0, "U0046"]] + + +def test_string_dtype_is_struct_of_double_and_string(): + pts = PointsInTimeSeries([1, 2], ["P108B", "U0046"]) + assert pts.dtype() == T.ArrayType( + T.StructType( + [ + T.StructField("tstart", T.DoubleType()), + T.StructField("value", T.StringType()), + ] + ) + ) + + +@pytest.mark.parametrize( + "op", + [ + lambda p: p + "x", + lambda p: "x" + p, + lambda p: p - 1, + lambda p: 1 - p, + lambda p: p * 2, + lambda p: p / 2, + ], +) +def test_string_arithmetic_raises(op): + pts = PointsInTimeSeries([1, 2], ["A", "B"]) + with pytest.raises(TypeError, match="string-valued"): + op(pts) + + +@pytest.mark.parametrize( + "op", + [ + lambda p: p > "A", + lambda p: p >= "A", + lambda p: p < "Z", + lambda p: p <= "Z", + ], +) +def test_string_ordering_comparison_raises(op): + pts = PointsInTimeSeries([1, 2], ["A", "B"]) + with pytest.raises(TypeError, match="string-valued"): + op(pts) + + +@pytest.mark.parametrize("reduction", ["sum", "mean", "min", "max"]) +def test_string_reductions_raise(reduction): + pts = PointsInTimeSeries([1, 2], ["A", "B"]) + with pytest.raises(TypeError, match="string-valued"): + getattr(pts, reduction)() + + +def test_string_count_is_allowed(): + # count is structural (not value-dependent), so it works for strings. + assert PointsInTimeSeries([1, 2, 3], ["A", "B", "C"]).count() == 3 + + # --- plane_sweep -------------------------------------------------------------------------------- From 110ad1a947b65660f39df9cdebab04b6258291bc Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 06:59:22 +0200 Subject: [PATCH 03/17] wip corrected github action --- .github/workflows/acceptance.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 218c38b..96819e2 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -37,10 +37,6 @@ jobs: labels: linux-ubuntu-latest permissions: id-token: write - # Fork PRs get no OIDC token / secrets from GitHub, so JFrog auth (and therefore - # dependency installation) cannot run. Skip CI for them; fork PRs are to be tested - # by the reviewer(s) / maintainer(s) before merging. - if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -67,8 +63,6 @@ jobs: needs: [ not-a-fork, lint ] permissions: id-token: write - # See the note on `lint`: fork PRs cannot authenticate to JFrog, so skip CI for them. - if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} strategy: fail-fast: false matrix: From 4e60ac4ae27c26bb589ce7914227e5a76fbaa45c Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 07:09:40 +0200 Subject: [PATCH 04/17] added update-api-docs to Makefile ran update-api-docs --- Makefile | 5 ++++- .../model/series/points_in_time_series.md | 16 +++++++++++++--- .../aggregations/stats_aggregator.md | 18 ++++++++++-------- .../impulse_reporting/config/config_parser.md | 4 ++++ .../api/impulse_reporting/core/report.md | 10 +++++++++- 5 files changed, 40 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 3d13e27..445f0ab 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,9 @@ coverage: build: uv build --require-hashes --build-constraints=.build-constraints.txt +update-api-docs: + cd docs/impulse && uv run pydoc-markdown + lock-dependencies: UV_LOCKED := 0 lock-dependencies: uv lock @@ -56,4 +59,4 @@ fork-sync: ./.github/scripts/fork-sync-pr.sh $(PR) .DEFAULT: all -.PHONY: all clean dev lint fmt test coverage build lock-dependencies fork-sync +.PHONY: all clean dev lint fmt test coverage build update-api-docs lock-dependencies fork-sync diff --git a/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md b/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md index 50cbf77..ef8b12c 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md @@ -37,9 +37,14 @@ def dtype() Returns the Spark data type for PointsInTimeSeries. +For numeric values the element is a homogeneous ``[tstart, value]`` double +pair (``ArrayType(ArrayType(DoubleType))``). String-valued series cannot use +that homogeneous nested array, so their element is a ``(tstart, value)`` +struct with a double timestamp and a string value. + **Returns**: -`pyspark.sql.types.ArrayType`: Spark ArrayType for points in time series: [[tstart_1, value_1], ...]. +`pyspark.sql.types.ArrayType`: Spark ArrayType matching ``get_data``'s shape for this series' value type. #### get\_data @@ -47,11 +52,16 @@ Returns the Spark data type for PointsInTimeSeries. def get_data() -> list ``` -Returns the series as a list of [tstart, value] lists. +Returns the series as a list of ``[tstart, value]`` pairs. + +For numeric values this is a list of two-element double lists. For string +values, ``column_stack`` would coerce the timestamps to strings, so the +pairs are built explicitly as ``[float(tstart), str(value)]`` — matching the +struct element type declared by :meth:`dtype`. **Returns**: -`list`: List of [tstart, value] pairs. +`list`: List of ``[tstart, value]`` pairs. #### \_\_len\_\_ diff --git a/docs/impulse/docs/references/api/impulse_reporting/aggregations/stats_aggregator.md b/docs/impulse/docs/references/api/impulse_reporting/aggregations/stats_aggregator.md index 9782ce8..14b7a93 100644 --- a/docs/impulse/docs/references/api/impulse_reporting/aggregations/stats_aggregator.md +++ b/docs/impulse/docs/references/api/impulse_reporting/aggregations/stats_aggregator.md @@ -190,14 +190,16 @@ Only includes computation-affecting attributes: - input_expressions - statistics to be calculated - event expression if there is any -- custom statistics (name, kind, declared input indices, and function - bytecode, so implementation or input-wiring changes invalidate cached - results; only appended when custom statistics are configured so - aggregators without them keep their previous hash) - -Excludes: name, desc, signal_name, units, page_number, report_id, and the -cross-channel descriptors' channel_name (presentation metadata, like -channel_names). +- channel_names, and each cross-channel descriptor's channel_name. These + are the fact table's ``channel_name`` merge key, so a rename must force + a recompute (a changed definition recomputes and prunes all containers); + otherwise, in incremental mode, already-processed containers would keep + rows under the old name. +- custom statistics (labels, kind, declared input indices, params, and + function bytecode, so implementation or input-wiring changes invalidate + cached results; only appended when custom statistics are configured) + +Excludes: name, desc, units, page_number, report_id. **Returns**: diff --git a/docs/impulse/docs/references/api/impulse_reporting/config/config_parser.md b/docs/impulse/docs/references/api/impulse_reporting/config/config_parser.md index 3658b3a..5201c85 100644 --- a/docs/impulse/docs/references/api/impulse_reporting/config/config_parser.md +++ b/docs/impulse/docs/references/api/impulse_reporting/config/config_parser.md @@ -102,6 +102,10 @@ Configuration for data sink location in Unity Catalog. - `catalog` (`str`): Target catalog name for output tables. - `schema` (`str`): Target schema name for output tables. - `table_prefix` (`str`): Prefix to use for generated output table names. +- `cleanup_temp_tables` (`bool`): When ``True``, the intermediate ``__impulse_temp_*`` tables written to this +sink during batch solving are dropped after ``persist_results()`` completes +successfully. Defaults to ``False`` (temp tables are retained for inspection +and only cleared at the start of the next report run). ## Comparator diff --git a/docs/impulse/docs/references/api/impulse_reporting/core/report.md b/docs/impulse/docs/references/api/impulse_reporting/core/report.md index fd9a5bf..7dc38f2 100644 --- a/docs/impulse/docs/references/api/impulse_reporting/core/report.md +++ b/docs/impulse/docs/references/api/impulse_reporting/core/report.md @@ -299,7 +299,7 @@ Get the list of calculated channels associated with the report. #### persist\_results ```python -def persist_results() +def persist_results(cleanup_temp_tables: bool | None = None) ``` Persist report results using appropriate strategy based on definition changes. @@ -308,6 +308,14 @@ Uses tracked state from determine_report() to decide persistence strategy: - Changed definitions: replaceWhere (atomic delete + insert) - Unchanged definitions: MERGE (upsert) +**Arguments**: + +- `cleanup_temp_tables` (`bool`): Whether to drop the batch-solving ``__impulse_temp_*`` tables from the +sink schema after persistence completes successfully. +- True/False: use this value, overriding the config flag. +- None (default): fall back to ``config.unity_sink.cleanup_temp_tables`` + (which itself defaults to False). + **Returns**: `None`: From a4d6e2c580f7deb011191702eb8d1d7bdbe94230 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 14:18:30 +0200 Subject: [PATCH 05/17] added poi_series_integration.md and marked differences from design to impl --- demos/data/reporting/channel_metrics.csv | 32 +- demos/data/reporting/channel_tags.csv | 36 + demos/data/reporting/poi_channels.csv | 13 + demos/reporting_pipeline.ipynb | 95 +- .../metadata/time_series_expression.md | 77 +- .../query/aggregations/statistic_type.md | 2 + .../analyze/query/query_builder.md | 34 + .../analyze/query/solvers/solver_config.md | 36 + .../model/series/points_in_time_series.md | 27 +- poi_series_integration.md | 896 ++++++++++++++++++ .../metadata/time_series_expression.py | 142 ++- .../analyze/query/query_builder.py | 44 + .../analyze/query/solvers/blob_solver.py | 7 +- .../analyze/query/solvers/default_solver.py | 127 ++- .../analyze/query/solvers/empty_cache.py | 7 +- .../analyze/query/solvers/series_cache.py | 25 +- .../analyze/query/solvers/solver_config.py | 25 + src/impulse_query_engine/measurement_db.py | 21 + .../model/series/points_in_time_series.py | 34 +- src/impulse_query_engine/schema.py | 20 + src/impulse_reporting/config/config_parser.py | 1 + tests/conftest.py | 31 + .../integration/poi_channel_solve_test.py | 243 +++++ ...default_solver_wide_column_mapping_test.py | 1 + .../solvers/default_solver_wide_only_test.py | 1 + .../query/solvers/solver_config_test.py | 12 +- .../data/basic_narrow_csv/channel_metrics.csv | 2 + .../data/unit_test_csv/1_channel_metrics.csv | 2 + .../data/unit_test_csv/1_channel_tags.csv | 2 + 29 files changed, 1894 insertions(+), 101 deletions(-) create mode 100644 demos/data/reporting/poi_channels.csv create mode 100644 poi_series_integration.md create mode 100644 tests/impulse_query_engine/integration/poi_channel_solve_test.py diff --git a/demos/data/reporting/channel_metrics.csv b/demos/data/reporting/channel_metrics.csv index 5637f58..7846174 100644 --- a/demos/data/reporting/channel_metrics.csv +++ b/demos/data/reporting/channel_metrics.csv @@ -1,13 +1,19 @@ -container_id,channel_id,sample_count,min,max,mean,begin_ms,end_ms,duration_ms,sample_rate,value_type -1,4,56667,-11.0,-3.0,-8.35955670848995,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE -1,5,56667,0.0,3385.0,1572.765489614767,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE -1,7,56667,0.0,201.0,70.67870188998889,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE -1,9,56667,-33.0,130.0,2.3332627455132617,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE -2,2,47336,-8.0,6.0,-6.105247591684975,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE -2,5,47336,0.0,3177.0,1736.7744422849416,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE -2,7,47336,0.0,188.0,83.95660807841811,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE -2,8,47336,-26.0,117.0,0.8931046138245733,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE -3,2,54775,-32.0,134.0,9.632861706983112,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE -3,7,54775,0.0,2545.0,1308.4330990415335,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE -3,9,54775,0.0,125.0,44.4629849383843,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE -3,10,54775,-3.0,2.0,-2.157334550433592,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE +container_id,channel_id,sample_count,min,max,mean,begin_ms,end_ms,duration_ms,sample_rate,value_type,series_type +1,4,56667,-11.0,-3.0,-8.35955670848995,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE +1,5,56667,0.0,3385.0,1572.765489614767,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE +1,7,56667,0.0,201.0,70.67870188998889,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE +1,9,56667,-33.0,130.0,2.3332627455132617,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE +2,2,47336,-8.0,6.0,-6.105247591684975,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE +2,5,47336,0.0,3177.0,1736.7744422849416,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE +2,7,47336,0.0,188.0,83.95660807841811,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE +2,8,47336,-26.0,117.0,0.8931046138245733,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE +3,2,54775,-32.0,134.0,9.632861706983112,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE +3,7,54775,0.0,2545.0,1308.4330990415335,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE +3,9,54775,0.0,125.0,44.4629849383843,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE +3,10,54775,-3.0,2.0,-2.157334550433592,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE +1,90,3,,,,1519629856439000,1519633356439000,3500000,,STRING,POINTS_IN_TIME +1,91,3,1.0,3.0,2.0,1519629856439000,1519633356439000,3500000,,DOUBLE,POINTS_IN_TIME +2,90,2,,,,1519756824107000,1519758824107000,2000000,,STRING,POINTS_IN_TIME +2,91,2,1.0,2.0,1.5,1519756824107000,1519758824107000,2000000,,DOUBLE,POINTS_IN_TIME +3,90,1,,,,1519926478375000,1519926478375000,0,,STRING,POINTS_IN_TIME +3,91,1,1.0,1.0,1.0,1519926478375000,1519926478375000,0,,DOUBLE,POINTS_IN_TIME diff --git a/demos/data/reporting/channel_tags.csv b/demos/data/reporting/channel_tags.csv index a35cf01..5918595 100644 --- a/demos/data/reporting/channel_tags.csv +++ b/demos/data/reporting/channel_tags.csv @@ -119,3 +119,39 @@ container_id,channel_id,key,value 3,10,model,Leon 3,10,to_city,RT 3,10,unit,C +1,90,brand,Seat +1,90,channel_name,DTC +1,90,model,Leon +1,90,experiment_id,experiment_4 +1,90,ecu,Engine_ECU +1,90,bus,CAN1 +1,90,code_system,P +1,91,brand,Seat +1,91,channel_name,DTC_count +1,91,model,Leon +1,91,experiment_id,experiment_4 +1,91,ecu,Engine_ECU +2,90,brand,Seat +2,90,channel_name,DTC +2,90,model,Leon +2,90,experiment_id,experiment_4 +2,90,ecu,Engine_ECU +2,90,bus,CAN1 +2,90,code_system,P +2,91,brand,Seat +2,91,channel_name,DTC_count +2,91,model,Leon +2,91,experiment_id,experiment_4 +2,91,ecu,Engine_ECU +3,90,brand,Seat +3,90,channel_name,DTC +3,90,model,Leon +3,90,experiment_id,experiment_4 +3,90,ecu,Body_ECU +3,90,bus,CAN2 +3,90,code_system,U +3,91,brand,Seat +3,91,channel_name,DTC_count +3,91,model,Leon +3,91,experiment_id,experiment_4 +3,91,ecu,Body_ECU diff --git a/demos/data/reporting/poi_channels.csv b/demos/data/reporting/poi_channels.csv new file mode 100644 index 0000000..8435727 --- /dev/null +++ b/demos/data/reporting/poi_channels.csv @@ -0,0 +1,13 @@ +container_id,channel_id,timestamp,value_double,value_string,dtype +1,90,1519629856439000,,P0301,string +1,90,1519631856439000,,P0301,string +1,90,1519633356439000,,P0135,string +1,91,1519629856439000,1.0,,double +1,91,1519631856439000,2.0,,double +1,91,1519633356439000,3.0,,double +2,90,1519756824107000,,P0420,string +2,90,1519758824107000,,P0128,string +2,91,1519756824107000,1.0,,double +2,91,1519758824107000,2.0,,double +3,90,1519926478375000,,U0100,string +3,91,1519926478375000,1.0,,double diff --git a/demos/reporting_pipeline.ipynb b/demos/reporting_pipeline.ipynb index 7d08294..15921e1 100644 --- a/demos/reporting_pipeline.ipynb +++ b/demos/reporting_pipeline.ipynb @@ -16,21 +16,21 @@ } }, "source": [ - "# Impulse — Reporting Pipeline Demo\n", + "# Impulse \u2014 Reporting Pipeline Demo\n", "\n", "The **Impulse Framework** is a Python library that enables\n", "automotive and industrial engineers to process, aggregate,\n", "and analyze petabytes of time-series measurement data on\n", - "Databricks — without requiring Apache Spark expertise.\n", + "Databricks \u2014 without requiring Apache Spark expertise.\n", "\n", "It provides **TSAL** (Time Series Analytics Language),\n", "a Pythonic expression language for defining signals,\n", "events, and aggregations.\n", "\n", "**What this notebook builds:**\n", - "A complete reporting pipeline — RPM histograms,\n", + "A complete reporting pipeline \u2014 RPM histograms,\n", "RPM-vs-speed heatmaps, per-distance-bin statistics, and\n", - "channel values sampled at every 10 km milestone — across\n", + "channel values sampled at every 10 km milestone \u2014 across\n", "3 test drives, persisted as a Gold-layer star schema and\n", "visualized inline with matplotlib.\n", "\n", @@ -61,9 +61,9 @@ "\n", "Impulse sits between a governed silver layer and a gold-layer star schema in Unity Catalog and provides three components:\n", "\n", - "- **TSAL (Time Series Analytics Language)** — a declarative Python DSL for expressing signals, events, and aggregations in natural Python, without requiring Spark expertise.\n", - "- **Query Engine** — pluggable and distributed; compiles TSAL expressions into Spark execution plans and adapts to any silver-layer layout via interchangeable solvers.\n", - "- **Aggregations** — domain-aware physical aggregations, including duration- and distance-weighted 1D/2D histograms and event-scoped statistics." + "- **TSAL (Time Series Analytics Language)** \u2014 a declarative Python DSL for expressing signals, events, and aggregations in natural Python, without requiring Spark expertise.\n", + "- **Query Engine** \u2014 pluggable and distributed; compiles TSAL expressions into Spark execution plans and adapts to any silver-layer layout via interchangeable solvers.\n", + "- **Aggregations** \u2014 domain-aware physical aggregations, including duration- and distance-weighted 1D/2D histograms and event-scoped statistics." ] }, { @@ -238,7 +238,7 @@ " (e.g., one test drive)\n", "- **Channel** = one sensor signal within a container\n", " (e.g., Engine RPM), stored as raw\n", - " `(timestamp, value)` samples — the framework\n", + " `(timestamp, value)` samples \u2014 the framework\n", " automatically converts these to intervals on the fly" ] }, @@ -275,7 +275,7 @@ "SILVER = [\n", " \"container_metrics\", \"container_tags\",\n", " \"channel_metrics\", \"channel_tags\",\n", - " \"channels\",\n", + " \"channels\", \"poi_channels\",\n", "]\n", "for t in SILVER:\n", " pdf = pd.read_csv(f\"{csv_dir}/{t}.csv\")\n", @@ -362,13 +362,13 @@ "# 2. Initialize the Report\n", "\n", "The `Report` orchestrator takes a config specifying:\n", - "- **`source`** — Silver layer tables\n", - "- **`unity_sink`** — Gold layer output\n", - "- **`query_engine.solver`** — `DefaultSolver` for\n", + "- **`source`** \u2014 Silver layer tables\n", + "- **`unity_sink`** \u2014 Gold layer output\n", + "- **`query_engine.solver`** \u2014 `DefaultSolver` for\n", " parallel per-container execution\n", - "- **`query_engine.data_type`** — `RAW` for raw\n", + "- **`query_engine.data_type`** \u2014 `RAW` for raw\n", " timestamp data (auto-converted to intervals)\n", - "- **`measurement_dimensions`** — container metadata\n", + "- **`measurement_dimensions`** \u2014 container metadata\n", " to carry into Gold layer" ] }, @@ -412,6 +412,7 @@ " \"container_metrics_table\": f\"{pfx}_container_metrics\",\n", " \"channel_metrics_table\": f\"{pfx}_channel_metrics\",\n", " \"channels_uri\": f\"{pfx}_channels\",\n", + " \"poi_channels_uri\": f\"{pfx}_poi_channels\",\n", " \"container_tags_table\": f\"{pfx}_container_tags\",\n", " \"channel_tags_table\": f\"{pfx}_channel_tags\",\n", " },\n", @@ -456,7 +457,7 @@ "source": [ "# 3. Select Physical Channels\n", "\n", - "Channels are selected by **metadata tags** —\n", + "Channels are selected by **metadata tags** \u2014\n", "no column names, no SQL, no joins.\n", "These are **lazy expressions**: no data is read yet." ] @@ -519,7 +520,7 @@ "# 4. Define Virtual Signals & Events\n", "\n", "**TSAL** uses Python operators to build lazy\n", - "expression trees — no Spark knowledge needed.\n", + "expression trees \u2014 no Spark knowledge needed.\n", "\n", "**Virtual signals** derive from physical channels.\n", "**Events** are time windows where a condition holds." @@ -558,7 +559,7 @@ ")\n", "\n", "# Instant the trip odometer crosses each additional\n", - "# 10 km — a set of points in time, not an interval.\n", + "# 10 km \u2014 a set of points in time, not an interval.\n", "distance_milestones = (distance_km % 10).falling_edges()" ] }, @@ -580,9 +581,9 @@ "source": [ "# 5. Register Events\n", "\n", - "- **BasicEvent** — from a TSAL boolean expression\n", - "- **ContainerEvent** — spans the entire recording\n", - "- **PointsInTimeEvent** — a set of instants (e.g. each 10 km milestone)" + "- **BasicEvent** \u2014 from a TSAL boolean expression\n", + "- **ContainerEvent** \u2014 spans the entire recording\n", + "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone)" ] }, { @@ -653,10 +654,10 @@ "source": [ "# 6. Define Aggregations\n", "\n", - "- **Histogram** — 1D duration-weighted distribution\n", - "- **Histogram2D** — 2D heatmap of two signals\n", - "- **StatisticsAggregator** — min, median, mean, max per event\n", - "- **PointValueAggregator** — channel value sampled at each instant of a points-in-time event" + "- **Histogram** \u2014 1D duration-weighted distribution\n", + "- **Histogram2D** \u2014 2D heatmap of two signals\n", + "- **StatisticsAggregator** \u2014 min, median, mean, max per event\n", + "- **PointValueAggregator** \u2014 channel value sampled at each instant of a points-in-time event" ] }, { @@ -740,7 +741,7 @@ "))\n", "\n", "# Sample Vehicle Speed & Engine RPM at each 10 km\n", - "# milestone — one value per channel per instant.\n", + "# milestone \u2014 one value per channel per instant.\n", "page.add_aggregation(PointValueAggregator(\n", " name=\"values_at_distance_milestones\",\n", " input_expressions=[veh_spd, eng_rpm],\n", @@ -769,8 +770,8 @@ "source": [ "# 7. Compute & Persist\n", "\n", - "- `determine_report()` — parallel execution\n", - "- `persist_results()` — writes star schema" + "- `determine_report()` \u2014 parallel execution\n", + "- `persist_results()` \u2014 writes star schema" ] }, { @@ -820,11 +821,11 @@ "Read the Gold-layer tables back and render the\n", "results inline with **matplotlib**:\n", "\n", - "- **Bar** — RPM histogram\n", - "- **Heatmap** — RPM vs Speed\n", - "- **Table** — per-container statistics\n", - "- **Scatter** — Speed & RPM at each 10 km milestone\n", - " (markers only — values exist only *at* each instant)" + "- **Bar** \u2014 RPM histogram\n", + "- **Heatmap** \u2014 RPM vs Speed\n", + "- **Table** \u2014 per-container statistics\n", + "- **Scatter** \u2014 Speed & RPM at each 10 km milestone\n", + " (markers only \u2014 values exist only *at* each instant)" ] }, { @@ -850,12 +851,12 @@ "source": [ "import matplotlib.pyplot as plt\n", "\n", - "# ─── Table prefix ───\n", + "# \u2500\u2500\u2500 Table prefix \u2500\u2500\u2500\n", "T = f\"{pfx}\"\n", "\n", - "# ════════════════════════════════════════════════════════════════════\n", - "# 1. BAR — RPM Histogram (aggregated across all containers)\n", - "# ════════════════════════════════════════════════════════════════════\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 1. BAR \u2014 RPM Histogram (aggregated across all containers)\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", "hist_df = (\n", " spark.read.table(f\"{T}_histogram_fact\")\n", " .join(\n", @@ -873,14 +874,14 @@ "ax.bar(hist_df[\"bin_name\"], hist_df[\"duration_s\"], color=\"steelblue\", edgecolor=\"white\")\n", "ax.set_xlabel(\"Engine RPM bin\")\n", "ax.set_ylabel(\"Duration (s)\")\n", - "ax.set_title(\"RPM Histogram — Duration in Each RPM Band (all containers)\")\n", + "ax.set_title(\"RPM Histogram \u2014 Duration in Each RPM Band (all containers)\")\n", "plt.xticks(rotation=45, ha=\"right\", fontsize=8)\n", "plt.tight_layout()\n", "plt.show()\n", "\n", - "# ════════════════════════════════════════════════════════════════════\n", - "# 2. HEATMAP — RPM vs Speed\n", - "# ════════════════════════════════════════════════════════════════════\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 2. HEATMAP \u2014 RPM vs Speed\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", "heat_df = (\n", " spark.read.table(f\"{T}_histogram2d_fact\")\n", " .groupBy(\"x_bin_id\", \"y_bin_id\", \"x_bin_name\", \"y_bin_name\",\n", @@ -917,14 +918,14 @@ "ax.set_yticklabels([lbl[1] for lbl in y_labels], fontsize=7)\n", "ax.set_xlabel(\"Engine RPM\")\n", "ax.set_ylabel(\"Vehicle Speed (km/h)\")\n", - "ax.set_title(\"RPM vs Speed Heatmap — Duration (s)\")\n", + "ax.set_title(\"RPM vs Speed Heatmap \u2014 Duration (s)\")\n", "plt.colorbar(im, ax=ax, label=\"Duration (s)\")\n", "plt.tight_layout()\n", "plt.show()\n", "\n", - "# ════════════════════════════════════════════════════════════════════\n", - "# 3. TABLE — Per-container Statistics (container_stats)\n", - "# ════════════════════════════════════════════════════════════════════\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 3. TABLE \u2014 Per-container Statistics (container_stats)\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", "stats_df = (\n", " spark.read.table(f\"{T}_stats_aggregator_fact\")\n", " .join(\n", @@ -952,9 +953,9 @@ " ),\n", ")\n", "\n", - "# ════════════════════════════════════════════════════════════════════\n", - "# 4. SCATTER — Speed & RPM at Each 10 km Milestone\n", - "# ════════════════════════════════════════════════════════════════════\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 4. SCATTER \u2014 Speed & RPM at Each 10 km Milestone\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", "milestone_df = (\n", " spark.read.table(f\"{T}_stats_aggregator_fact\")\n", " .join(\n", diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md index 253c03d..67c0c08 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md @@ -3,6 +3,39 @@ sidebar_label: time_series_expression title: impulse_query_engine.analyze.metadata.time_series_expression --- +## SeriesType + +```python +class SeriesType(StrEnum) +``` + +How a channel's samples are interpreted (mirrors :class:`RawEncoder`). + +``SAMPLE`` — the default; ``[tstart, tend)`` intervals over which the value is +*valid* (reconstructed by an interpolation method, zero-order hold today), +backed by :class:`SampleSeries`. + +``POINTS_IN_TIME`` — ``(tᵢ, vᵢ)`` points valid *only at* their timestamps, no +between-point validity, backed by :class:`PointsInTimeSeries`. + + +## PoiValueType + +```python +class PoiValueType(StrEnum) +``` + +The value data type of a POI channel — selects its ``poi_channels`` value + +column and which in-memory :class:`PointsInTimeSeries` variant is built. + +``DOUBLE`` — numeric points (``poi_channels.value_double``); the full +arithmetic / ordering / reduction operator set applies. + +``STRING`` — string points (``poi_channels.value_string``, e.g. DTC codes); +only sampling and equality apply (see :class:`PointsInTimeSeries`). + + ## TimeSeriesSelector ```python @@ -12,7 +45,10 @@ class TimeSeriesSelector(TimeSeriesExpression, RequiresDeserialization) #### \_\_init\_\_ ```python -def __init__(expr, uses_alias: bool = False) +def __init__(expr, + uses_alias: bool = False, + series_type: SeriesType = SeriesType.SAMPLE, + value_type: PoiValueType = PoiValueType.DOUBLE) ``` Initialize a TimeSeriesSelector. @@ -20,6 +56,18 @@ Initialize a TimeSeriesSelector. **Arguments**: - `expr` (`TagExpression`): Tag expression to select. +- `uses_alias` (`bool`): Whether the channel resolves via the channel-alias table. +- `series_type` (`SeriesType`): How the selected channel's samples are interpreted. ``SAMPLE`` +(default) builds a :class:`SampleSeries` — today's behavior, +unchanged. ``POINTS_IN_TIME`` builds a :class:`PointsInTimeSeries` +(values valid only at their timestamps); identification / matching is +identical, only the built object and its result dtype differ. This is +the plan-time source of truth for the series type (so ``dtype()`` is +correct for a bare POI selection with no per-channel metadata lookup). +- `value_type` (`PoiValueType`): For a ``POINTS_IN_TIME`` selection, the declared value data type +(``DOUBLE`` / ``STRING``). Ignored for ``SAMPLE``. Drives plan-time +typing and string-op gating; validated against the silver +``poi_channels.dtype`` at solve time (assertion contract). #### dtype @@ -31,7 +79,10 @@ Returns the Spark data type. **Returns**: -`pyspark.sql.types.DataType`: Data type (BinaryType). +`pyspark.sql.types.DataType`: ``BinaryType`` for a SAMPLE selection (serialized ``SampleSeries``), +or the value-type-aware ``PointsInTimeSeries.dtype()`` for a +POINTS_IN_TIME selection (``array>`` for numeric, +``array>`` for string). #### deserialize @@ -39,7 +90,11 @@ Returns the Spark data type. def deserialize(d) ``` -Deserialize sample series after collection/toPandas. +Deserialize a SAMPLE result after collection/toPandas. + +POINTS_IN_TIME results are serialized by ``get_data()`` (a plain +``[[t, v], ...]`` list) and need no deserialization, so they are returned +as-is; only a SAMPLE (binary) blob is decoded to a :class:`SampleSeries`. **Arguments**: @@ -47,23 +102,21 @@ Deserialize sample series after collection/toPandas. **Returns**: -`SampleSeries`: Deserialized sample series. +`SampleSeries or Any`: Deserialized sample series (SAMPLE), else *d* unchanged. #### build ```python -def build(cache: SeriesCache) -> SampleSeries +def build(cache: SeriesCache) ``` -Instantiate a SampleSeries from given cache data. +Instantiate the selected series from cache data. -**Arguments**: - -- `cache` (`SeriesCache`): Cache containing time series data. +Resolution is identical regardless of series type — resolve the matching +candidates, take the first ``(container_id, channel_id)``, and let the +cache build the right object. The **data** is authoritative for the built +type: :meth:`TimeSeriesCache.load_blob` returns a -**Returns**: - -`SampleSeries`: Built sample series. #### get\_required\_tag\_exprs diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/aggregations/statistic_type.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/aggregations/statistic_type.md index 4420d14..6646dd6 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/aggregations/statistic_type.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/aggregations/statistic_type.md @@ -20,4 +20,6 @@ Enumeration of supported statistic types for aggregations. - `MAX` (`str`): Maximum value statistic. - `MEAN` (`str`): Mean (average) value statistic. - `MEDIAN` (`str`): Median value statistic. +- `START` (`str`): First value in the interval. +- `END` (`str`): Last value in the interval. diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md index 40fffeb..1f5fb82 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md @@ -119,6 +119,40 @@ Create a time series selector for the given channel tags. `TimeSeriesSelector`: Time series selector object. +#### poi\_channel + +```python +def poi_channel(dtype: PoiValueType = PoiValueType.DOUBLE, + **kwargs) -> TimeSeriesSelector +``` + +Create a Points-in-Time (POI) channel selector. + +Parallel to :meth:`channel` — it builds the **same** ``TimeSeriesSelector`` +from a tag/column match on ``**kwargs`` (e.g. +``poi_channel(channel_name="DTC")``), differing only in that it is stamped +``series_type=POINTS_IN_TIME`` (so it solves to a +:class:`~impulse_query_engine.model.series.points_in_time_series.PointsInTimeSeries` +— a value valid only *at* each timestamp — rather than a ``SampleSeries``) +and carries the declared value ``dtype``. + +Channel *identification* (tag/column match, ``get_selector_expr``, +``required_tags``, ``selector_id``) is identical to :meth:`channel`; only +the built object and its result dtype differ. + +**Arguments**: + +- `dtype` (`PoiValueType`): The POI channel's value data type: ``DOUBLE`` (default, numeric) or +``STRING`` (e.g. DTC codes — only sampling and equality apply). This +declared type drives plan-time result typing and string-op gating; it +is validated against the silver ``poi_channels.dtype`` at solve time +(an actual/declared mismatch raises). +- `**kwargs` (`dict`): Channel tag-value pairs, matched exactly like :meth:`channel`'s. + +**Returns**: + +`TimeSeriesSelector`: A selector stamped ``series_type=POINTS_IN_TIME`` with the given value type. + #### select ```python diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md index 8d65a48..e16b99c 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md @@ -233,6 +233,42 @@ def value_col() -> str Internal column name for the signal value on the channels table. +#### poi\_timestamp\_col + +```python +def poi_timestamp_col() -> str +``` + +Internal column name for the point timestamp on the poi_channels table. + + +#### poi\_value\_double\_col + +```python +def poi_value_double_col() -> str +``` + +Internal column name for the numeric value on the poi_channels table. + + +#### poi\_value\_string\_col + +```python +def poi_value_string_col() -> str +``` + +Internal column name for the string value on the poi_channels table. + + +#### poi\_dtype\_col + +```python +def poi_dtype_col() -> str +``` + +Internal column name for the per-row value-dtype discriminator on poi_channels. + + #### tag\_key\_col ```python diff --git a/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md b/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md index ef8b12c..bdf55c3 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md @@ -24,6 +24,11 @@ A PointsInTimeSeries associates a value to each timestamp. Unlike a SampleSeries a value is only defined *at* its timestamp and is not considered valid in between consecutive timestamps. +The value type (numeric vs string) is inferred from *values*. An **empty** +series has no values to infer from and therefore defaults to numeric; use +:meth:`empty_string` when an explicitly string-typed empty series is needed +(e.g. plan-time result typing of a bare string-POI selection). + **Arguments**: - `tstarts` (`Sized`): Array-like of time points. @@ -420,9 +425,27 @@ Returns a string representation for debugging. def empty() -> PointsInTimeSeries ``` -Returns an empty PointsInTimeSeries. +Returns an empty (numeric) PointsInTimeSeries. + +**Returns**: + +`PointsInTimeSeries`: Empty numeric PointsInTimeSeries object. + +#### empty\_string + +```python +def empty_string() -> PointsInTimeSeries +``` + +Returns an empty **string-valued** PointsInTimeSeries. + +An empty series has no values to infer a type from, so the constructor +defaults to numeric; this factory forces the string value type. Used for +plan-time result typing of a bare string-POI selection, where the empty +series must report the string ``dtype()`` and reject numeric-only ops +(e.g. ``mean()``) before any data is read. **Returns**: -`PointsInTimeSeries`: Empty PointsInTimeSeries object. +`PointsInTimeSeries`: Empty string-valued PointsInTimeSeries object. diff --git a/poi_series_integration.md b/poi_series_integration.md new file mode 100644 index 0000000..5c975be --- /dev/null +++ b/poi_series_integration.md @@ -0,0 +1,896 @@ +--- +sidebar_position: 1 +title: POI Series Integration +--- + +# Design: Integrating Points-in-Time (POI) Series into the Silver Layer + +**Status:** Proposed  ·  **Scope:** `impulse_query_engine` silver-layer +data model + `DefaultSolver` solve stage  ·  **Non-goal:** changing the +6-stage filter pipeline. + +## 1. Summary + +Impulse currently models every channel as a **sample series** — a sequence of +`[tstart, tend)` intervals over which the series is assumed to be **valid** (this +validity is what channel synchronization relies on). It does *not* intrinsically +assume the value is *held constant* over the interval: how a value is reconstructed +within `[tstart, tend)` is an **interpolation** choice. Today the only interpolation +used is **zero-order hold** (the value at `tstart` carries forward), but additional +interpolation methods could be added in the future without changing the underlying +validity model. We want to add a second kind of channel, a **Points-in-Time (POI) +Series**: a list of `(tᵢ, vᵢ)` pairs where each value is defined **only at its +timestamp** and **no assumption of validity (and hence no interpolation) is made +between two consecutive timestamps**. + +The backend model class already exists — +[`PointsInTimeSeries`](../references/api/impulse_query_engine/model/series/points_in_time_series.md) +— and already implements arithmetic, comparisons, `synchronized` / `synchronized_all`, +and the reducing aggregations (`count`, `sum`, `mean`, `min`, `max`). The +integration work is therefore **not** about series math; it is about: + +1. **Where POI samples live in silver** (a new `poi_channels` table), and +2. **How the solver knows a channel is POI** — table membership (data in + `poi_channels` ⇒ POI) plus the query author's `poi_channel(...)` selector; no + explicit `series_type` column is needed (see [§3.2](#32-discriminator-table-membership-which-table-holds-the-channels-data)), and +3. **How the solve step builds a `PointsInTimeSeries` instead of a `SampleSeries`** + for those channels. + +The central design observation is that the entire metadata **filter pipeline is +already series-type-agnostic**, so POI support drops into the *solve* stage only. + +:::note Terminology + +- **Sample series** — the existing channel type; `[tstart, tend)` intervals over + which the series is *valid*, with values reconstructed by an interpolation method + (zero-order hold today). Backed by `SampleSeries`. +- **POI series** — the new channel type; `(tᵢ, vᵢ)` points valid *only at* their + timestamps, with no between-point validity or interpolation. Backed by + `PointsInTimeSeries`. + +::: + +### 1.1 Motivating example: ECU defect / error codes (DTCs) + +The canonical real-world POI series in vehicle testing is the stream of **defect +codes** (a.k.a. error codes, or **Diagnostic Trouble Codes — DTCs**) emitted by a +vehicle's Electronic Control Units (ECUs). When an ECU's diagnostic monitor detects +a fault — a misfire, a sensor reading out of range, a lost CAN message — it emits a +code at the **instant the fault is registered**. In a test fleet these are captured +off the CAN/UDS bus (e.g. via the `ReadDTCInformation` service, UDS `0x19`) and +logged with the timestamp at which the ECU reported them. + +A DTC event stream is a **textbook POI series**, and specifically a **string-valued** +one: + +- **Event-driven, not continuous.** A code exists *at* the moment the ECU raised it + and says **nothing** about the time between two codes. Interpolating "the value + between two error codes" is meaningless — which is exactly the POI validity model + (no between-point validity), and exactly what the held-over-interval `SampleSeries` + model would get *wrong*. +- **String values.** The standardized code is a short alphanumeric string in the + `P0301` form (1 letter for the system — **P**owertrain / **C**hassis / **B**ody / + **U**network — a generic/OEM digit, a subsystem family digit, and a 2-digit fault + index; e.g. `P0301` = cylinder-1 misfire). This is why POI channels need the + string `value_type` from [§3.4](#34-per-channel-value-dtype-double-vs-string): the + natural analysis is *equality* ("when did `P0301` occur?"), never arithmetic or + ordering on the code — matching the equality-only operator set we implement for + string POI series. + +This use case also motivates **mix-and-match** ([§2](#2-background-why-this-fits-so-cleanly)), +because DTCs are almost always analyzed **together with the continuous signals** +recorded in the same container: + +- **"Freeze-frame"-style analysis.** ECUs snapshot continuous PIDs (engine RPM, + vehicle speed, coolant temperature, …) at the instant a code is set. In Impulse + this is the exact shape of `PointValueAggregator` / a `PointsInTimeEvent`: sample a + `SampleSeries` channel (`Engine_RPM`) **at the timestamps of** a POI channel + (`DTC == "P0301"`). The POI channel supplies the instants; the sample channel + supplies the values valid at those instants — one query, one container, both series + types in the same pandas UDF. +- **Counting / windowing.** "How many `P0301` events occurred while + `Engine_RPM > 4000`?" combines a string-POI equality filter with an interval + derived from a sample series — again both series types in one expression. + +Sketched in the query API this design proposes ([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)), +the string DTC channel is selected with the dedicated `poi_channel(...)` method +and its `dtype`, and mixes freely with an ordinary `channel(...)` sample selection: + +```python +dtc = query.poi_channel(channel_name="DTC", dtype="string") # string POI series +rpm = query.channel(channel_name="Engine_RPM") # sample series + +# "freeze-frame": RPM at the instants DTC == "P0301" +rpm.where(dtc == "P0301") + +# equality is the only comparator defined on a string POI series (§3.4) +``` + +:::note Timestamp caveat (informative) + +DTCs do not universally carry an absolute wall-clock timestamp in the ECU fault +memory — the reliable instant usually comes from the logger/gateway that timestamps +the event when it reads the code (GPS/NTP-synced), and any per-DTC snapshot/extended +records are OEM-dependent. For Impulse this is an **ingestion** concern: whatever +timestamp the silver pipeline lands on `poi_channels.timestamp` is the instant the +engine treats as the point's `tᵢ`. It does not affect the data-model or solver +design below. + +::: + +## 2. Background: why this fits so cleanly + +The [`DefaultSolver` filter pipeline](../references/query_engine/query_solvers.md) +runs six stages, but only ever passes **identity + selector metadata** between +them: + +``` +filter_container_tags → filter_container_metrics → filter_channel_tags → +filter_channel_metrics → (alias resolution) → solve +``` + +Every stage up to `solve` produces at most +`(container_id, channel_id, selector_ids)` (plus optional unit columns). **None of +these stages read `tstart` / `tend` / `value` or make any interval-validity or +interpolation assumption.** The validity-and-interpolation semantics enter the +system in exactly one place: + +- `DefaultSolver.solve` reads the `channels` table, joins it to the channel-match + frame, and runs a grouped-map UDF (`_solve_udf`). +- Inside the UDF, `TimeSeriesCache.load_blob(...)` constructs a **`SampleSeries`** + from the `(ts, te, val)` columns. +- `TimeSeriesSelector.build(cache)` calls `cache.load_blob(...)` and returns that + `SampleSeries` to the expression tree. + +So a channel becomes a `SampleSeries` at `TimeSeriesCache.load_blob`, and nowhere +else. If we can make that one call return a `PointsInTimeSeries` for POI channels, +the rest of the engine — expression evaluation, events, aggregations — already +works, because `PointsInTimeSeries` and `SampleSeries` share the operator and +synchronization protocol, and `SampleSeries.where(PointsInTime)` / +`PointsInTimeSeries.plane_sweep` already bridge the two representations. + +**Mixing is the common case, and it is a per-container concern.** POI and sample +channels live in the **same containers**, and users routinely combine them in one +expression (e.g. `poi_channel - sample_channel`). Cross-type alignment +(`synchronized`) happens **inside** the per-container pandas UDF, on the in-memory +series objects. That imposes a hard requirement: **both series types for a container +must be present in the same UDF invocation.** A design that solved sample and POI +channels in two *separate* UDFs and unioned the results would break mixing — each +UDF would see only half of a container's channels and could not evaluate a +cross-type expression. The design below therefore feeds **one unified pandas frame +per container** (sample + POI channel data together) into a **single** grouped-map +UDF, and a unified cache builds the correct series type per channel. + +```mermaid +flowchart TB + subgraph pipeline["Filter pipeline (UNCHANGED — series-type-agnostic)"] + direction LR + A[container tags] --> B[container metrics] --> C[channel tags] --> D[channel metrics] + end + D -->|"(container_id, channel_id, selector_ids, series_type)"| J + + subgraph solve["solve stage (the ONLY place that touches series semantics)"] + direction TB + RS["read channels
(SAMPLE rows)"] --> J + RP["read poi_channels
(POI rows)"] --> J + J["union sample + POI sample data
keyed by (container_id, channel_id)
→ ONE frame per container"] + J --> U["grouped-map UDF, grouped by container_id
(both series types in the same pandas frame)"] + U --> C2["unified cache builds per channel:
SampleSeries (valid over interval; ZOH today)
or PointsInTimeSeries (valid only at points)"] + C2 --> EV["evaluate expression tree
cross-type ops align via synchronized"] + end + EV --> OUT([one wide row per container]) +``` + +## 3. Chosen design + +### 3.1 Storage: a separate `poi_channels` table + +POI samples are stored in a **new silver table `poi_channels`**, parallel to +`channels` but carrying a single timestamp (no derived `tend`, because a POI point +has no notion of a validity interval) and **two typed value columns** plus a +per-row `dtype` discriminator, since a POI value may be numeric **or** a string: + +| Column | Type | Nullable | Description | +|----------------|----------|----------|-------------------------------------------------------------------| +| `container_id` | `long` | No | Parent container identifier (join key). | +| `channel_id` | `int` | No | Channel identifier. | +| `timestamp` | `long` | No | Point timestamp (microseconds). | +| `value_double` | `double` | Yes | Value at this timestamp **when `dtype = double`**; else null. | +| `value_string` | `string` | Yes | Value at this timestamp **when `dtype = string`**; else null. | +| `dtype` | `string` | No | Value data type: `double` or `string`. Selects the value column. | + +For any given row exactly one of `value_double` / `value_string` is populated, +chosen by `dtype`. `dtype` is expected to be **constant per +`(container_id, channel_id)`** — a channel is either a numeric POI channel or a +string POI channel, not a mix (see [§3.4](#34-per-channel-value-dtype-double-vs-string)). + +`container_id` follows the same +[type rules as the rest of the silver layer](../data_model/silver_layer_schema.md) +— it may be `long` / `int` / `string`, but must be **consistent across all silver +tables** since the engine joins on it. This matches the CLAUDE.md invariant that +`container_id` / `channel_id` types are derived dynamically and never hardcoded. + +**Why a separate table rather than reusing `channels`:** + +- **No semantic overloading of `tend`.** The `channels` RLE format treats a + trailing zero-duration `[t, t)` row as a *closed endpoint* of a sample series (an + interval of validity that has collapsed to a single instant). Reusing that row + shape for a *whole* POI channel would require every + reader (the solve UDF, the RLE/interval encoders, `SampleSeries` construction) to + disambiguate "closed endpoint of a sample series" from "a genuine point". A + dedicated table keeps the two data shapes physically and semantically distinct. +- **Cleaner ingestion contract.** Producers write POI points as `(timestamp, value)` + with no obligation to synthesize a `tend`, which they cannot do correctly for POI + data anyway. +- **Minimal disturbance to the sample-series path.** The existing `channels` read + and RLE/interval encoding are untouched, and a `SAMPLE` channel still builds the + identical `SampleSeries`. The cache does gain a per-channel series-type dispatch + (required so sample and POI channels can be mixed in one UDF — see + [§4.3](#43-one-unified-per-container-frame-one-udf-one-dispatching-cache)), but the + sample branch's behavior is unchanged. + +The cost is a new configured table + a new read path + a branch in the solve +prelude — all localized to `DefaultSolver.solve` / `MeasurementDB` (see §4). + +### 3.2 Discriminator: a `series_type` column on `channel_metrics` + +:::note Implemented differently — see [§9](#9-aspects-which-differ-from-the-design) +The `series_type` column described below was **not** added. Table membership +(`channels` vs `poi_channels`) is the discriminator instead. See [§9](#9-aspects-which-differ-from-the-design). +::: + +A channel is marked POI by a **new `series_type` column on `channel_metrics`**: + +| Column | Type | Nullable | Description | +|---------------|----------|----------|--------------------------------------------------------------------| +| `series_type` | `string` | Yes | `SAMPLE` (default) or `POINTS_IN_TIME`. Null/absent ⇒ `SAMPLE`. | + +Design points: + +- **Backward compatible.** Existing tables without the column, or with `NULL`, + resolve to `SAMPLE`, so every current deployment behaves exactly as today. +- **Rides the pipeline as pass-through metadata.** `channel_metrics` is already + read in `filter_channel_metrics`; `series_type` is just one more column carried + on the channel-match rows through to `solve`. It participates in **no** filtering + decision. +- **Not `value_type`.** `channel_metrics.value_type` already exists but describes + the *value's data type* (`double`, `int`, …). Overloading it to also encode + *series semantics* would conflate two orthogonal concepts and is rejected. A new, + purpose-specific column keeps the discriminator explicit and self-documenting. +- **Introduce a `SeriesType` enum** (mirroring `RawEncoder`) so the string literals + live in one place and are referenced by `SolverConfig.series_type_col` / + the solve branch rather than being sprinkled as bare strings. + +`series_type` is added to `SolverConfig` as an internal column name property +(`series_type_col`, default `"series_type"`), so a physical layout that names the +column differently maps it via `channel_metrics.column_name_mapping` exactly like +every other column. + +### 3.3 Data model after the change + +```mermaid +erDiagram + container_metrics { + long container_id PK + } + channel_metrics { + long container_id FK + int channel_id FK + string series_type "SAMPLE | POINTS_IN_TIME (null ⇒ SAMPLE)" + } + channels { + long container_id FK + int channel_id FK + long tstart + long tend + double value + } + poi_channels { + long container_id FK + int channel_id FK + long timestamp + double value_double "when dtype = double" + string value_string "when dtype = string" + string dtype "double | string" + } + + container_metrics ||--o{ channel_metrics : container_id + channel_metrics ||--o{ channels : "SAMPLE channels" + channel_metrics ||--o{ poi_channels : "POINTS_IN_TIME channels" +``` + +A given `(container_id, channel_id)` has its samples in **exactly one** of +`channels` or `poi_channels`, selected by its `series_type` row in +`channel_metrics`. + +### 3.4 Per-channel value dtype: double vs string + +The `poi_channels.dtype` column determines which value column +(`value_double` / `value_string`) carries the point value. We treat `dtype` as a +**per-channel** property: all rows of a `(container_id, channel_id)` share one +`dtype`. This keeps a channel's value type stable, matches how measurement channels +behave in practice, and lets the solve step pick the value column **once** per +channel rather than per row. + +The two dtypes are **not** symmetric, because the backend model represents them +differently. `PointsInTimeSeries` **cannot represent string values today** — its +constructor hardcodes `np.array(values, dtype=np.float64)`, which would coerce +strings to `NaN`. We close this gap by extending the **single** +[`PointsInTimeSeries`](../references/api/impulse_query_engine/model/series/points_in_time_series.md) +class to hold values of either kind, rather than adding a second class. + +**Chosen model change — dual value arrays + a `value_type` property:** + +- **Keep the existing `float64` value array** for numeric values (unchanged; today's + numeric behavior is preserved bit-for-bit). +- **Add a second value array of dtype `object`** to hold string values. + *(Implemented differently — a single value array whose type is inferred at + construction; see [§9](#9-aspects-which-differ-from-the-design).)* +- **Add a `value_type` property on the class** distinguishing a **numeric** from a + **string** POI series. This is the single source of truth for which value array is + populated and which operations are legal. (Constructors/factories set it; a + numeric series leaves the object array empty and vice-versa.) +- **Spark `dtype()` becomes `value_type`-aware:** `ArrayType(ArrayType(DoubleType))` + for numeric (unchanged), `ArrayType(ArrayType(StringType))` for string. + +**Operations on a string POI series (this iteration):** + +- **Only the equality comparator (`==`) is implemented.** It matches the numeric + behavior — synchronize on shared timestamps, compare values, return the + `PointsInTime` where values are equal — but over string values. + *(Implemented more permissively — both `==` and `!=` are supported for strings; + see [§9](#9-aspects-which-differ-from-the-design).)* +- **All other comparators (`<`, `<=`, `>`, `>=`) return a + `NotImplementedError`** for a string series, as do the numeric-only reductions and + arithmetic (`sum`, `mean`, `min`, `max`, `+`, `-`, `*`, `/`). These raise a clear, + explicit error rather than silently coercing to `NaN`. +- Value-type-independent operations remain valid regardless of `value_type`: + `count`, `start_time` / `end_time`, `to_points_in_time`, `plane_sweep`, and the + timestamp side of `synchronized`. + +:::note "series type" appears on three distinct axes — keep them straight + +| Where | Values | Meaning | +|-------|--------|---------| +| table membership (`channels` / `poi_channels`) | sample vs POI | Which table holds the channel's data — *this* is the sample-vs-POI discriminator (no `series_type` column; see [§9](#9-aspects-which-differ-from-the-design)). | +| `poi_channels.dtype` (silver column) | `double` / `string` | A POI channel's value type — selects `value_double` vs `value_string`. | +| `PointsInTimeSeries.value_type` (class property) | numeric / string | Which in-memory value array is active and which operations are legal. | + +The middle and bottom rows are the same distinction on two sides of the Arrow +boundary: `poi_channels.dtype` on a channel becomes `PointsInTimeSeries.value_type` +on the object the cache builds for it. + +::: + +The **selectable operations are gated by `value_type`** so that, e.g., +`string_poi.mean()` fails up front (via `evaluation_type()` — see [§4.4](#44-result-typing)) +rather than producing `NaN`. + +:::note Scope check + +String POI support is the one part of this design that requires touching the +backend model (`PointsInTimeSeries`). Everything else — storage, discriminator, +pipeline, solve branch — is additive. If string POI is not needed in the first +iteration, the numeric (`double`) path can ship alone: the solver simply routes +only `dtype = double` channels and rejects (or ignores, per config) `string` +channels until the model work lands. + +::: + +### 3.5 Example: tag & metric entries for DTC POI channels + +Concrete rows for the [DTC example](#11-motivating-example-ecu-defect--error-codes-dtcs), +on an existing recording `container_id = 1`. Two POI channels are added on +`channel_id`s not used by any sample channel in that container: a **string** DTC-code +channel (`channel_id = 90`) and a **numeric** fault-occurrence-count channel +(`channel_id = 91`). + +#### Channel level — where POI-specific entries naturally live + +**Channel selection metadata.** In the EAV layout these are `channel_tags` rows +(`container_id, channel_id, key, value`); in the wide layout the same facts are +columns on `channel_metrics`. A DTC channel is selected by its `channel_name` and +described by ECU/bus context: + +| container_id | channel_id | key | value | +|--------------|------------|----------------|--------------| +| 1 | 90 | `channel_name` | `DTC` | +| 1 | 90 | `ecu` | `Engine_ECU` | +| 1 | 90 | `bus` | `CAN1` | +| 1 | 90 | `code_system` | `P` (powertrain) | +| 1 | 91 | `channel_name` | `DTC_count` | +| 1 | 91 | `ecu` | `Engine_ECU` | + +**Channel metrics** (`channel_metrics`). The **new `series_type`** marks the channel +as POI; the **existing `value_type`** records the value data type. Crucially, the +numeric statistic columns behave differently by value type — they are **undefined +(null) for a string POI channel**, and meaningful (computed over the point values, +**unweighted** — there are no durations) for a numeric one: + +| Column | DTC string channel (90) | DTC count numeric channel (91) | Notes | +|----------------|-------------------------|--------------------------------|-------| +| `series_type` | `POINTS_IN_TIME` | `POINTS_IN_TIME` | new discriminator (§3.2) | +| `value_type` | `STRING` | `DOUBLE` | pre-existing data-type column | +| `channel_name` | `DTC` | `DTC_count` | selection key (wide layout) | +| `sample_count` | `3` (three events) | `3` | number of points | +| `begin_s`/`end_s` | first/last event time | first/last event time | point extent, not a validity span | +| `min`/`max`/`mean`/`std` | **null** | computed over point values | undefined for strings; unweighted for numeric POI | +| `pz1`/`pz10`/`pz90`/`pz99` | **null** | optional | percentiles undefined for strings | +| `nan_ratio` | **null** | **null** | duration-weighted → N/A for POI | + +The per-row **`dtype`** (`string` / `double`) lives on `poi_channels`, not here (§3.1); +`series_type` on `channel_metrics` is what routes the channel to `poi_channels`. + +#### Container level — optional summaries for pre-filtering + +A container is a whole recording and owns **both** sample and POI channels, so +container-level tags/metrics are **not** POI-specific — the usual `vehicle_key`, +`brand`, `model`, `project` entries are unchanged. What POI *optionally* adds here is +**summary metadata that lets you pre-filter containers** without scanning +`poi_channels` (the same role the percentile columns play for sample channels): + +EAV `container_tags` (`container_id, key, value`): + +| container_id | key | value | Purpose | +|--------------|------------------|-------------|---------| +| 1 | `vehicle_key` | `Seat_Leon` | existing — unchanged | +| 1 | `has_dtc` | `true` | optional — "recordings that logged any fault" | +| 1 | `ecu_sw_version` | `4.11.2` | optional — correlate faults with firmware | + +Wide `container_metrics` can carry the analogous optional column +`num_dtc_events = 3` for the same pre-filtering purpose. + +These container-level additions are **purely optional and additive**: omit them and +POI channels still work; add them only to enable "find recordings where a `P0301` +occurred"-style container filters before the channel stage. A query like +`query.havingTag(has_dtc="true")` then narrows containers exactly as any other +container tag does — no POI-specific pipeline behavior. + + +## 4. Implementation plan + +The change is localized. Nothing in stages 1–5 of the pipeline changes. + +### 4.1 Config & schema + +1. `SolverConfig`: add `poi_channels: TableConfig`, add the `series_type_col` + property (`"series_type"`), and add a `poi_channels_uri` slot to + `MeasurementDBConfig` (+ `for_unity_catalog` / `for_debug` wiring, mirroring + `channels_uri`). `poi_channels_uri = None` means "no POI channels configured". +2. `MeasurementDB.poi_channels(spark)` reader, mirroring `channels(...)`. +3. `schema.py`: add a reference `POI_CHANNELS_SCHEMA` (`container_id`, `channel_id`, + `timestamp`, `value_double`, `value_string`, `dtype`) and add `series_type` to + `CHANNEL_METRICS`. As documented in CLAUDE.md these are **reference** schemas, + not enforced on read. +4. Add a `SeriesType` StrEnum (`SAMPLE`, `POINTS_IN_TIME`) next to `RawEncoder`, and + a `PoiValueType` StrEnum (`double`, `string`) for the per-row `dtype`. +5. Add `SolverConfig` internal-name properties for the new POI columns + (`poi_timestamp_col`, `poi_value_double_col`, `poi_value_string_col`, + `poi_dtype_col`) so physical layouts remap them via + `poi_channels.column_name_mapping` like every other table. +6. Extend `SolverConfig.col_map` (the short-key → column-name map handed to the UDF + cache, today `cid/ch/ts/te/val/conv`) with `series_type`, `value_string`, and + `dtype` keys so the unified cache (§4.3) can locate them in the pandas frame. +7. Add two optional fields to `TimeSeriesSelector` (`series_type`, `value_type`), + defaulting to `SAMPLE` / numeric so existing `channel(...)` selectors are + unchanged, and add `QueryBuilder.poi_channel(*, dtype=PoiValueType.double, + **kwargs)` (see [§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)). + +### 4.2 Query API and carrying the discriminators to solve + +#### `QueryBuilder.poi_channel(...)` + +POI channels are selected through a dedicated **`poi_channel(...)` factory method** +on `QueryBuilder`, parallel to the existing `channel(...)` / `channel_with_alias(...)`: + +```python +def poi_channel(self, *, dtype: PoiValueType = PoiValueType.double, **kwargs) -> TimeSeriesSelector: + # same tag/column matching as channel(...) — builds the selector expr from **kwargs + return TimeSeriesSelector(expr, series_type=POINTS_IN_TIME, value_type=dtype) +``` + +Design points: + +- **No new selector class.** `poi_channel` returns the **same `TimeSeriesSelector`** + that `channel(...)` returns; channel *identification* (tag/column match, + `get_selector_expr`, `required_tags`, `selector_id`, the direct/aliased split) is + identical for POI and sample channels, so there is nothing to override. The method + is a **factory**, not a subclass — it just stamps the selector with its + `series_type` (`POINTS_IN_TIME`) and the caller-declared value `dtype`. +- **Explicit intent at the call site.** `query.poi_channel(channel_name="DTC")` + reads as "this is an event stream, not a signal," and gives POI-only knobs + (the `dtype`) a natural home. `dtype` defaults to `double`, so the common numeric + case stays terse; a string DTC channel is `poi_channel(channel_name="DTC", dtype=string)`. +- **The selector now carries `series_type` + `value_type`.** `TimeSeriesSelector` + gains two optional fields (defaulting to `SAMPLE` / numeric so `channel(...)` is + unchanged). This makes the selector the **plan-time** source of truth for the + series type — which is what simplifies result typing (see [§4.4](#44-result-typing)): + `evaluation_type()` / `dtype()` and the string-op gating work **without** any + pre-pipeline `channel_metrics` lookup, and `string_poi.mean()` can be rejected at + **build time** before Spark is involved. + +:::caution Declared `dtype` is validated against the data, not trusted over it + +The user-declared `dtype` and the silver data are **two sources that must agree**. +The contract is **assertion, not authority**: + +> The check validates against the **data itself**, not a `channel_metrics.series_type` +> column (which was dropped — see [§9](#9-aspects-which-differ-from-the-design)): a POI +> point row carries a null `tend`, so a `poi_channel(...)` that resolves to +> interval-shaped rows is a SAMPLE channel, and an all-null value column exposes a +> declared/actual `dtype` mismatch. + +- The declared `series_type` / `dtype` drive **plan-time** typing and op-gating. +- At **solve time** the data remains authoritative: if the resolved channel's actual + shape **disagrees** with what the selector declared, the solver **raises a clear + error** (mirroring the existing unit-conversion conflict check), rather than silently + reading the wrong value column or overriding the data. + +This keeps the ergonomic win (no plan-time lookup, early validation) without letting +a wrong declaration silently mis-read a channel (e.g. a `dtype=double` hint on a +string channel yielding all-null `value_double`). + +::: + +#### Carrying the discriminators through the pipeline + +`filter_channel_metrics` already reads and column-maps `channel_metrics`. Include +`series_type` in the projected channel-match columns (defaulting null → `SAMPLE` +via `F.coalesce`). It travels alongside `selector_ids` with no effect on any +filter, exactly like the existing per-channel metadata. This solve-time +`series_type` (and, for POI, `dtype`) is what the **assertion check above** +validates the selector's declared values against. + +### 4.3 One unified per-container frame, one UDF, one dispatching cache + +Because sample and POI channels share containers and are mixed in a single +expression, they **must be solved together in one grouped-map UDF per container** +(see the requirement established in [§2](#2-background-why-this-fits-so-cleanly)). +The design keeps the existing single-UDF shape and makes the *cache* series-type +aware, rather than forking the UDF. + +**Step 1 — normalize both sample sources into one Spark frame.** In +`_prepare_channels_join`, read and column-map **both** tables and project them into +a common superset schema keyed by `(container_id, channel_id)`, carrying a +`series_type` discriminator (and, for POI, `dtype`): + +| Column | SAMPLE row source | POI row source | +|----------------|-----------------------|---------------------------------------| +| `container_id` | `channels` | `poi_channels` | +| `channel_id` | `channels` | `poi_channels` | +| `series_type` | `SAMPLE` | `POINTS_IN_TIME` | +| `tstart` | `channels.tstart` | `poi_channels.timestamp` | +| `tend` | `channels.tend` | `null` (POI has no validity interval) | +| `value` | `channels.value` | `poi_channels.value_double` | +| `value_string` | `null` | `poi_channels.value_string` | +| `dtype` | `null` (⇒ numeric) | `poi_channels.dtype` | + +`unionByName` the two projections into a single DataFrame, join it to the +channel-match frame on `(container_id, channel_id)`, then — exactly as today — +`groupBy(container_id).apply(udf)`. Only channels that survived the filter pipeline +are shipped, so the union stays small. A container's sample and POI rows now land in +the **same** pandas frame. + +**Step 2 — a unified cache that dispatches per channel.** Generalize +`TimeSeriesCache` (or add a `UnifiedSeriesCache` that subsumes it) so `load_blob` +inspects the channel slice's `series_type` and builds the right object: + +- `series_type == SAMPLE` → `SampleSeries(tstart, tend, value)` (today's behavior, + unchanged). +- `series_type == POINTS_IN_TIME` and `dtype == double` → numeric + `PointsInTimeSeries(tstart, value)` (the POI timestamp lives in the `tstart` + column of the unified frame). +- `series_type == POINTS_IN_TIME` and `dtype == string` → the string point series + from [§3.4](#34-per-channel-value-dtype-double-vs-string), built from + `(tstart, value_string)`. + +The cache keeps the same `(cid, ch) → (start, stop)` range-index over the sorted +frame; the only change is which columns each slice reads and which class it +instantiates. Because `series_type` and `dtype` are constant per channel, the +dispatch is decided **once** per `(cid, ch)` slice, not per row. + +**Step 3 — expression evaluation is unchanged.** `TimeSeriesSelector.build(cache)` +still just calls `cache.load_blob(...)`; it now transparently gets a `SampleSeries` +or a point series. A mixed expression such as `poi_channel - sample_channel` is +evaluated on the two in-memory objects, and `PointsInTimeSeries._apply_basic_op` +already handles the cross-type case by aligning against the `SampleSeries` at the +POI timestamps via `synchronized`. **No new math and no second UDF.** + +The `series_type` / `dtype` discriminators are carried the same pass-through way as +the existing per-channel metadata (they originate on `channel_metrics` / +`poi_channels`; see [§8](#8-open-questions)), so both the cache and the result-typing +step (§4.4) know each channel's kind without scanning its data. + +:::note Why not two UDFs? + +Splitting SAMPLE and POI into two grouped-map UDFs and unioning their **outputs** +would be simpler to write but is **incorrect** for the common mix-and-match case: +each UDF would receive only a subset of a container's channels, so an expression +referencing one channel of each type could not be evaluated — one operand would +always be missing from that UDF's frame. Unifying the **input** frame and keeping a +single UDF is what makes cross-type expressions work. + +::: + +### 4.4 Result typing + +`QueryBuilder._determine_result_objects_dtypes` builds each selection against an +`EmptyTimeSeriesCache` to learn its result `dtype`. Today `EmptyTimeSeriesCache.load_blob` +always returns an empty `SampleSeries`, so a bare POI selection would be mistyped +as `BinaryType` (the `SampleSeries` serialization dtype) instead of +`PointsInTimeSeries.dtype()` (`ArrayType(ArrayType(DoubleType))`). + +**Because the selector now carries its own `series_type` / `value_type` +([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)), this resolves with +no plan-time metadata lookup.** `EmptyTimeSeriesCache.load_blob` simply consults the +calling selector and returns an empty series of the matching kind: + +- a `SAMPLE` selector → empty `SampleSeries` (today's behavior); +- a numeric POI selector → empty numeric `PointsInTimeSeries`; +- a string POI selector → empty `PointsInTimeSeries` with `value_type = string`. + +`evaluation_type()` / `dtype()` are then correct for bare POI selections and for +expressions whose output type depends on the input type — and the string-op gating +fires **at build time**: `string_poi.mean()` builds an empty string point series +whose `mean()` raises `NotImplementedError`, so the selection is rejected up front +rather than producing a silent `NaN`, before Spark is involved. + +This removes the earlier need to pre-resolve each selector's type from +`channel_metrics` and inject it into the empty cache: the declared type on the +selector *is* the plan-time source. (The silver metadata still has the final say at +solve time via the [§4.2 assertion check](#42-query-api-and-carrying-the-discriminators-to-solve).) +It also mirrors how `PointsInTimeEvent` and `PointValueAggregator` already validate +`evaluation_type()` up front, so the mechanism is consistent with existing code. + +### 4.5 `PointsInTimeSeries` model change + +The one backend-model change (see [§3.4](#34-per-channel-value-dtype-double-vs-string)): + +- Add a second value array (dtype `object`) alongside the existing `float64` array, + and a `value_type` property (numeric / string) selecting which is active. +- Constructors/factories set `value_type`: the numeric path keeps today's + `np.array(values, dtype=np.float64)`; the string path stores values as an `object` + array and leaves the numeric array empty. +- Make `dtype()` return `ArrayType(ArrayType(StringType))` when `value_type` is + string (numeric unchanged). +- Implement **`__eq__` for string series** (synchronize on timestamps → compare + string values → `PointsInTime`). Have `__ne__`, `__lt__`, `__le__`, `__gt__`, + `__ge__`, the arithmetic operators, and the numeric reductions (`sum`, `mean`, + `min`, `max`) **raise `NotImplementedError`** when `value_type` is string. +- Leave `count`, `start_time` / `end_time`, `to_points_in_time`, `plane_sweep`, and + the timestamp handling in `synchronized` value-type-independent (they already are). + +### 4.6 Extend the existing test dataset with DTC POI channels + +Rather than build a bespoke POI fixture, **extend the existing session-scoped silver +dataset** so POI channels live alongside the current sample channels in the **same +containers** — this is what exercises the mix-and-match path (§4.3) end to end and +mirrors the [DTC motivating example](#11-motivating-example-ecu-defect--error-codes-dtcs). +The guiding constraint is **additive, non-destructive**: every existing test must +keep passing untouched. + +The `setup_basic_db` fixture (autouse, session-scoped) loads +`tests/unit/data/basic_narrow_csv/` into `spark_catalog.silver.*`. Use the concrete +rows from [§3.5](#35-example-tag--metric-entries-for-dtc-poi-channels) (DTC string +channel `channel_id = 90`, numeric count channel `channel_id = 91` on +`container_id = 1`) as the fixture data. The plan: + +1. **New `poi_channels` data file.** Add + `basic_narrow_csv/poi_channels.csv` with + `container_id, channel_id, timestamp, value_double, value_string, dtype` and a + couple of **DTC channels** on **existing** `container_id`s (e.g. a `DTC` string + channel with points like `(t₁, "P0301")`, `(t₂, "P0420")`, and a numeric POI + channel such as a fault-occurrence counter). Choose `channel_id`s **not already + used** by that container in `channel_data.csv` so the two sample sources stay + disjoint per the design invariant (a channel lives in exactly one of + `channels` / `poi_channels`). +2. **Append POI rows to `channel_metrics.csv`.** Add one row per new POI channel + carrying the new `series_type = POINTS_IN_TIME` column. **Backfill existing rows + with `series_type = SAMPLE`** (or leave blank and rely on the null ⇒ `SAMPLE` + default — pick one and be consistent). Existing sample channels are unaffected. +3. **Load `poi_channels` in the fixture.** Extend `setup_basic_db` to read the new + CSV and write `spark_catalog.silver.poi_channels`, and add its slot to the + `MeasurementDBConfig` used by the basic-db fixtures (`poi_channels_uri`). Because + `poi_channels_uri` defaults to `None`, **any db config that does not opt in is + unchanged**, so unrelated fixtures/tests see no difference. +4. **EAV + wide tag/metric parity.** So POI channels are *selectable* the same way + in both channel-selection modes: + - **EAV fixtures** (`setup_narrow_db`, `unit_test_csv/`): append POI rows to + `1_channel_tags.csv` (e.g. `channel_name = "DTC"`) and `1_channel_metrics.csv`, + plus any container-level tags/metrics needed, so a + `query.poi_channel(channel_name="DTC", dtype="string")` resolves the POI channel + through the pivot path (identification is identical to `channel(...)`; only the + selector's declared `series_type` / `value_type` differ — [§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)). + - **Wide fixtures** (`basic_narrow_csv`): the `channel_name` column already on + `channel_metrics` covers direct selection; just ensure the appended POI rows + carry a distinct `channel_name` (e.g. `"DTC"`). + +:::caution Two different columns both once called "value type" + +`basic_narrow_csv/channel_metrics.csv` **already** has a `value_type` column holding +values like `DOUBLE` (and the EAV `1_channel_metrics.csv` has `numerical`). That is +the **pre-existing** per-channel data-type column and is **not** the discriminator +this design adds. Keep them separate: + +- **existing `channel_metrics.value_type`** — untouched; describes the value data + type and is not read by the solver for routing. +- **new `channel_metrics.series_type`** — `SAMPLE` / `POINTS_IN_TIME`; routes to + `channels` vs `poi_channels` (§3.2). +- **new `poi_channels.dtype`** — `double` / `string`; selects `value_double` / + `value_string` (§3.4). + +Do **not** overload the existing `value_type` column for either new purpose — the +column names in the fixtures must stay distinct, and existing tests that read +`value_type` must be left as-is. + +::: + +**Regression guard.** Run the full existing suite after extending the fixtures and +confirm it is green *before* adding POI-specific tests (§7). Because the changes are +purely additive — new file, appended rows with a defaulting column, an opt-in table +slot — no existing assertion (row counts, computed means, dimension contents) should +move. If any does, the extension was not additive and must be corrected. + +## 5. What explicitly does **not** change + +- **The 6-stage filter pipeline.** `filter_container_tags` → + `filter_container_metrics` → `filter_channel_tags` → `filter_channel_metrics` → + alias resolution are untouched. POI channels are identified by the *same* + `TimeSeriesSelector` class, tag/column matching, and tag/metric filters as sample + channels — `poi_channel(...)` is a factory over the same selector, not a new + selection path ([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)). +- **`channels` table and RLE/interval encoders.** The sample-series read and + raw→interval encoding are untouched. +- **The sample-series *behavior* in the cache.** `TimeSeriesCache` gains a + per-channel dispatch (§4.3), but for a `SAMPLE` channel it builds the exact same + `SampleSeries(tstart, tend, value)` as today — the sample path's semantics and + output are unchanged. (This is a behavior guarantee, not a "no code changed" + claim: the cache does gain POI-aware branching.) +- **The single grouped-map UDF per container.** The solve stage still groups by + `container_id` and applies one UDF; POI does **not** add a second UDF or a + post-hoc union of two result sets. The input frame is widened to carry both series + types, not the execution model. +- **Persistence / gold layer.** Aggregations over POI series already reduce to + scalars (`mean`, `sum`, `count`, …) or `PointsInTime` events, which the existing + fact/dimension tables already accept (`PointsInTimeEvent`, `PointValueAggregator`). +- **`PointsInTimeSeries` for numeric (`double`) channels.** No new methods needed; + it already implements the full operator/sync/aggregation protocol for `float64` + values. (String POI is the exception — it requires the model change in + [§3.4](#34-per-channel-value-dtype-double-vs-string).) +- **`SampleSeries` interpolation semantics.** This design does not change how + sample-series values are reconstructed within `[tstart, tend)`. Zero-order hold + remains the only interpolation today; adding further interpolation methods later + is an **orthogonal** effort. The distinction that matters for POI is *validity* + (does a value exist between two timestamps at all?), not *which* interpolation is + applied where validity holds. + +## 6. Alternatives considered + +| Alternative | Why not chosen | +|-------------|----------------| +| **Store POI in `channels` with `tend == tstart`** | Overloads the "closed endpoint" meaning of zero-duration rows; forces every reader/encoder to disambiguate a whole POI channel from a sample-series endpoint. | +| **Store POI in the RAW `channels` (timestamp, value) format + a skip-encoding flag** | Couples POI to RAW mode and to the raw→interval encoder; a channel's storage shape would depend on an unrelated `data_type` setting. | +| **Overload the existing `value_type` column as the discriminator** | Conflates value *data type* with *series semantics*; two orthogonal concerns in one column, harder to reason about and to validate. | +| **A new dedicated POI solver class** | Unnecessary — the filter pipeline is shared and identical; only `load_blob` differs. A per-channel branch inside `DefaultSolver.solve` is far less code than a parallel solver. | +| **Two grouped-map UDFs (one SAMPLE, one POI), union the outputs** | **Incorrect** for the common mix-and-match case: each UDF sees only a subset of a container's channels, so an expression combining a POI and a sample channel (e.g. `poi - sample`) has a missing operand. Cross-type `synchronized` must run on both in-memory series inside **one** UDF. | + +## 7. Testing strategy + +Following the repo's fixture-reuse convention (CLAUDE.md → *Testing patterns*). +The POI tests run against the **extended shared dataset from [§4.6](#46-extend-the-existing-test-dataset-with-dtc-poi-channels)** +(DTC channels added to the existing `spark_catalog.silver.*` fixtures) rather than a +throwaway db, so they cover the real read path and the mix-and-match case: + +- Assert on **real computed values**, not row counts: e.g. a numeric POI `mean()` + equals the unweighted mean of the point values (contrast with the duration-weighted + `SampleSeries.mean()`, whose weighting follows from interval validity), proving the + between-point validity is genuinely absent. +- A **string POI** test: `query.poi_channel(channel_name="DTC", dtype="string")` + builds a `PointsInTimeSeries` with `value_type = string`; the **equality comparator** + (`== "P0301"` → `PointsInTime` on matching timestamps) and value-type-independent ops + (`count`, `to_points_in_time`, point sampling) work, while every **other comparator** + (`!=`, `<`, `<=`, `>`, `>=`), the arithmetic operators, and the numeric reductions + (`mean`, `sum`, `min`, `max`) raise `NotImplementedError` — asserted both directly on + the series object and, for a reduction inside a selection, at `evaluation_type()` + **build time** (not as a silent `NaN`, and before Spark runs). +- A **mix-and-match test (the primary correctness case)**: a single container owning + both a SAMPLE channel and a numeric POI channel, selected with `query.channel(...)` + and `query.poi_channel(...)` respectively, with **one expression referencing both** + (`rpm.where(dtc == "P0301")`, and `poi - sample`). This asserts both series land in + the *same* per-container pandas frame, are built by the unified cache, and align via + `synchronized` — the behavior a two-UDF design would break. Assert the computed + values, not just that it runs. +- A **declared-vs-actual `dtype` assertion test** ([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)): + `query.poi_channel(channel_name="DTC", dtype="double")` on a channel whose silver + `dtype` is `string` (or a `poi_channel` on a `SAMPLE` channel) raises a clear error + at solve time — the data stays authoritative, the wrong declaration is not silently + honored. +- A backward-compat test: a `channel_metrics` with no `series_type` column still + solves as SAMPLE, and existing `channel(...)` selections are unaffected by the new + optional selector fields. + +## 8. Open questions + +- **Should `series_type` be validated against the presence of data in the matching + table?** (e.g. a POI-marked channel with rows only in `channels`.) Proposed: + no hard validation initially; document that the marker is authoritative and the + non-matching table is not read for that channel. +- **~~Where should the POI value `dtype` be resolved for planning?~~ (Resolved.)** + The user declares `dtype` on `query.poi_channel(...)` and the selector carries it + ([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)), so plan-time + result typing (§4.4) needs **no** pre-pipeline `poi_channels` / `channel_metrics` + scan. The silver `poi_channels.dtype` remains authoritative at solve time and is + validated against the declared value (assertion contract). *Remaining sub-question:* + should the engine also support **inferring** `dtype` when the user omits it (rather + than defaulting to `double`) — e.g. a cheap `distinct` on `channel_metrics` — for + callers who prefer not to declare it? Proposed: keep the explicit `double` default + for now; add inference only if a concrete need appears. +- **Enforcing the constant-`dtype` invariant.** A channel is entirely numeric or + entirely string — `dtype` is constant per `(container_id, channel_id)` by + contract. This is a settled invariant, not an open question; the only decision is + whether to *defend* it. Proposed: an optional validate-and-raise (like the + unit-conversion conflict check) that flags any channel carrying more than one + distinct `dtype`, so a malformed ingest fails loudly instead of picking an + arbitrary value column. +- **String value column when scaling.** If more non-numeric dtypes appear later + (e.g. `bool`, `int`), revisit whether a typed-column-per-dtype layout still scales + or whether a single `value` string column + cast is preferable. +- **Calculated channels producing POI output.** `solve_calculated_channels` emits a + narrow `[container_id, channel_id, tstart, tend, value]` frame. Emitting a POI + *calculated* channel would need a narrow POI shape (`timestamp, value`). Deferred — + out of scope for ingesting POI *input* series. + +## 9. Aspects which differ from the design + +A few things landed differently than sections 3–4 describe. The shipped code is the +source of truth; those sections are left as the original proposal, and each spot that +changed points here. None of these change what the feature does — they mostly remove +machinery the design added that turned out to be unnecessary once the selector became +the source of truth for a channel's series type. + +### 9.1 No `series_type` column on `channel_metrics` (§3.2) + +The design added a `series_type` marker to `channel_metrics` so the solver could tell a +POI channel from a sample channel. We dropped it. A channel's data lives in exactly one +of `channels` or `poi_channels`, so **which table it comes from already tells us the +series type** — the extra column was redundant, and nothing ever read it at solve time. +Today the only way to get a `PointsInTimeSeries` is to read from `poi_channels`, so the +table membership is a complete answer. + +### 9.2 One value array, type inferred at construction (§3.4, §4.5) + +The design proposed keeping the numeric `float64` array and adding a *second* `object` +array for strings, with a `value_type` property choosing between them. In practice +`PointsInTimeSeries` keeps a **single** value array and infers whether it's string or +numeric from the values at construction time (an `_is_string` flag). It's less +bookkeeping — there's no pair of arrays to keep in sync, one always empty — and it +reads more naturally: you build the series from whatever values you have and it figures +out its own type. An explicit `empty_string()` factory covers the one case inference +can't (an empty series has nothing to infer from). + +### 9.3 String POI also supports `!=` (§3.4, §4.5) + +The design limited string POI series to equality (`==`) and had `!=` raise alongside +the ordering and arithmetic operators. We kept `!=` too. + +### 9.4 The declared-vs-actual check reads the data shape, not a marker (§4.2) + +The design validated the selector's declared `series_type` / `dtype` against the +`channel_metrics.series_type` column. With that column gone (9.1), the solve-time check +instead looks at the **data it resolved to** + +### 9.5 Series-type dispatch is driven by the selector, not a per-row column (§4.3) + +The design's solve stage stamped `series_type` (and `dtype`) onto every channel-data +row so the cache could inspect each slice. Since the selector already knows its own +type, we pass that into `load_blob` instead and drop the per-row markers from the frame +that crosses into the pandas UDF. Only `value_string` still rides along, because that's +real data a string channel needs, not a discriminator. The result is the same object +per channel with a bit less shipped across the Arrow boundary. + +### 9.6 Enum placement (§4.1) + +Minor: the design suggested putting `SeriesType` next to `RawEncoder` in +`solver_config.py`. It lives in `time_series_expression.py` instead, next to +`TimeSeriesSelector` (which carries it) and alongside the new `PoiValueType` enum. That's +where the selector-as-source-of-truth logic reads most naturally. diff --git a/src/impulse_query_engine/analyze/metadata/time_series_expression.py b/src/impulse_query_engine/analyze/metadata/time_series_expression.py index 0977dc3..87bd166 100644 --- a/src/impulse_query_engine/analyze/metadata/time_series_expression.py +++ b/src/impulse_query_engine/analyze/metadata/time_series_expression.py @@ -4,18 +4,50 @@ import operator import zlib from collections.abc import Callable, Iterable +from enum import StrEnum from typing import TYPE_CHECKING, Any import pyspark.sql.types as T import impulse_query_engine.util as U from impulse_query_engine.analyze.metadata.tag_expression import TagExpression +from impulse_query_engine.model.series.points_in_time_series import PointsInTimeSeries from impulse_query_engine.model.series.sample_series import SampleSeries if TYPE_CHECKING: from impulse_query_engine.analyze.query.solvers.series_cache import SeriesCache +class SeriesType(StrEnum): + """How a channel's samples are interpreted (mirrors :class:`RawEncoder`). + + ``SAMPLE`` — the default; ``[tstart, tend)`` intervals over which the value is + *valid* (reconstructed by an interpolation method, zero-order hold today), + backed by :class:`SampleSeries`. + + ``POINTS_IN_TIME`` — ``(tᵢ, vᵢ)`` points valid *only at* their timestamps, no + between-point validity, backed by :class:`PointsInTimeSeries`. + """ + + SAMPLE = "SAMPLE" + POINTS_IN_TIME = "POINTS_IN_TIME" + + +class PoiValueType(StrEnum): + """The value data type of a POI channel — selects its ``poi_channels`` value + column and which in-memory :class:`PointsInTimeSeries` variant is built. + + ``DOUBLE`` — numeric points (``poi_channels.value_double``); the full + arithmetic / ordering / reduction operator set applies. + + ``STRING`` — string points (``poi_channels.value_string``, e.g. DTC codes); + only sampling and equality apply (see :class:`PointsInTimeSeries`). + """ + + DOUBLE = "double" + STRING = "string" + + class RequiresDeserialization: pass @@ -619,7 +651,13 @@ def from_dict(obj: dict) -> TimeSeriesExpression: class TimeSeriesSelector(TimeSeriesExpression, RequiresDeserialization): - def __init__(self, expr, uses_alias: bool = False): + def __init__( + self, + expr, + uses_alias: bool = False, + series_type: SeriesType = SeriesType.SAMPLE, + value_type: PoiValueType = PoiValueType.DOUBLE, + ): """ Initialize a TimeSeriesSelector. @@ -627,18 +665,48 @@ def __init__(self, expr, uses_alias: bool = False): ---------- expr : TagExpression Tag expression to select. + uses_alias : bool, optional + Whether the channel resolves via the channel-alias table. + series_type : SeriesType, optional + How the selected channel's samples are interpreted. ``SAMPLE`` + (default) builds a :class:`SampleSeries` — today's behavior, + unchanged. ``POINTS_IN_TIME`` builds a :class:`PointsInTimeSeries` + (values valid only at their timestamps); identification / matching is + identical, only the built object and its result dtype differ. This is + the plan-time source of truth for the series type (so ``dtype()`` is + correct for a bare POI selection with no per-channel metadata lookup). + value_type : PoiValueType, optional + For a ``POINTS_IN_TIME`` selection, the declared value data type + (``DOUBLE`` / ``STRING``). Ignored for ``SAMPLE``. Drives plan-time + typing and string-op gating; validated against the silver + ``poi_channels.dtype`` at solve time (assertion contract). """ self._expr = expr self._uses_alias = uses_alias + self._series_type = series_type + self._value_type = value_type TimeSeriesExpression.__init__(self, is_single_signal=True) @property def uses_alias(self) -> bool: return self._uses_alias + @property + def series_type(self) -> SeriesType: + return self._series_type + + @property + def value_type(self) -> PoiValueType: + return self._value_type + @property def selector_id(self) -> int: - return zlib.crc32(str(self._expr).encode()) + # Include series_type so a SAMPLE and a POINTS_IN_TIME selection of the + # same tag expression resolve as distinct channels. SAMPLE keeps the + # historical id (bare ``str(expr)`` hash) for backward compatibility. + if self._series_type is SeriesType.SAMPLE: + return zlib.crc32(str(self._expr).encode()) + return zlib.crc32(f"{self._series_type}|{self._expr}".encode()) def dtype(self): """ @@ -647,13 +715,33 @@ def dtype(self): Returns ------- pyspark.sql.types.DataType - Data type (BinaryType). + ``BinaryType`` for a SAMPLE selection (serialized ``SampleSeries``), + or the value-type-aware ``PointsInTimeSeries.dtype()`` for a + POINTS_IN_TIME selection (``array>`` for numeric, + ``array>`` for string). """ + if self._series_type is SeriesType.POINTS_IN_TIME: + return self._empty_points_in_time().dtype() return T.BinaryType() + def _empty_points_in_time(self) -> PointsInTimeSeries: + """Empty POI series carrying this selector's declared value type. + + A string selector must build a string-typed empty series so ``dtype()`` + and the string-op gating (e.g. ``.mean()`` raising) reflect the declared + type before any data is read. + """ + if self._value_type is PoiValueType.STRING: + return PointsInTimeSeries.empty_string() + return PointsInTimeSeries.empty() + def deserialize(self, d): """ - Deserialize sample series after collection/toPandas. + Deserialize a SAMPLE result after collection/toPandas. + + POINTS_IN_TIME results are serialized by ``get_data()`` (a plain + ``[[t, v], ...]`` list) and need no deserialization, so they are returned + as-is; only a SAMPLE (binary) blob is decoded to a :class:`SampleSeries`. Parameters ---------- @@ -662,14 +750,26 @@ def deserialize(self, d): Returns ------- - SampleSeries - Deserialized sample series. + SampleSeries or Any + Deserialized sample series (SAMPLE), else *d* unchanged. """ + if self._series_type is SeriesType.POINTS_IN_TIME: + return d return SampleSeries.deserialize(d) - def build(self, cache: SeriesCache) -> SampleSeries: + def build(self, cache: SeriesCache): """ - Instantiate a SampleSeries from given cache data. + Instantiate the selected series from cache data. + + Resolution is identical regardless of series type — resolve the matching + candidates, take the first ``(container_id, channel_id)``, and let the + cache build the right object. The **data** is authoritative for the built + type: :meth:`TimeSeriesCache.load_blob` returns a + :class:`PointsInTimeSeries` for a ``POINTS_IN_TIME`` slice and a + :class:`SampleSeries` otherwise. The selector's own :attr:`series_type` / + :attr:`value_type` are used only for **plan-time** typing (:meth:`dtype` + against an empty cache), so a bare POI selection types correctly and a + string-only op is rejected before Spark runs. Parameters ---------- @@ -678,16 +778,25 @@ def build(self, cache: SeriesCache) -> SampleSeries: Returns ------- - SampleSeries - Built sample series. + SampleSeries or PointsInTimeSeries """ candidates = cache.resolve(self) if len(candidates) == 0: + if self._series_type is SeriesType.POINTS_IN_TIME: + return self._empty_points_in_time() return SampleSeries.empty() # TODO: select candidate mid = candidates.container_id.iloc[0] cid = candidates.channel_id.iloc[0] - return cache.load_blob(mid, cid, uses_alias=self.uses_alias) + # The selector is the source of truth for the series type: pass it to the + # cache so load_blob builds the right object without a per-row discriminator. + return cache.load_blob( + mid, + cid, + uses_alias=self.uses_alias, + series_type=self._series_type, + value_type=self._value_type, + ) def get_required_tag_exprs(self) -> set[TagExpression]: """ @@ -765,6 +874,8 @@ def as_dict(self) -> dict[str, Any]: obj["type"] = U.name_of(TimeSeriesSelector) obj["expr"] = self._expr.as_dict() obj["uses_alias"] = self._uses_alias + obj["series_type"] = str(self._series_type) + obj["value_type"] = str(self._value_type) return obj @staticmethod @@ -783,7 +894,14 @@ def from_dict(obj: dict): Selector instance. """ expr = TimeSeriesExpression.from_dict(obj["expr"]) - m = TimeSeriesSelector(expr, uses_alias=obj.get("uses_alias", False)) + # Default to SAMPLE / DOUBLE so selectors serialized before POI support + # (no series_type / value_type keys) deserialize unchanged. + m = TimeSeriesSelector( + expr, + uses_alias=obj.get("uses_alias", False), + series_type=SeriesType(obj.get("series_type", SeriesType.SAMPLE)), + value_type=PoiValueType(obj.get("value_type", PoiValueType.DOUBLE)), + ) if "alias" in obj and obj["alias"] is not None: m.alias(obj["alias"]) return m diff --git a/src/impulse_query_engine/analyze/query/query_builder.py b/src/impulse_query_engine/analyze/query/query_builder.py index 26ac47e..a97b465 100644 --- a/src/impulse_query_engine/analyze/query/query_builder.py +++ b/src/impulse_query_engine/analyze/query/query_builder.py @@ -7,7 +7,9 @@ from impulse_query_engine.analyze.metadata.metric_expression import MetricSelector from impulse_query_engine.analyze.metadata.tag_expression import TagSelector from impulse_query_engine.analyze.metadata.time_series_expression import ( + PoiValueType, RequiresDeserialization, + SeriesType, TimeSeriesExpression, TimeSeriesSelector, ) @@ -161,6 +163,48 @@ def channel_with_alias(self, **kwargs) -> TimeSeriesSelector: expr = expr & (TagSelector(k) == str(arg)) return TimeSeriesSelector(expr, uses_alias=True) + def poi_channel( + self, dtype: PoiValueType = PoiValueType.DOUBLE, **kwargs + ) -> TimeSeriesSelector: + """ + Create a Points-in-Time (POI) channel selector. + + Parallel to :meth:`channel` — it builds the **same** ``TimeSeriesSelector`` + from a tag/column match on ``**kwargs`` (e.g. + ``poi_channel(channel_name="DTC")``), differing only in that it is stamped + ``series_type=POINTS_IN_TIME`` (so it solves to a + :class:`~impulse_query_engine.model.series.points_in_time_series.PointsInTimeSeries` + — a value valid only *at* each timestamp — rather than a ``SampleSeries``) + and carries the declared value ``dtype``. + + Channel *identification* (tag/column match, ``get_selector_expr``, + ``required_tags``, ``selector_id``) is identical to :meth:`channel`; only + the built object and its result dtype differ. + + Parameters + ---------- + dtype : PoiValueType, optional + The POI channel's value data type: ``DOUBLE`` (default, numeric) or + ``STRING`` (e.g. DTC codes — only sampling and equality apply). This + declared type drives plan-time result typing and string-op gating; it + is validated against the silver ``poi_channels.dtype`` at solve time + (an actual/declared mismatch raises). + **kwargs : dict + Channel tag-value pairs, matched exactly like :meth:`channel`'s. + + Returns + ------- + TimeSeriesSelector + A selector stamped ``series_type=POINTS_IN_TIME`` with the given value type. + """ + expr = None + for k, arg in kwargs.items(): + if not expr: + expr = TagSelector(k) == str(arg) + else: + expr = expr & (TagSelector(k) == str(arg)) + return TimeSeriesSelector(expr, series_type=SeriesType.POINTS_IN_TIME, value_type=dtype) + def select(self, *args) -> Self: """ Set the selection expressions for the query. diff --git a/src/impulse_query_engine/analyze/query/solvers/blob_solver.py b/src/impulse_query_engine/analyze/query/solvers/blob_solver.py index 6e5bced..833f749 100644 --- a/src/impulse_query_engine/analyze/query/solvers/blob_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/blob_solver.py @@ -49,10 +49,15 @@ def resolve(self, selection): idx = selection._expr.build_pandas(self.df) return self.df[idx] - def load_blob(self, container_id, channel_id, uses_alias: bool = False): + def load_blob( + self, container_id, channel_id, uses_alias: bool = False, series_type=None, value_type=None + ): """ Load a time series blob from disk. + ``series_type`` / ``value_type`` are accepted for interface compatibility + with :class:`SeriesCache`; this blob cache serves only SAMPLE series. + Parameters ---------- container_id : Any diff --git a/src/impulse_query_engine/analyze/query/solvers/default_solver.py b/src/impulse_query_engine/analyze/query/solvers/default_solver.py index 5e2e5d7..ef0a527 100644 --- a/src/impulse_query_engine/analyze/query/solvers/default_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/default_solver.py @@ -11,6 +11,11 @@ from impulse_query_engine.analyze.metadata.metric_expression import MetricExpression from impulse_query_engine.analyze.metadata.tag_expression import TagExpression +from impulse_query_engine.analyze.metadata.time_series_expression import ( + PoiValueType, + SeriesType, +) +from impulse_query_engine.model.series.points_in_time_series import PointsInTimeSeries from impulse_query_engine.model.series.sample_series import SampleSeries from .query_solver import QuerySolver @@ -43,7 +48,11 @@ def __init__(self, pdf, col_map: dict[str, str]): col_map : dict[str, str] Mapping with keys ``"cid"``, ``"ch"``, ``"ts"``, ``"te"``, ``"val"``, ``"conv"`` to the actual column names in *pdf*. The - ``"conv"`` column is optional in *pdf*. + ``"conv"`` column is optional in *pdf*. For a POI (``POINTS_IN_TIME``) + selector, :meth:`load_blob` builds a :class:`PointsInTimeSeries` — the + **selector** (not a per-row column) chooses the series type; the + ``"value_string"`` key names the string value column that a string POI + slice reads. """ self._cid_col = col_map["cid"] self._ch_col = col_map["ch"] @@ -52,6 +61,9 @@ def __init__(self, pdf, col_map: dict[str, str]): self._val_col = col_map["val"] self._conv_col = col_map.get("conv") self._has_conversion = self._conv_col is not None and self._conv_col in pdf.columns + # String POI slices read their value from this column; series-type dispatch + # is driven by the selector passed to load_blob, not a per-row marker. + self._value_string_col = col_map.get("value_string") # *pdf* holds channel data for a whole container, so avoid creating unnecessary copies of the data. meta_cols = [ @@ -98,13 +110,20 @@ def resolve(self, selection): idx = selection._expr.build_pandas(self.mdf) return self.mdf[idx] - def load_blob(self, mid, cid, uses_alias: bool = False): + def load_blob(self, mid, cid, uses_alias: bool = False, series_type=None, value_type=None): """ Load a time series blob from the DataFrame. + The **calling selector** chooses the series type (via *series_type* / + *value_type*), so no per-row discriminator column is needed: a + ``POINTS_IN_TIME`` selector yields a :class:`PointsInTimeSeries` (string- + valued when *value_type* is ``STRING``, else numeric), otherwise a + :class:`SampleSeries`. The declared type is validated against the silver + metadata in the solve prelude, so the data stays authoritative. + When the underlying *pdf* carries a conversion-factor column (the column named by ``col_map["conv"]``) **and** the caller is an - aliased selector (``uses_alias=True``), the returned values are + aliased selector (``uses_alias=True``), the returned SAMPLE values are multiplied by that factor. Direct selectors on the same physical channel always receive raw values — unit conversion is a property of the alias, not of the channel. @@ -118,14 +137,28 @@ def load_blob(self, mid, cid, uses_alias: bool = False): uses_alias : bool, optional ``True`` when the calling selector resolved via channel_mapping. Gates the per-channel conversion factor; defaults to ``False``. + series_type : SeriesType, optional + The calling selector's series type; ``POINTS_IN_TIME`` builds a + :class:`PointsInTimeSeries`. ``None`` (default) => SAMPLE. + value_type : PoiValueType, optional + For a POI selector, its declared value type; ``STRING`` reads the + string value column, otherwise the numeric one. Returns ------- - SampleSeries - The loaded sample series object. + SampleSeries or PointsInTimeSeries """ lo, hi = self._ranges.get((mid, cid), (0, 0)) s = self.pdf.iloc[lo:hi] + + if series_type == SeriesType.POINTS_IN_TIME: + self._assert_poi_data(s, value_type) + if value_type == PoiValueType.STRING: + # value_string is a populated string column, so the constructor + # infers the string value type from it. + return PointsInTimeSeries(s[self._ts_col], s[self._value_string_col]) + return PointsInTimeSeries(s[self._ts_col], s[self._val_col]) + values = s[self._val_col] if self._has_conversion and len(s) > 0 and uses_alias: factor = s[self._conv_col].iloc[0] @@ -133,6 +166,52 @@ def load_blob(self, mid, cid, uses_alias: bool = False): values = values * factor return SampleSeries(s[self._ts_col], s[self._te_col], values) + def _assert_poi_data(self, s, value_type) -> None: + """Validate a POI selector against the data it resolved to. + + The selector drives series-type dispatch, but the silver data stays + authoritative: a ``poi_channel(...)`` selector must land on genuine POI + rows. POI rows carry a null ``tend`` (a point has no validity interval), + whereas SAMPLE rows always carry a real ``tend`` (non-nullable in + ``channels``); so a non-null ``tend`` on a POI-declared slice means the + selector was pointed at a SAMPLE channel. A ``STRING`` declaration + additionally requires a populated ``value_string``. Either mismatch raises + rather than silently reading the wrong column (mirrors the unit-conversion + conflict check). + """ + if len(s) == 0: + return + if pd.notna(s[self._te_col].iloc[0]): + raise ValueError( + "POI channel series-type mismatch: poi_channel(...) resolved to a SAMPLE " + "channel (its rows carry a validity interval). Use channel(...) for SAMPLE " + "channels and poi_channel(...) for POINTS_IN_TIME channels." + ) + + has_string_col = self._value_string_col is not None and self._value_string_col in s.columns + string_all_null = has_string_col and s[self._value_string_col].isna().all() + double_all_null = s[self._val_col].isna().all() + + if value_type == PoiValueType.STRING: + # A string POI channel must carry string values; all-null means the + # channel is actually numeric (declared the wrong dtype). + if not has_string_col or string_all_null: + raise ValueError( + "POI channel dtype mismatch: poi_channel(dtype=string) resolved to a channel " + "with no string values (it is a numeric POI channel). Pass dtype=double to " + "poi_channel(...)." + ) + else: + # A numeric POI channel must carry numeric values; all-null numeric + # with populated string values means the channel is actually a string + # channel (declared the wrong dtype). + if double_all_null and has_string_col and not string_all_null: + raise ValueError( + "POI channel dtype mismatch: poi_channel(dtype=double) resolved to a channel " + "whose numeric values are all null (it is a string POI channel). Pass " + "dtype=string to poi_channel(...)." + ) + class DefaultSolver(QuerySolver): """ @@ -1026,6 +1105,15 @@ def _prepare_channels_join(self, query, channels_df) -> tuple[DataFrame, DataFra self.config.value_col, ) + # POI channel data is unioned in AFTER RLE encoding above, so its + # zero-duration points are never run-length merged. The inner join to + # channels_df below drops any POI rows whose channel was not selected, so + # unioning whenever a poi_channels table is configured is correct (a + # pure-SAMPLE query simply matches no POI channel_ids). Which object each + # channel builds is decided by the selector (passed to load_blob), not a + # per-row marker — SAMPLE rows just lack value_string. + q = self._union_poi_channel_data(query, q) + joined_df = q.join( F.broadcast(channels_df), on=[self.config.container_id_col, self.config.channel_id_col], @@ -1033,6 +1121,35 @@ def _prepare_channels_join(self, query, channels_df) -> tuple[DataFrame, DataFra container_count = channels_df.select(self.config.container_id_col).distinct().count() return q, joined_df, container_count + def _union_poi_channel_data(self, query, channels_q: DataFrame) -> DataFrame: + """Union POI channel-data rows into the (already-encoded) channel-data frame. + + Reads ``poi_channels``, column-maps it, and projects it into the SAMPLE + channel-data superset — POI ``timestamp`` becomes ``tstart`` (``tend`` + **null**, since a point has no validity interval, which is also the signal + the cache validates a POI selector against), ``value_double`` becomes the + numeric ``value`` column, and ``value_string`` rides alongside for a string + POI channel. No per-row ``series_type`` / ``dtype`` marker is shipped: the + selector drives series-type dispatch in :meth:`TimeSeriesCache.load_blob`. + Returns *channels_q* unchanged when no ``poi_channels`` table is configured. + """ + db = query.db + if not (hasattr(db, "has_poi_channels") and db.has_poi_channels()): + return channels_q + + cfg = self.config + poi = db.poi_channels(self.spark) + poi = self._apply_column_mapping(poi, cfg.poi_channels.column_name_mapping) + poi_proj = poi.select( + F.col(cfg.container_id_col), + F.col(cfg.channel_id_col), + F.col(cfg.poi_timestamp_col).alias(cfg.tstart_col), + F.lit(None).cast(T.LongType()).alias(cfg.tend_col), + F.col(cfg.poi_value_double_col).alias(cfg.value_col), + F.col(cfg.poi_value_string_col).alias(cfg.poi_value_string_col), + ) + return channels_q.unionByName(poi_proj, allowMissingColumns=True) + def _apply_grouped_map(self, joined_df, container_count, schema, solve_udf) -> DataFrame: """Run *solve_udf* per container, or return an empty frame when none match.""" if container_count == 0: diff --git a/src/impulse_query_engine/analyze/query/solvers/empty_cache.py b/src/impulse_query_engine/analyze/query/solvers/empty_cache.py index 32ca0d1..9664126 100644 --- a/src/impulse_query_engine/analyze/query/solvers/empty_cache.py +++ b/src/impulse_query_engine/analyze/query/solvers/empty_cache.py @@ -25,7 +25,7 @@ def resolve(self, selection): """ return [] - def load_blob(self, mid, cid, uses_alias: bool = False): + def load_blob(self, mid, cid, uses_alias: bool = False, series_type=None, value_type=None): """ Return an empty SampleSeries for any container and channel ID. @@ -38,6 +38,11 @@ def load_blob(self, mid, cid, uses_alias: bool = False): uses_alias : bool, optional Unused by this cache; accepted for interface compatibility with :class:`SeriesCache`. + series_type, value_type : optional + Accepted for interface compatibility. The empty-series typing for a + POI selector is handled by ``TimeSeriesSelector.build`` (its length-0 + branch), which returns the correctly typed empty series without + reaching this method. Returns ------- diff --git a/src/impulse_query_engine/analyze/query/solvers/series_cache.py b/src/impulse_query_engine/analyze/query/solvers/series_cache.py index 081d53b..8debf8d 100644 --- a/src/impulse_query_engine/analyze/query/solvers/series_cache.py +++ b/src/impulse_query_engine/analyze/query/solvers/series_cache.py @@ -24,7 +24,14 @@ def resolve(self, selection) -> pd.DataFrame: pass @abstractmethod - def load_blob(self, mid, cid, uses_alias: bool = False) -> SampleSeries: + def load_blob( + self, + mid, + cid, + uses_alias: bool = False, + series_type=None, + value_type=None, + ) -> SampleSeries: """ Resolve given mid and cid to a series. @@ -41,10 +48,22 @@ def load_blob(self, mid, cid, uses_alias: bool = False) -> SampleSeries: conversion factor when this is ``True``, so a direct selector on the same physical channel always returns raw values. Defaults to ``False`` (direct / no-conversion semantics). + series_type : SeriesType, optional + The calling selector's series type. The selector — not a per-row + data column — is the source of truth for which object to build: + ``POINTS_IN_TIME`` builds a :class:`PointsInTimeSeries`, otherwise a + :class:`SampleSeries`. ``None`` (default) means SAMPLE, so callers + that predate POI are unchanged. + value_type : PoiValueType, optional + For a ``POINTS_IN_TIME`` selector, its declared value type + (``DOUBLE`` / ``STRING``) — selects the numeric vs string value + column. Ignored for SAMPLE. The declared type is validated against + the silver metadata in the solve prelude, so the data stays + authoritative. Returns ------- - SampleSeries - The loaded sample series object. + SampleSeries or PointsInTimeSeries + The loaded series object. """ pass diff --git a/src/impulse_query_engine/analyze/query/solvers/solver_config.py b/src/impulse_query_engine/analyze/query/solvers/solver_config.py index 0ca7113..0eb1122 100644 --- a/src/impulse_query_engine/analyze/query/solvers/solver_config.py +++ b/src/impulse_query_engine/analyze/query/solvers/solver_config.py @@ -143,6 +143,7 @@ class SolverConfig(BaseModel): channel_metrics: TableConfig = TableConfig() channel_mapping: ChannelMappingConfig = ChannelMappingConfig() channels: TableConfig = TableConfig() + poi_channels: TableConfig = TableConfig() unit_conversion: TableConfig = TableConfig() # ------------------------------------------------------------------ @@ -231,6 +232,26 @@ def value_col(self) -> str: """Internal column name for the signal value on the channels table.""" return "value" + @property + def poi_timestamp_col(self) -> str: + """Internal column name for the point timestamp on the poi_channels table.""" + return "timestamp" + + @property + def poi_value_double_col(self) -> str: + """Internal column name for the numeric value on the poi_channels table.""" + return "value_double" + + @property + def poi_value_string_col(self) -> str: + """Internal column name for the string value on the poi_channels table.""" + return "value_string" + + @property + def poi_dtype_col(self) -> str: + """Internal column name for the per-row value-dtype discriminator on poi_channels.""" + return "dtype" + @property def tag_key_col(self) -> str: """Internal column name for the attribute key on the container_tags (EAV) table.""" @@ -379,4 +400,8 @@ def col_map(self) -> dict[str, str]: "te": self.tend_col, "val": self.value_col, "conv": self.conversion_factor_col, + # String POI slices read their value from this column. Series-type + # dispatch is driven by the selector (passed to load_blob), so no + # per-row series_type / dtype marker column is needed in the frame. + "value_string": self.poi_value_string_col, } diff --git a/src/impulse_query_engine/measurement_db.py b/src/impulse_query_engine/measurement_db.py index c0ba27c..3ebdcde 100644 --- a/src/impulse_query_engine/measurement_db.py +++ b/src/impulse_query_engine/measurement_db.py @@ -14,6 +14,7 @@ def __init__( channel_tags_table=None, channel_metrics_table=None, channels_uri=None, + poi_channels_uri=None, channel_mapping_table=None, unit_conversion_table=None, table_locations: str = "external_locations", @@ -23,6 +24,9 @@ def __init__( self.channel_tags_table = channel_tags_table self.channel_metrics_table = channel_metrics_table self.channels_uri = channels_uri + # Optional Points-in-Time (POI) channel-data table. ``None`` means no POI + # channels are configured, so POI-unaware deployments are unchanged. + self.poi_channels_uri = poi_channels_uri self.channel_mapping_table = channel_mapping_table self.unit_conversion_table = unit_conversion_table self.table_locations = table_locations @@ -34,6 +38,7 @@ def for_unity_catalog( core_schema_name: str = "core", channel_mapping_table: str | None = None, unit_conversion_table: str | None = None, + poi_channels_uri: str | None = None, ): return MeasurementDBConfig( container_tags_table=f"{catalog_name}.{core_schema_name}.container_tags", @@ -41,6 +46,7 @@ def for_unity_catalog( channel_tags_table=f"{catalog_name}.{core_schema_name}.channel_tags", channel_metrics_table=f"{catalog_name}.{core_schema_name}.channel_metrics", channels_uri=f"{catalog_name}.{core_schema_name}.channels", + poi_channels_uri=poi_channels_uri, channel_mapping_table=channel_mapping_table, unit_conversion_table=unit_conversion_table, table_locations="unity_catalog", @@ -58,6 +64,7 @@ def for_debug(debug_tables): "channel_metrics" if "channel_metrics" in debug_tables else None ), channels_uri="channels" if "channels" in debug_tables else None, + poi_channels_uri="poi_channels" if "poi_channels" in debug_tables else None, channel_mapping_table=( "channel_mapping" if "channel_mapping" in debug_tables else None ), @@ -103,6 +110,20 @@ def channel_metrics(self, spark) -> DataFrame: def channels(self, spark) -> DataFrame: return self._read_table(spark, self.config.channels_uri) + def has_poi_channels(self) -> bool: + """Whether a Points-in-Time (POI) channel-data table is configured.""" + return getattr(self.config, "poi_channels_uri", None) is not None + + def poi_channels(self, spark) -> DataFrame: + """Read the Points-in-Time (POI) channel-data table. + + Parallel to :meth:`channels`. Raises if no ``poi_channels_uri`` is + configured — callers should gate on :meth:`has_poi_channels` first. + """ + if not self.has_poi_channels(): + raise ValueError("poi_channels_uri is not configured") + return self._read_table(spark, self.config.poi_channels_uri) + def channel_mapping(self, spark) -> DataFrame: if self.config.channel_mapping_table is None: raise ValueError("channel_mapping_table is not configured") diff --git a/src/impulse_query_engine/model/series/points_in_time_series.py b/src/impulse_query_engine/model/series/points_in_time_series.py index 192a73d..44d8dcd 100644 --- a/src/impulse_query_engine/model/series/points_in_time_series.py +++ b/src/impulse_query_engine/model/series/points_in_time_series.py @@ -53,6 +53,11 @@ def __init__(self, tstarts: Sized, values: Sized): a value is only defined *at* its timestamp and is not considered valid in between consecutive timestamps. + The value type (numeric vs string) is inferred from *values*. An **empty** + series has no values to infer from and therefore defaults to numeric; use + :meth:`empty_string` when an explicitly string-typed empty series is needed + (e.g. plan-time result typing of a bare string-POI selection). + Parameters ---------- tstarts : Sized @@ -65,8 +70,6 @@ def __init__(self, tstarts: Sized, values: Sized): # string-valued series support sampling (``synchronized`` / ``.where``) # and equality comparisons (``==`` / ``!=``) only — arithmetic, ordering # and numeric reductions are rejected (see the ``@_numeric_only`` methods). - # An empty series has no observed value type, so it defaults to numeric - # (the safe, backward-compatible case). self.tstarts = np.array(tstarts, dtype=np.float64) self._is_string = np.asarray(values).dtype.kind in ("U", "S", "O") if self._is_string: @@ -604,11 +607,34 @@ def __repr__(self) -> str: @staticmethod def empty() -> PointsInTimeSeries: """ - Returns an empty PointsInTimeSeries. + Returns an empty (numeric) PointsInTimeSeries. Returns ------- PointsInTimeSeries - Empty PointsInTimeSeries object. + Empty numeric PointsInTimeSeries object. """ return PointsInTimeSeries([], []) + + @staticmethod + def empty_string() -> PointsInTimeSeries: + """ + Returns an empty **string-valued** PointsInTimeSeries. + + An empty series has no values to infer a type from, so the constructor + defaults to numeric; this factory forces the string value type. Used for + plan-time result typing of a bare string-POI selection, where the empty + series must report the string ``dtype()`` and reject numeric-only ops + (e.g. ``mean()``) before any data is read. + + Returns + ------- + PointsInTimeSeries + Empty string-valued PointsInTimeSeries object. + """ + # A single-element object array makes the constructor infer string, then + # slice back to empty so no value is retained. + series = PointsInTimeSeries([], []) + series._is_string = True + series.values = np.asarray([], dtype=object) + return series diff --git a/src/impulse_query_engine/schema.py b/src/impulse_query_engine/schema.py index 8323182..f64465b 100644 --- a/src/impulse_query_engine/schema.py +++ b/src/impulse_query_engine/schema.py @@ -52,6 +52,26 @@ ] ) +# Points-in-Time (POI) channel samples: a value defined only *at* its timestamp +# (no derived tend / validity interval). Two typed value columns plus a per-row +# dtype discriminator, since a POI value may be numeric or a string; exactly one of +# value_double / value_string is populated per row, selected by dtype. +# +# A channel is a POI channel iff its data lives here rather than in ``channels`` — +# table membership *is* the series-type discriminator, so no ``series_type`` column +# is needed on ``channel_metrics``. A given (container_id, channel_id) lives in +# exactly one of ``channels`` / ``poi_channels``. +POI_CHANNELS_SCHEMA = T.StructType( + [ + T.StructField("container_id", T.LongType(), nullable=False), + T.StructField("channel_id", T.IntegerType(), nullable=False), + T.StructField("timestamp", T.LongType(), nullable=False), + T.StructField("value_double", T.DoubleType()), + T.StructField("value_string", T.StringType()), + T.StructField("dtype", T.StringType(), nullable=False), + ] +) + CHANNELS_SCHEMA = T.StructType( [ T.StructField("container_id", T.LongType(), nullable=False), diff --git a/src/impulse_reporting/config/config_parser.py b/src/impulse_reporting/config/config_parser.py index bef26e3..ffcc5f9 100644 --- a/src/impulse_reporting/config/config_parser.py +++ b/src/impulse_reporting/config/config_parser.py @@ -150,6 +150,7 @@ class Source(BaseModel): channel_mapping_table: Annotated[str, AfterValidator(is_valid_table_name)] | None = None unit_conversion_table: Annotated[str, AfterValidator(is_valid_table_name)] | None = None + #todo probably add poi here as well so users can configure it class UnitySink(BaseModel): """ diff --git a/tests/conftest.py b/tests/conftest.py index 0ea8731..d571329 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,6 +47,7 @@ def basic_narrow_db(spark, mock_workspace_client) -> MeasurementDB: tables["container_metrics"] = spark.read.table("spark_catalog.silver.container_metrics") tables["channel_metrics"] = spark.read.table("spark_catalog.silver.channel_metrics") tables["channels"] = spark.read.table("spark_catalog.silver.channels") + tables["poi_channels"] = spark.read.table("spark_catalog.silver.poi_channels") cfg = MeasurementDBConfig.for_debug(tables) return MeasurementDB(cfg, ws=mock_workspace_client) @@ -96,6 +97,20 @@ def setup_narrow_db(spark): ), schema=S.CHANNELS_SCHEMA, ) + poi_channels = spark.createDataFrame( + pd.read_csv( + f"{base_path}/tests/unit/data/unit_test_csv/1_poi_channels.csv", + dtype={ + "container_id": np.int64, + "channel_id": np.int32, + "timestamp": np.longlong, + "value_double": np.float64, + "value_string": "object", + "dtype": "object", + }, + ), + schema=S.POI_CHANNELS_SCHEMA, + ) container_tags.write.format("delta").mode("overwrite").saveAsTable( "spark_catalog.silver_narrow_db.container_tags" @@ -112,6 +127,9 @@ def setup_narrow_db(spark): channels.write.format("delta").mode("overwrite").saveAsTable( "spark_catalog.silver_narrow_db.channels" ) + poi_channels.write.format("delta").mode("overwrite").saveAsTable( + "spark_catalog.silver_narrow_db.poi_channels" + ) @pytest.fixture(scope="session", autouse=True) @@ -131,11 +149,20 @@ def setup_basic_db(spark): container_metric_path = f"{base_path}/tests/unit/data/basic_narrow_csv/container_metrics.csv" channel_metric_path = f"{base_path}/tests/unit/data/basic_narrow_csv/channel_metrics.csv" channels_path = f"{base_path}/tests/unit/data/basic_narrow_csv/channel_data.csv" + poi_channels_path = f"{base_path}/tests/unit/data/basic_narrow_csv/poi_channels.csv" options = {"header": "True", "delimiter": ",", "inferSchema": "True"} container_metrics = spark.read.options(**options).csv(container_metric_path) channel_metrics = spark.read.options(**options).csv(channel_metric_path) channels = spark.read.options(**options).csv(channels_path) + # POI channel data: explicit schema so empty value columns keep their nullable + # typed shape (value_double double / value_string string) rather than being + # inferred as all-null strings. + poi_channels = ( + spark.read.schema(S.POI_CHANNELS_SCHEMA) + .options(header="True", delimiter=",") + .csv(poi_channels_path) + ) container_metrics.write.format("delta").mode("overwrite").saveAsTable( "spark_catalog.silver.container_metrics" @@ -150,6 +177,9 @@ def setup_basic_db(spark): "spark_catalog.silver.channel_metrics" ) channels.write.format("delta").mode("overwrite").saveAsTable("spark_catalog.silver.channels") + poi_channels.write.format("delta").mode("overwrite").saveAsTable( + "spark_catalog.silver.poi_channels" + ) @pytest.fixture(scope="session") @@ -236,6 +266,7 @@ def narrow_db(spark, setup_narrow_db, mock_workspace_client) -> MeasurementDB: "spark_catalog.silver_narrow_db.channel_metrics" ) debug_tables["channels"] = spark.read.table("spark_catalog.silver_narrow_db.channels") + debug_tables["poi_channels"] = spark.read.table("spark_catalog.silver_narrow_db.poi_channels") cfg = MeasurementDBConfig.for_debug(debug_tables) return MeasurementDB(cfg, ws=mock_workspace_client) diff --git a/tests/impulse_query_engine/integration/poi_channel_solve_test.py b/tests/impulse_query_engine/integration/poi_channel_solve_test.py new file mode 100644 index 0000000..8b032ff --- /dev/null +++ b/tests/impulse_query_engine/integration/poi_channel_solve_test.py @@ -0,0 +1,243 @@ +"""End-to-end integration tests for Points-in-Time (POI) channels. + +Exercises the full solve pipeline for ``query.poi_channel(...)`` against the shared +``basic_narrow_db`` (wide) and ``narrow_db`` (EAV) fixtures, which carry POI channels +on ``container_id = 1`` alongside the existing sample channels (see conftest / +``poi_channels.csv``): + +- ``channel_id = 90`` — a **string** DTC-code channel (``P0301`` / ``P0420`` / ``P0301``) +- ``channel_id = 91`` — a **numeric** DTC-count channel (values ``1, 2, 3``) + +Covers: numeric POI unweighted reductions, string POI equality + op gating, the +mix-and-match case (a SAMPLE and a POI channel in one expression), the declared-vs-actual +dtype/series-type assertion, and SAMPLE backward-compatibility. +""" + +import math + +import pytest +import pyspark.sql.types as T +from pyspark.sql import SparkSession + +from impulse_query_engine.analyze.metadata.time_series_expression import PoiValueType +from impulse_query_engine.analyze.query.solvers.default_solver import DefaultSolver +from impulse_query_engine.measurement_db import MeasurementDB + + +class TestNumericPoi: + def test_numeric_poi_mean_is_unweighted(self, spark: SparkSession, basic_narrow_db): + """A numeric POI ``mean()`` is the plain (unweighted) mean of the point values — + POI points have no duration to weight by, unlike ``SampleSeries.mean()``.""" + solver = DefaultSolver(spark) + q = basic_narrow_db.query + dtc_count = q.poi_channel(channel_name="DTC_count") # values 1, 2, 3 + + result = q.select(dtc_count.mean().alias("m")).solve(spark=spark, solver=solver) + + rows = {r.container_id: r.m for r in result.collect()} + assert rows[1] == 2.0 # unweighted mean of (1, 2, 3) + + def test_numeric_poi_sum_and_count(self, spark: SparkSession, basic_narrow_db): + solver = DefaultSolver(spark) + q = basic_narrow_db.query + dtc_count = q.poi_channel(channel_name="DTC_count") + + result = q.select( + dtc_count.sum().alias("s"), + dtc_count.count().alias("c"), + ).solve(spark=spark, solver=solver) + + row = {r.container_id: r for r in result.collect()}[1] + assert row.s == 6.0 # 1 + 2 + 3 + assert row.c == 3 + + def test_bare_numeric_poi_selection_types_as_points_in_time( + self, spark: SparkSession, basic_narrow_db + ): + """A bare numeric POI selection serializes as ``array>`` + (PointsInTimeSeries), not the SAMPLE ``binary`` blob type.""" + solver = DefaultSolver(spark) + q = basic_narrow_db.query + dtc_count = q.poi_channel(channel_name="DTC_count").alias("pit") + + result = q.select(dtc_count).solve(spark=spark, solver=solver) + + assert result.schema["pit"].dataType == T.ArrayType(T.ArrayType(T.DoubleType())) + rows = {r.container_id: r.pit for r in result.collect()} + # three points [t, v], values 1..3 (unweighted, in timestamp order) + assert [pt[1] for pt in rows[1]] == [1.0, 2.0, 3.0] + + +class TestStringPoi: + def test_string_poi_equality_selects_matching_instants( + self, spark: SparkSession, basic_narrow_db + ): + """``string_poi == "P0301"`` yields the instants where the code equals P0301. + + Sampling the count channel at those instants (via ``.where``) picks out the two + P0301 occurrences, proving the string equality drove the point selection. + """ + solver = DefaultSolver(spark) + q = basic_narrow_db.query + dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + + # DTC == "P0301" is a PointsInTime; serialize it directly. + result = q.select((dtc == "P0301").alias("hits")).solve(spark=spark, solver=solver) + + rows = {r.container_id: r.hits for r in result.collect()} + # P0301 occurs at the 1st and 3rd of the three DTC timestamps. + assert len(rows[1]) == 2 + + def test_string_poi_count_and_sampling_allowed(self, spark: SparkSession, basic_narrow_db): + solver = DefaultSolver(spark) + q = basic_narrow_db.query + dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + + result = q.select(dtc.count().alias("c")).solve(spark=spark, solver=solver) + + assert {r.container_id: r.c for r in result.collect()}[1] == 3 + + @pytest.mark.parametrize("reduction", ["mean", "sum", "min", "max"]) + def test_string_poi_numeric_reduction_rejected_at_build( + self, spark: SparkSession, basic_narrow_db, reduction + ): + """A numeric reduction on a string POI selection is rejected at plan/build time + (before Spark runs), not as a silent NaN.""" + q = basic_narrow_db.query + dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + selection = getattr(dtc, reduction)().alias("bad") + with pytest.raises(TypeError, match="string-valued"): + q.select(selection)._determine_result_objects_dtypes() + + +class TestMixAndMatch: + """The primary correctness case: a SAMPLE and a POI channel in one expression, both + in the same per-container pandas frame, aligned via ``synchronized``.""" + + def test_sample_channel_sampled_at_poi_instants(self, spark: SparkSession, narrow_db): + """Sample the ``seed`` SAMPLE channel at the instants of the numeric POI channel. + + narrow_db container 1: ``seed`` sample channel has values 1..10 over t=0..10; the + numeric POI channel (91) has points at t = 2, 5, 8. Sampling seed at those instants + picks the seed values valid there. + """ + solver = DefaultSolver(spark) + q = narrow_db.query + seed = q.channel(seed="0") + dtc_count = q.poi_channel(channel_name="DTC_count") + + # Sample the sample-series at the POI points (cross-type synchronize). + result = q.select(seed.where(dtc_count.to_points_in_time()).alias("sampled")).solve( + spark=spark, solver=solver + ) + + rows = {r.container_id: r.sampled for r in result.collect()} + # three sampled points at the POI instants t = 2, 5, 8 + assert [pt[0] for pt in rows[1]] == [2.0, 5.0, 8.0] + for pt in rows[1]: + assert not math.isnan(pt[1]) + + def test_string_poi_and_sample_freeze_frame(self, spark: SparkSession, narrow_db): + """Freeze-frame: sample the seed channel at the instants where DTC == "P0301".""" + solver = DefaultSolver(spark) + q = narrow_db.query + seed = q.channel(seed="0") + dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + + result = q.select(seed.where(dtc == "P0301").alias("frozen")).solve( + spark=spark, solver=solver + ) + + rows = {r.container_id: r.frozen for r in result.collect()} + # P0301 at t = 2 and 8 (EAV fixture); seed sampled there. + assert [pt[0] for pt in rows[1]] == [2.0, 8.0] + + def test_sample_channel_sampled_at_poi_instants_wide( + self, spark: SparkSession, basic_narrow_db + ): + """Wide-mode counterpart of the mix-and-match case (``basic_narrow_db``). + + Uses a POI numeric channel to filter/sample a real sample channel. Only + "Ambient Air Temperature" (channel 6) spans all three POI instants in the + ``basic_narrow_csv`` fixture (the other channels end earlier), so it is the + channel sampled here. Proves POI-drives-channel-selection works through the + wide (columns-on-channel_metrics) path, not just the EAV pivot path. + """ + solver = DefaultSolver(spark) + q = basic_narrow_db.query + amb = q.channel(channel_name="Ambient Air Temperature") + dtc_count = q.poi_channel(channel_name="DTC_count") # points at 3 POI instants + + result = q.select( + amb.where(dtc_count.to_points_in_time()).alias("sampled") + ).solve(spark=spark, solver=solver) + + rows = {r.container_id: r.sampled for r in result.collect()} + # The three POI instants (microsecond epochs) from basic_narrow_csv/poi_channels.csv. + poi_instants = [1499929300000000.0, 1499931000000000.0, 1499933000000000.0] + assert [pt[0] for pt in rows[1]] == poi_instants + # Each instant sampled a real Ambient-Air-Temp value (not a miss / NaN). + for pt in rows[1]: + assert not math.isnan(pt[1]) + + def test_string_poi_freeze_frame_wide(self, spark: SparkSession, basic_narrow_db): + """Wide-mode freeze-frame: sample "Ambient Air Temperature" where DTC == "P0301". + + In ``basic_narrow_csv`` P0301 occurs at the 1st and 3rd DTC instants, so the + string-POI equality predicate selects exactly those two instants of the + sample channel — the freeze-frame case resolved via the wide channel path. + """ + solver = DefaultSolver(spark) + q = basic_narrow_db.query + amb = q.channel(channel_name="Ambient Air Temperature") + dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + + result = q.select(amb.where(dtc == "P0301").alias("frozen")).solve( + spark=spark, solver=solver + ) + + rows = {r.container_id: r.frozen for r in result.collect()} + # P0301 at the 1st and 3rd instants (basic_narrow_csv/poi_channels.csv). + assert [pt[0] for pt in rows[1]] == [1499929300000000.0, 1499933000000000.0] + for pt in rows[1]: + assert not math.isnan(pt[1]) + + +class TestDeclaredVsActual: + def test_poi_channel_declared_double_on_string_channel_raises( + self, spark: SparkSession, basic_narrow_db + ): + """Declaring ``dtype=double`` on a channel whose silver dtype is ``string`` raises + at solve time — the data stays authoritative.""" + solver = DefaultSolver(spark) + q = basic_narrow_db.query + # DTC is a numeric-less (string) channel; declaring double resolves rows + # whose value_double is all null → dtype mismatch raised in the solve UDF. + bad = q.poi_channel(channel_name="DTC", dtype=PoiValueType.DOUBLE) + with pytest.raises(Exception, match="dtype mismatch"): + q.select(bad.count().alias("c")).solve(spark=spark, solver=solver).collect() + + def test_poi_channel_on_sample_channel_raises(self, spark: SparkSession, basic_narrow_db): + """``poi_channel`` on a SAMPLE channel raises the series-type mismatch. + + The SAMPLE channel's rows carry a real (non-null) validity interval, which + is the signal load_blob validates a POI-declared selector against. + """ + solver = DefaultSolver(spark) + q = basic_narrow_db.query + bad = q.poi_channel(channel_name="Engine RPM") + with pytest.raises(Exception, match="series-type mismatch"): + q.select(bad.count().alias("c")).solve(spark=spark, solver=solver).collect() + + +class TestBackwardCompat: + def test_sample_channel_unaffected_by_poi(self, spark: SparkSession, basic_narrow_db): + """An ordinary SAMPLE ``channel(...)`` selection is unchanged by POI support.""" + solver = DefaultSolver(spark) + q = basic_narrow_db.query + rpm = q.channel(channel_name="Engine RPM") + + result = q.select(rpm.mean().alias("rpm_mean")).solve(spark=spark, solver=solver) + + rows = {r.container_id for r in result.collect()} + assert rows == {1, 2, 3} diff --git a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_column_mapping_test.py b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_column_mapping_test.py index 0e53a80..b236f0e 100644 --- a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_column_mapping_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_column_mapping_test.py @@ -611,6 +611,7 @@ def test_col_map_always_returns_internal_names(self, spark): "te": "tend", "val": "value", "conv": "conversion_factor", + "value_string": "value_string", } def test_mapping_entries_stored_correctly(self, spark): diff --git a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_only_test.py b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_only_test.py index 9ab2b68..b74b1cd 100644 --- a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_only_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_only_test.py @@ -505,6 +505,7 @@ def test_col_map_always_returns_internal_names(self, spark: SparkSession): "te": "tend", "val": "value", "conv": "conversion_factor", + "value_string": "value_string", } def test_config_properties_return_internal_names(self, spark: SparkSession): diff --git a/tests/impulse_query_engine/unit/analyze/query/solvers/solver_config_test.py b/tests/impulse_query_engine/unit/analyze/query/solvers/solver_config_test.py index c692c2e..524b97d 100644 --- a/tests/impulse_query_engine/unit/analyze/query/solvers/solver_config_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/solvers/solver_config_test.py @@ -38,6 +38,7 @@ "te": "tend", "val": "value", "conv": "conversion_factor", + "value_string": "value_string", } @@ -148,7 +149,15 @@ class TestColMap: def test_col_map_keys(self, cfg: SolverConfig): """col_map should contain exactly the expected short keys.""" - assert set(cfg.col_map.keys()) == {"cid", "ch", "ts", "te", "val", "conv"} + assert set(cfg.col_map.keys()) == { + "cid", + "ch", + "ts", + "te", + "val", + "conv", + "value_string", + } def test_col_map_default_config(self): """Default SolverConfig col_map should match hardcoded defaults.""" @@ -160,6 +169,7 @@ def test_col_map_default_config(self): "te": "tend", "val": "value", "conv": "conversion_factor", + "value_string": "value_string", } def test_col_map_consistent_with_properties(self, cfg: SolverConfig): diff --git a/tests/unit/data/basic_narrow_csv/channel_metrics.csv b/tests/unit/data/basic_narrow_csv/channel_metrics.csv index d43f432..887cb03 100644 --- a/tests/unit/data/basic_narrow_csv/channel_metrics.csv +++ b/tests/unit/data/basic_narrow_csv/channel_metrics.csv @@ -11,3 +11,5 @@ container_id,channel_id,channel_name,group_idx,channel_idx,unit,sample_count,min 2,6,Ambient Air Temperature,2,2,C,57240,21,33,28.793081761006288,1499367269349000,1499372240481000,4971132000,11.514480001738036,DOUBLE 1,7,Vehicle Speed Sensor,3,1,km/h,59625,0,217,68.22906498951782,1499929242072000,1499934640063000,5397991000,11.045776104480352,DOUBLE 1,5,Engine RPM,2,1,RPM,59625,0,3658,1490.707790356394,1499929242072000,1499934640063000,5397991000,11.045776104480352,DOUBLE +1,90,DTC,0,0,,3,,,,1499929300000000,1499933000000000,3700000,,STRING +1,91,DTC_count,0,0,,3,1,3,2.0,1499929300000000,1499933000000000,3700000,,DOUBLE diff --git a/tests/unit/data/unit_test_csv/1_channel_metrics.csv b/tests/unit/data/unit_test_csv/1_channel_metrics.csv index 46dc975..59749d5 100644 --- a/tests/unit/data/unit_test_csv/1_channel_metrics.csv +++ b/tests/unit/data/unit_test_csv/1_channel_metrics.csv @@ -1,2 +1,4 @@ container_id,channel_id,value_type,sample_count,nan_ratio,begin_s,end_s,duration_ms,original_sample_count,original_sr,min,max,mean,std,pz1,pz10,pz90,pz99 1,1,numerical,1,1.0,0.0,100.0,1,1,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +1,90,string,3,,2.0,8.0,0,3,,,,,,,,, +1,91,numerical,3,,2.0,8.0,0,3,,1.0,3.0,2.0,,,,, diff --git a/tests/unit/data/unit_test_csv/1_channel_tags.csv b/tests/unit/data/unit_test_csv/1_channel_tags.csv index b572465..4120877 100644 --- a/tests/unit/data/unit_test_csv/1_channel_tags.csv +++ b/tests/unit/data/unit_test_csv/1_channel_tags.csv @@ -1,2 +1,4 @@ container_id,channel_id,key,value 1,1,seed,0 +1,90,channel_name,DTC +1,91,channel_name,DTC_count From f33c320468700243ab59c6c62eafe7135816e498 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 15:07:10 +0200 Subject: [PATCH 06/17] added poi example to notebook Added missing poi in Source Basemodel --- demos/data/reporting/channel_metrics.csv | 38 ++--- demos/reporting_pipeline.ipynb | 144 +++++++++++++++++- .../analyze/query/query_builder.py | 12 +- src/impulse_reporting/config/config_parser.py | 6 +- .../unit/analyze/query/query_builder_test.py | 55 +++++++ .../unit/config/config_parser_test.py | 51 +++++++ 6 files changed, 278 insertions(+), 28 deletions(-) diff --git a/demos/data/reporting/channel_metrics.csv b/demos/data/reporting/channel_metrics.csv index 7846174..46fe67c 100644 --- a/demos/data/reporting/channel_metrics.csv +++ b/demos/data/reporting/channel_metrics.csv @@ -1,19 +1,19 @@ -container_id,channel_id,sample_count,min,max,mean,begin_ms,end_ms,duration_ms,sample_rate,value_type,series_type -1,4,56667,-11.0,-3.0,-8.35955670848995,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE -1,5,56667,0.0,3385.0,1572.765489614767,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE -1,7,56667,0.0,201.0,70.67870188998889,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE -1,9,56667,-33.0,130.0,2.3332627455132617,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE -2,2,47336,-8.0,6.0,-6.105247591684975,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE -2,5,47336,0.0,3177.0,1736.7744422849416,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE -2,7,47336,0.0,188.0,83.95660807841811,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE -2,8,47336,-26.0,117.0,0.8931046138245733,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE -3,2,54775,-32.0,134.0,9.632861706983112,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE -3,7,54775,0.0,2545.0,1308.4330990415335,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE -3,9,54775,0.0,125.0,44.4629849383843,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE -3,10,54775,-3.0,2.0,-2.157334550433592,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE -1,90,3,,,,1519629856439000,1519633356439000,3500000,,STRING,POINTS_IN_TIME -1,91,3,1.0,3.0,2.0,1519629856439000,1519633356439000,3500000,,DOUBLE,POINTS_IN_TIME -2,90,2,,,,1519756824107000,1519758824107000,2000000,,STRING,POINTS_IN_TIME -2,91,2,1.0,2.0,1.5,1519756824107000,1519758824107000,2000000,,DOUBLE,POINTS_IN_TIME -3,90,1,,,,1519926478375000,1519926478375000,0,,STRING,POINTS_IN_TIME -3,91,1,1.0,1.0,1.0,1519926478375000,1519926478375000,0,,DOUBLE,POINTS_IN_TIME +container_id,channel_id,sample_count,min,max,mean,begin_ms,end_ms,duration_ms,sample_rate,value_type +1,4,56667,-11.0,-3.0,-8.35955670848995,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE +1,5,56667,0.0,3385.0,1572.765489614767,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE +1,7,56667,0.0,201.0,70.67870188998889,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE +1,9,56667,-33.0,130.0,2.3332627455132617,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE +2,2,47336,-8.0,6.0,-6.105247591684975,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE +2,5,47336,0.0,3177.0,1736.7744422849416,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE +2,7,47336,0.0,188.0,83.95660807841811,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE +2,8,47336,-26.0,117.0,0.8931046138245733,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE +3,2,54775,-32.0,134.0,9.632861706983112,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE +3,7,54775,0.0,2545.0,1308.4330990415335,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE +3,9,54775,0.0,125.0,44.4629849383843,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE +3,10,54775,-3.0,2.0,-2.157334550433592,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE +1,90,3,,,,1519629856439000,1519633356439000,3500000,,STRING +1,91,3,1.0,3.0,2.0,1519629856439000,1519633356439000,3500000,,DOUBLE +2,90,2,,,,1519756824107000,1519758824107000,2000000,,STRING +2,91,2,1.0,2.0,1.5,1519756824107000,1519758824107000,2000000,,DOUBLE +3,90,1,,,,1519926478375000,1519926478375000,0,,STRING +3,91,1,1.0,1.0,1.0,1519926478375000,1519926478375000,0,,DOUBLE diff --git a/demos/reporting_pipeline.ipynb b/demos/reporting_pipeline.ipynb index 15921e1..37bda54 100644 --- a/demos/reporting_pipeline.ipynb +++ b/demos/reporting_pipeline.ipynb @@ -501,6 +501,51 @@ ")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3a. Select POI Channels (Diagnostic Trouble Codes)\n", + "\n", + "Not every channel is a continuous signal. A **Points-in-Time (POI)** channel is\n", + "an *event stream*: each value exists **only at its timestamp**, with no validity\n", + "in between. The textbook example is **DTCs** (Diagnostic Trouble Codes) \u2014 the\n", + "fault codes an ECU emits at the instant it detects a problem (`P0301` = cylinder-1\n", + "misfire, \u2026).\n", + "\n", + "POI channels are selected with **`poi_channel(...)`** instead of `channel(...)`.\n", + "Identification is identical (same metadata tags); only the semantics differ \u2014\n", + "a POI channel solves to a `PointsInTimeSeries`, not a `SampleSeries`.\n", + "\n", + "| | `channel(...)` | `poi_channel(...)` |\n", + "|---|---|---|\n", + "| Shape | `[tstart, tend)` intervals | `(t\u1d62, v\u1d62)` points |\n", + "| Valid between points? | yes (interpolated) | **no** |\n", + "| Backed by | `SampleSeries` | `PointsInTimeSeries` |" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "# String POI channel: the DTC code emitted at each fault instant.\n", + "# dtype=\"string\" -> equality is the natural operation (\"when did P0301 occur?\"),\n", + "# never arithmetic or ordering on a code.\n", + "dtc = db.query.poi_channel(\n", + " channel_name=\"DTC\",\n", + " dtype=\"string\",\n", + " brand=\"Seat\", model=\"Leon\",\n", + ")\n", + "\n", + "# Numeric POI channel: a running fault-occurrence counter.\n", + "dtc_count = db.query.poi_channel(\n", + " channel_name=\"DTC_count\",\n", + " brand=\"Seat\", model=\"Leon\",\n", + ")" + ] + }, { "cell_type": "markdown", "metadata": { @@ -583,7 +628,8 @@ "\n", "- **BasicEvent** \u2014 from a TSAL boolean expression\n", "- **ContainerEvent** \u2014 spans the entire recording\n", - "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone)" + "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone)\n", + "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone, or each **P0301 misfire** from the DTC channel)" ] }, { @@ -633,7 +679,17 @@ " expr=distance_milestones,\n", " desc=\"Each 10 km driven (instant)\",\n", ")\n", - "report.add_event(milestone_event)" + "report.add_event(milestone_event)\n", + "\n", + "# POI freeze-frame: the instants a P0301 misfire was logged.\n", + "# `dtc == \"P0301\"` is a PointsInTime \u2014 the set of timestamps where the\n", + "# string code equals P0301 \u2014 exactly what a PointsInTimeEvent wants.\n", + "p0301_event = PointsInTimeEvent(\n", + " name=\"p0301_misfires\",\n", + " expr=(dtc == \"P0301\"),\n", + " desc=\"Each instant a P0301 misfire code was set\",\n", + ")\n", + "report.add_event(p0301_event)" ] }, { @@ -749,9 +805,50 @@ " event=milestone_event,\n", " desc=\"Speed & RPM at each 10 km milestone\",\n", "))\n", + "\n", + "# \u2500\u2500 POI aggregations \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n", + "# Freeze-frame: Engine RPM & Vehicle Speed at each P0301 misfire instant.\n", + "# The POI event supplies the timestamps; the sample channels supply the\n", + "# values valid there \u2014 both series types in one aggregation.\n", + "page.add_aggregation(PointValueAggregator(\n", + " name=\"values_at_p0301\",\n", + " input_expressions=[eng_rpm, veh_spd],\n", + " channel_names=[\"Engine RPM\", \"Vehicle Speed\"],\n", + " event=p0301_event,\n", + " desc=\"RPM & Speed at each P0301 misfire\",\n", + "))\n", + "\n", "print(f\"{len(page.aggregations)} aggregations added\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Numeric POI: fault counts per recording\n", + "\n", + "A numeric POI channel reduces like any signal \u2014 but the reductions are\n", + "**unweighted** (points have no duration). `count()` / `max()` on `dtc_count`\n", + "answer \"how many faults did each recording log?\" directly from the query engine.\n", + "\n", + "(The report-level `StatsAggregator` is designed for continuous `SampleSeries`\n", + "inputs, so a per-container POI count is shown here as a direct query instead.)" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "dtc_summary = db.query.select(\n", + " dtc_count.count().alias(\"n_faults\"),\n", + " dtc_count.max().alias(\"peak_count\"),\n", + ").solve(spark=spark, solver=report.get_solver())\n", + "\n", + "display(dtc_summary.orderBy(\"container_id\"))" + ] + }, { "cell_type": "markdown", "metadata": { @@ -825,7 +922,9 @@ "- **Heatmap** \u2014 RPM vs Speed\n", "- **Table** \u2014 per-container statistics\n", "- **Scatter** \u2014 Speed & RPM at each 10 km milestone\n", - " (markers only \u2014 values exist only *at* each instant)" + " (markers only \u2014 values exist only *at* each instant)\n", + "\n", + "Includes two **POI** views: Engine RPM sampled at each P0301 misfire (freeze-frame), and fault-code counts per recording." ] }, { @@ -984,7 +1083,42 @@ "ax.set_title(\"Speed & RPM at Each 10 km Milestone\")\n", "ax.legend()\n", "plt.tight_layout()\n", - "plt.show()" + "plt.show()\n", + "\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 5. POI \u2014 Engine RPM at Each P0301 Misfire (freeze-frame)\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# PointValueAggregator writes to the shared stats_aggregator_fact table;\n", + "# select its visual by name, like the milestone scatter above.\n", + "p0301_df = (\n", + " spark.read.table(f\"{T}_stats_aggregator_fact\")\n", + " .join(\n", + " spark.read.table(f\"{T}_stats_aggregator_dimension\")\n", + " .filter(\"name = 'values_at_p0301'\"),\n", + " on=\"visual_id\",\n", + " )\n", + " .select(\"container_id\", \"channel_name\", \"event_instance_id\", \"statistic_value\")\n", + " .toPandas()\n", + ")\n", + "\n", + "if not p0301_df.empty:\n", + " rpm_hits = p0301_df[p0301_df[\"channel_name\"] == \"Engine RPM\"]\n", + " fig, ax = plt.subplots(figsize=(10, 4))\n", + " for cid, grp in rpm_hits.groupby(\"container_id\"):\n", + " ax.scatter(\n", + " grp[\"event_instance_id\"], grp[\"statistic_value\"],\n", + " label=f\"Container {cid}\", s=90, alpha=0.85,\n", + " edgecolors=\"k\", linewidths=0.5,\n", + " )\n", + " ax.set_xlabel(\"P0301 occurrence #\")\n", + " ax.set_ylabel(\"Engine RPM at fault instant\")\n", + " ax.set_title(\"Freeze-frame: Engine RPM at each P0301 misfire\")\n", + " ax.legend()\n", + " plt.tight_layout()\n", + " plt.show()\n", + "else:\n", + " print(\"No P0301 misfires in the demo data.\")\n", + "" ] }, { @@ -1211,4 +1345,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} +} \ No newline at end of file diff --git a/src/impulse_query_engine/analyze/query/query_builder.py b/src/impulse_query_engine/analyze/query/query_builder.py index a97b465..26469b8 100644 --- a/src/impulse_query_engine/analyze/query/query_builder.py +++ b/src/impulse_query_engine/analyze/query/query_builder.py @@ -183,9 +183,10 @@ def poi_channel( Parameters ---------- - dtype : PoiValueType, optional + dtype : PoiValueType or str, optional The POI channel's value data type: ``DOUBLE`` (default, numeric) or - ``STRING`` (e.g. DTC codes — only sampling and equality apply). This + ``STRING`` (e.g. DTC codes — only sampling and equality apply). Accepts + either the enum or its string value (``"double"`` / ``"string"``). This declared type drives plan-time result typing and string-op gating; it is validated against the silver ``poi_channels.dtype`` at solve time (an actual/declared mismatch raises). @@ -197,13 +198,18 @@ def poi_channel( TimeSeriesSelector A selector stamped ``series_type=POINTS_IN_TIME`` with the given value type. """ + # Accept a plain string ("string" / "double") as well as the enum, so + # poi_channel(..., dtype="string") behaves identically to the enum form. + value_type = PoiValueType(dtype) expr = None for k, arg in kwargs.items(): if not expr: expr = TagSelector(k) == str(arg) else: expr = expr & (TagSelector(k) == str(arg)) - return TimeSeriesSelector(expr, series_type=SeriesType.POINTS_IN_TIME, value_type=dtype) + return TimeSeriesSelector( + expr, series_type=SeriesType.POINTS_IN_TIME, value_type=value_type + ) def select(self, *args) -> Self: """ diff --git a/src/impulse_reporting/config/config_parser.py b/src/impulse_reporting/config/config_parser.py index ffcc5f9..3dc0f9a 100644 --- a/src/impulse_reporting/config/config_parser.py +++ b/src/impulse_reporting/config/config_parser.py @@ -127,6 +127,10 @@ class Source(BaseModel): Full Unity Catalog path to the channel metrics table. channels_uri : str Full Unity Catalog path to the channels data table. + poi_channels_uri : str, optional + Full Unity Catalog path to the Points-in-Time (POI) channel data table. + Required only when the report selects POI channels via ``poi_channel()``; + omit it for sample-only data models. channel_mapping_table : str, optional Full Unity Catalog path to the channel mapping table. Required when using ``channel_with_alias()`` for logical alias resolution. @@ -147,10 +151,10 @@ class Source(BaseModel): container_metrics_table: Annotated[str, AfterValidator(is_valid_table_name)] channel_metrics_table: Annotated[str, AfterValidator(is_valid_table_name)] channels_uri: Annotated[str, AfterValidator(is_valid_table_name)] + poi_channels_uri: Annotated[str, AfterValidator(is_valid_table_name)] | None = None channel_mapping_table: Annotated[str, AfterValidator(is_valid_table_name)] | None = None unit_conversion_table: Annotated[str, AfterValidator(is_valid_table_name)] | None = None - #todo probably add poi here as well so users can configure it class UnitySink(BaseModel): """ diff --git a/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py b/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py index 008b811..ef361f9 100644 --- a/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py @@ -4,6 +4,8 @@ from impulse_query_engine.analyze.metadata.tag_expression import TagSelector from impulse_query_engine.analyze.metadata.time_series_expression import ( + PoiValueType, + SeriesType, TimeSeriesSelector, ) from impulse_query_engine.model.series import Intervals @@ -195,3 +197,56 @@ def test_timeseries_selector_dtype_matches_sample_series_dtype(): ts = TimeSeriesSelector(TagSelector("name") == "test") ss = SampleSeries.empty() assert ts.dtype() == ss.dtype() + + +# --------------------------------------------------------------------------- +# QueryBuilder.poi_channel — dtype accepts the enum OR its string value +# --------------------------------------------------------------------------- +class TestPoiChannelDtypeArg: + """``poi_channel(dtype=...)`` must accept both ``PoiValueType`` and the plain + string value (``"double"`` / ``"string"``). A regression guard: a plain + ``dtype="string"`` used to be stored verbatim (a ``str``, not the enum), so the + ``is PoiValueType.STRING`` identity checks silently fell through and a string + POI channel behaved as numeric — blowing up on the first string comparison. + """ + + def test_default_dtype_is_double(self, narrow_db): + sel = narrow_db.query.poi_channel(channel_name="DTC_count") + assert sel.series_type is SeriesType.POINTS_IN_TIME + assert sel.value_type is PoiValueType.DOUBLE + + def test_enum_string_dtype(self, narrow_db): + sel = narrow_db.query.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + assert sel.value_type is PoiValueType.STRING + + def test_plain_string_dtype_coerced_to_enum(self, narrow_db): + # the design-doc form: poi_channel(..., dtype="string") + sel = narrow_db.query.poi_channel(channel_name="DTC", dtype="string") + assert sel.value_type is PoiValueType.STRING + + def test_plain_string_double_dtype_coerced_to_enum(self, narrow_db): + sel = narrow_db.query.poi_channel(channel_name="DTC_count", dtype="double") + assert sel.value_type is PoiValueType.DOUBLE + + def test_invalid_dtype_raises(self, narrow_db): + with pytest.raises(ValueError): + narrow_db.query.poi_channel(channel_name="DTC", dtype="int") + + def test_string_poi_types_as_struct_regardless_of_arg_form(self, narrow_db): + # both arg forms must produce an identical string-typed result dtype + enum_sel = narrow_db.query.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + str_sel = narrow_db.query.poi_channel(channel_name="DTC", dtype="string") + assert enum_sel.dtype() == str_sel.dtype() + # string POI serializes as array>, not array> + assert isinstance(str_sel.dtype(), T.ArrayType) + assert isinstance(str_sel.dtype().elementType, T.StructType) + + def test_string_poi_equality_evaluates_to_points_in_time(self, narrow_db): + # dtype="string" must yield a string series so `== "code"` works at plan time + dtc = narrow_db.query.poi_channel(channel_name="DTC", dtype="string") + assert (dtc == "P0301").evaluation_type() is PointsInTime + + def test_string_poi_mean_rejected_at_build_time(self, narrow_db): + dtc = narrow_db.query.poi_channel(channel_name="DTC", dtype="string") + with pytest.raises(TypeError, match="string-valued"): + dtc.mean().evaluation_type() diff --git a/tests/impulse_reporting/unit/config/config_parser_test.py b/tests/impulse_reporting/unit/config/config_parser_test.py index 28aa3e7..fbdfb54 100644 --- a/tests/impulse_reporting/unit/config/config_parser_test.py +++ b/tests/impulse_reporting/unit/config/config_parser_test.py @@ -112,6 +112,57 @@ def test_impulse_config_drop_implausible_data_enabled(): assert config.query_engine.drop_implausible_data is True +# --------------------------------------------------------------------------- +# Source.poi_channels_uri — must survive parsing AND reach the MeasurementDB. +# Regression: the field was missing from the Source model, so pydantic silently +# dropped it and the whole reporting-layer POI path was inert (has_poi_channels +# always False) even when the config supplied a poi_channels_uri. +# --------------------------------------------------------------------------- +def test_source_poi_channels_uri_parsed(): + config_json = { + **impulse_config_JSON, + "source": { + **impulse_config_JSON["source"], + "poi_channels_uri": "impulse_demo.silver.poi_channels", + }, + } + config = ImpulseConfig.model_validate(config_json) + assert config.source.poi_channels_uri == "impulse_demo.silver.poi_channels" + + +def test_source_poi_channels_uri_defaults_to_none(): + config = ImpulseConfig.model_validate(impulse_config_JSON) + assert config.source.poi_channels_uri is None + + +def test_poi_channels_uri_reaches_measurement_db(): + """End-to-end passthrough: a poi_channels_uri in the config makes the built + MeasurementDB POI-aware (this is what was silently broken).""" + from unittest.mock import create_autospec + + from databricks.sdk import WorkspaceClient + + from impulse_reporting.core.report import Report + + with_poi = ImpulseConfig.model_validate( + { + **impulse_config_JSON, + "source": { + **impulse_config_JSON["source"], + "poi_channels_uri": "impulse_demo.silver.poi_channels", + }, + } + ) + db = Report.create_measurement_db(with_poi, create_autospec(WorkspaceClient)) + assert db.has_poi_channels() + assert db.config.poi_channels_uri == "impulse_demo.silver.poi_channels" + + # ...and a config without it stays POI-unaware. + without_poi = ImpulseConfig.model_validate(impulse_config_JSON) + db2 = Report.create_measurement_db(without_poi, create_autospec(WorkspaceClient)) + assert not db2.has_poi_channels() + + def test_impulse_config_drop_implausible_data_rejects_rle(): """drop_implausible_data=True with RLE data must raise ValidationError. From ae1371e9fcc1c375ebaa24d5ee54c3979481743e Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 15:27:21 +0200 Subject: [PATCH 07/17] added more poi features to the demo notebook --- demos/reporting_pipeline.ipynb | 77 ++++++++++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 4 deletions(-) diff --git a/demos/reporting_pipeline.ipynb b/demos/reporting_pipeline.ipynb index 37bda54..8492dc9 100644 --- a/demos/reporting_pipeline.ipynb +++ b/demos/reporting_pipeline.ipynb @@ -605,7 +605,17 @@ "\n", "# Instant the trip odometer crosses each additional\n", "# 10 km \u2014 a set of points in time, not an interval.\n", - "distance_milestones = (distance_km % 10).falling_edges()" + "distance_milestones = (distance_km % 10).falling_edges()\n", + "\n", + "# POI as a time-window anchor. A POI instant has no duration, but we often\n", + "# want the signal *around* it. `.expand(w)` turns each P0301 instant into a\n", + "# [t - w, t + w] interval (w in the data's time unit \u2014 microseconds here),\n", + "# so `\u00b110 s` is 10e6. Overlapping windows are merged.\n", + "WINDOW_US = 10e6 # \u00b110 seconds\n", + "p0301_window = (dtc == \"P0301\").expand(WINDOW_US)\n", + "\n", + "# All Engine RPM samples recorded within \u00b110 s of a misfire.\n", + "rpm_around_p0301 = eng_rpm.where(p0301_window)" ] }, { @@ -629,7 +639,8 @@ "- **BasicEvent** \u2014 from a TSAL boolean expression\n", "- **ContainerEvent** \u2014 spans the entire recording\n", "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone)\n", - "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone, or each **P0301 misfire** from the DTC channel)" + "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone, or each **P0301 misfire** from the DTC channel)\n", + "- **BasicEvent on a POI window** \u2014 `(dtc == \"P0301\").expand(\u00b110 s)` turns each fault instant into an interval, so you can aggregate the signal *around* each event" ] }, { @@ -689,7 +700,16 @@ " expr=(dtc == \"P0301\"),\n", " desc=\"Each instant a P0301 misfire code was set\",\n", ")\n", - "report.add_event(p0301_event)" + "report.add_event(p0301_event)\n", + "\n", + "# The \u00b110 s misfire windows as an interval event, so aggregations can run\n", + "# \"within 10 s of a P0301\" the same way they run within any other event.\n", + "p0301_window_event = BasicEvent(\n", + " name=\"p0301_window\",\n", + " expr=p0301_window,\n", + " desc=\"Within \u00b110 s of a P0301 misfire\",\n", + ")\n", + "report.add_event(p0301_window_event)" ] }, { @@ -818,6 +838,19 @@ " desc=\"RPM & Speed at each P0301 misfire\",\n", "))\n", "\n", + "\n", + "# Derived-value-around-POI: min/mean/max of the continuous signals in the\n", + "# \u00b110 s window around each misfire. `eng_rpm.where(p0301_window)` is a\n", + "# SampleSeries (values valid over the window), so StatsAggregator applies.\n", + "page.add_aggregation(StatsAggregator(\n", + " name=\"signals_around_p0301\",\n", + " input_expressions=[eng_rpm, veh_spd],\n", + " channel_names=[\"Engine RPM\", \"Vehicle Speed\"],\n", + " statistics=[\"min\", \"mean\", \"max\"],\n", + " event=p0301_window_event,\n", + " desc=\"Signal stats within \u00b110 s of a P0301 misfire\",\n", + "))\n", + "\n", "print(f\"{len(page.aggregations)} aggregations added\")" ] }, @@ -1118,7 +1151,43 @@ " plt.show()\n", "else:\n", " print(\"No P0301 misfires in the demo data.\")\n", - "" + "\n", + "\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 6. POI WINDOW \u2014 Engine RPM min/mean/max within \u00b110 s of each P0301\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "win_df = (\n", + " spark.read.table(f\"{T}_stats_aggregator_fact\")\n", + " .join(\n", + " spark.read.table(f\"{T}_stats_aggregator_dimension\")\n", + " .filter(\"name = 'signals_around_p0301'\"),\n", + " on=\"visual_id\",\n", + " )\n", + " .select(\"container_id\", \"channel_name\", \"aggregation_label\", \"statistic_value\")\n", + " .toPandas()\n", + ")\n", + "if not win_df.empty:\n", + " rpm_win = (\n", + " win_df[win_df[\"channel_name\"] == \"Engine RPM\"]\n", + " .pivot_table(index=\"container_id\", columns=\"aggregation_label\",\n", + " values=\"statistic_value\")\n", + " .reset_index()\n", + " )\n", + " fig, ax = plt.subplots(figsize=(9, 4))\n", + " x = range(len(rpm_win))\n", + " ax.bar(x, rpm_win[\"max\"] - rpm_win[\"min\"], bottom=rpm_win[\"min\"],\n", + " color=\"lightsteelblue\", edgecolor=\"steelblue\",\n", + " label=\"min\u2013max range\")\n", + " ax.scatter(x, rpm_win[\"mean\"], color=\"crimson\", zorder=3, label=\"mean\")\n", + " ax.set_xticks(list(x))\n", + " ax.set_xticklabels([f\"Container {c}\" for c in rpm_win[\"container_id\"]])\n", + " ax.set_ylabel(\"Engine RPM\")\n", + " ax.set_title(\"Engine RPM within \u00b110 s of a P0301 misfire (min\u2013max range + mean)\")\n", + " ax.legend()\n", + " plt.tight_layout()\n", + " plt.show()\n", + "else:\n", + " print(\"No P0301 misfire windows in the demo data.\")" ] }, { From 5ff2600ca085a51fcfa73308b4040c000b97acb3 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 15:34:26 +0200 Subject: [PATCH 08/17] switched statsagg to histogramm to showcase poi extension --- demos/reporting_pipeline.ipynb | 62 +++++++++++++++------------------- 1 file changed, 28 insertions(+), 34 deletions(-) diff --git a/demos/reporting_pipeline.ipynb b/demos/reporting_pipeline.ipynb index 8492dc9..6dabbe9 100644 --- a/demos/reporting_pipeline.ipynb +++ b/demos/reporting_pipeline.ipynb @@ -839,16 +839,18 @@ "))\n", "\n", "\n", - "# Derived-value-around-POI: min/mean/max of the continuous signals in the\n", - "# \u00b110 s window around each misfire. `eng_rpm.where(p0301_window)` is a\n", - "# SampleSeries (values valid over the window), so StatsAggregator applies.\n", - "page.add_aggregation(StatsAggregator(\n", - " name=\"signals_around_p0301\",\n", - " input_expressions=[eng_rpm, veh_spd],\n", - " channel_names=[\"Engine RPM\", \"Vehicle Speed\"],\n", - " statistics=[\"min\", \"mean\", \"max\"],\n", + "# Derived-value-around-POI: the Engine RPM distribution within \u00b110 s of each\n", + "# misfire. The window event filters the (SampleSeries) channel to those\n", + "# intervals, so a duration-weighted histogram shows *what the engine was doing*\n", + "# around the fault \u2014 richer than a single min/mean/max.\n", + "page.add_aggregation(HistogramDuration(\n", + " name=\"rpm_around_p0301\",\n", + " base_expr=eng_rpm,\n", + " bins=[float(i) for i in range(0, 5000, 250)],\n", " event=p0301_window_event,\n", - " desc=\"Signal stats within \u00b110 s of a P0301 misfire\",\n", + " desc=\"Engine RPM distribution within \u00b110 s of a P0301 misfire\",\n", + " channel_name=\"Engine RPM\",\n", + " bins_unit=\"RPM\", values_unit=\"s\",\n", "))\n", "\n", "print(f\"{len(page.aggregations)} aggregations added\")" @@ -1152,38 +1154,30 @@ "else:\n", " print(\"No P0301 misfires in the demo data.\")\n", "\n", - "\n", "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", - "# 6. POI WINDOW \u2014 Engine RPM min/mean/max within \u00b110 s of each P0301\n", + "# 6. POI WINDOW \u2014 Engine RPM distribution within \u00b110 s of each P0301\n", "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", - "win_df = (\n", - " spark.read.table(f\"{T}_stats_aggregator_fact\")\n", + "win_hist = (\n", + " spark.read.table(f\"{T}_histogram_fact\")\n", " .join(\n", - " spark.read.table(f\"{T}_stats_aggregator_dimension\")\n", - " .filter(\"name = 'signals_around_p0301'\"),\n", + " spark.read.table(f\"{T}_histogram_dimension\")\n", + " .filter(\"name = 'rpm_around_p0301'\"),\n", " on=\"visual_id\",\n", " )\n", - " .select(\"container_id\", \"channel_name\", \"aggregation_label\", \"statistic_value\")\n", + " .groupBy(\"bin_id\", \"bin_name\")\n", + " .agg(F.sum(\"hist_value\").alias(\"total_us\"))\n", + " .orderBy(\"bin_id\")\n", " .toPandas()\n", ")\n", - "if not win_df.empty:\n", - " rpm_win = (\n", - " win_df[win_df[\"channel_name\"] == \"Engine RPM\"]\n", - " .pivot_table(index=\"container_id\", columns=\"aggregation_label\",\n", - " values=\"statistic_value\")\n", - " .reset_index()\n", - " )\n", - " fig, ax = plt.subplots(figsize=(9, 4))\n", - " x = range(len(rpm_win))\n", - " ax.bar(x, rpm_win[\"max\"] - rpm_win[\"min\"], bottom=rpm_win[\"min\"],\n", - " color=\"lightsteelblue\", edgecolor=\"steelblue\",\n", - " label=\"min\u2013max range\")\n", - " ax.scatter(x, rpm_win[\"mean\"], color=\"crimson\", zorder=3, label=\"mean\")\n", - " ax.set_xticks(list(x))\n", - " ax.set_xticklabels([f\"Container {c}\" for c in rpm_win[\"container_id\"]])\n", - " ax.set_ylabel(\"Engine RPM\")\n", - " ax.set_title(\"Engine RPM within \u00b110 s of a P0301 misfire (min\u2013max range + mean)\")\n", - " ax.legend()\n", + "if not win_hist.empty:\n", + " win_hist[\"duration_s\"] = win_hist[\"total_us\"] / 1e6\n", + " fig, ax = plt.subplots(figsize=(10, 4))\n", + " ax.bar(win_hist[\"bin_name\"], win_hist[\"duration_s\"],\n", + " color=\"indianred\", edgecolor=\"white\")\n", + " ax.set_xlabel(\"Engine RPM bin\")\n", + " ax.set_ylabel(\"Duration (s)\")\n", + " ax.set_title(\"Engine RPM distribution within \u00b110 s of a P0301 misfire\")\n", + " plt.xticks(rotation=45, ha=\"right\", fontsize=8)\n", " plt.tight_layout()\n", " plt.show()\n", "else:\n", From 15aee307f944cdda1761486e5e3b510097156341 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Mon, 15 Jun 2026 14:40:10 +0200 Subject: [PATCH 09/17] draft solution for fork pr handling --- .github/workflows/acceptance.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 96819e2..218c38b 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -37,6 +37,10 @@ jobs: labels: linux-ubuntu-latest permissions: id-token: write + # Fork PRs get no OIDC token / secrets from GitHub, so JFrog auth (and therefore + # dependency installation) cannot run. Skip CI for them; fork PRs are to be tested + # by the reviewer(s) / maintainer(s) before merging. + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -63,6 +67,8 @@ jobs: needs: [ not-a-fork, lint ] permissions: id-token: write + # See the note on `lint`: fork PRs cannot authenticate to JFrog, so skip CI for them. + if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} strategy: fail-fast: false matrix: From 40377604bc63fcd887c6dcbb8aec912339e99ee3 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 06:57:41 +0200 Subject: [PATCH 10/17] Added functionality for PointsInTimeSeries to support string as well --- .../model/series/points_in_time_series.py | 88 +++++++++++++- .../series/points_in_time_series_test.py | 111 ++++++++++++++++++ 2 files changed, 193 insertions(+), 6 deletions(-) diff --git a/src/impulse_query_engine/model/series/points_in_time_series.py b/src/impulse_query_engine/model/series/points_in_time_series.py index ba46f83..192a73d 100644 --- a/src/impulse_query_engine/model/series/points_in_time_series.py +++ b/src/impulse_query_engine/model/series/points_in_time_series.py @@ -2,7 +2,8 @@ from __future__ import annotations -from collections.abc import Sized +import functools +from collections.abc import Callable, Sized import numpy as np import numpy.typing as npt @@ -15,6 +16,34 @@ FloatOrNaN = float | np.float64 +def _numeric_only(method: Callable) -> Callable: + """Decorator that rejects the wrapped method on a string-valued series. + + String-valued :class:`PointsInTimeSeries` support only sampling and equality + (``==`` / ``!=``); arithmetic, ordering and numeric reductions have no meaning + for them. Numpy would either raise (``-``, ``/``, ``mean``) or — worse — + silently succeed with a nonsensical result (``+`` concatenates, ``*`` repeats, + ``sum`` concatenates), so guard those methods explicitly and fail loudly. + + Applied to arithmetic, ordering-comparison and reduction methods. + + Raises + ------ + TypeError + When the decorated method is called on a string-valued series. + """ + + @functools.wraps(method) + def wrapper(self: PointsInTimeSeries, *args, **kwargs): + if self._is_string: + raise TypeError( + f"{method.__name__} is not supported for string-valued PointsInTimeSeries" + ) + return method(self, *args, **kwargs) + + return wrapper + + class PointsInTimeSeries: def __init__(self, tstarts: Sized, values: Sized): """ @@ -32,31 +61,62 @@ def __init__(self, tstarts: Sized, values: Sized): Array-like of values, one per time point. """ assert len(tstarts) == len(values) + # Timestamps are always numeric. Values may be numeric or string: + # string-valued series support sampling (``synchronized`` / ``.where``) + # and equality comparisons (``==`` / ``!=``) only — arithmetic, ordering + # and numeric reductions are rejected (see the ``@_numeric_only`` methods). + # An empty series has no observed value type, so it defaults to numeric + # (the safe, backward-compatible case). self.tstarts = np.array(tstarts, dtype=np.float64) - self.values = np.array(values, dtype=np.float64) + self._is_string = np.asarray(values).dtype.kind in ("U", "S", "O") + if self._is_string: + self.values = np.asarray(values, dtype=object) + else: + self.values = np.array(values, dtype=np.float64) def dtype(self): """ Returns the Spark data type for PointsInTimeSeries. + For numeric values the element is a homogeneous ``[tstart, value]`` double + pair (``ArrayType(ArrayType(DoubleType))``). String-valued series cannot use + that homogeneous nested array, so their element is a ``(tstart, value)`` + struct with a double timestamp and a string value. + Returns ------- pyspark.sql.types.ArrayType - Spark ArrayType for points in time series: [[tstart_1, value_1], ...]. - """ + Spark ArrayType matching ``get_data``'s shape for this series' value type. + """ + if self._is_string: + return T.ArrayType( + T.StructType( + [ + T.StructField("tstart", T.DoubleType()), + T.StructField("value", T.StringType()), + ] + ) + ) return T.ArrayType(T.ArrayType(T.DoubleType())) def get_data(self) -> list: """ - Returns the series as a list of [tstart, value] lists. + Returns the series as a list of ``[tstart, value]`` pairs. + + For numeric values this is a list of two-element double lists. For string + values, ``column_stack`` would coerce the timestamps to strings, so the + pairs are built explicitly as ``[float(tstart), str(value)]`` — matching the + struct element type declared by :meth:`dtype`. Returns ------- list - List of [tstart, value] pairs. + List of ``[tstart, value]`` pairs. """ if len(self) == 0: return [] + if self._is_string: + return [[float(t), str(v)] for t, v in zip(self.tstarts, self.values, strict=True)] return np.column_stack([self.tstarts, self.values]).tolist() def __len__(self) -> int: @@ -354,34 +414,42 @@ def _apply_basic_rop(self, operation, other: float | SampleSeries | PointsInTime return PointsInTimeSeries(s0.tstarts, operation(s1.values, s0.values)) return PointsInTimeSeries(self.tstarts, operation(other, self.values)) + @_numeric_only def __add__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Add another series or scalar to this series.""" return self._apply_basic_op(np.add, other) + @_numeric_only def __radd__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Add this series to another series or scalar (reversed operands).""" return self._apply_basic_rop(np.add, other) + @_numeric_only def __sub__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Subtract another series or scalar from this series.""" return self._apply_basic_op(np.subtract, other) + @_numeric_only def __rsub__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Subtract this series from another series or scalar (reversed operands).""" return self._apply_basic_rop(np.subtract, other) + @_numeric_only def __mul__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Multiply this series by another series or scalar.""" return self._apply_basic_op(np.multiply, other) + @_numeric_only def __rmul__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Multiply another series or scalar by this series (reversed operands).""" return self._apply_basic_rop(np.multiply, other) + @_numeric_only def __truediv__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Divide this series by another series or scalar.""" return self._apply_basic_op(np.true_divide, other) + @_numeric_only def __rtruediv__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTimeSeries: """Divide another series or scalar by this series (reversed operands).""" return self._apply_basic_rop(np.true_divide, other) @@ -411,18 +479,22 @@ def __apply_op( idx = operation(self.values, other) return PointsInTime(self.tstarts[idx]) + @_numeric_only def __gt__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTime: """Return points where this series is greater than another.""" return self.__apply_op(np.greater, other) + @_numeric_only def __ge__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTime: """Return points where this series is greater than or equal to another.""" return self.__apply_op(np.greater_equal, other) + @_numeric_only def __lt__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTime: """Return points where this series is less than another.""" return self.__apply_op(np.less, other) + @_numeric_only def __le__(self, other: float | SampleSeries | PointsInTimeSeries) -> PointsInTime: """Return points where this series is less than or equal to another.""" return self.__apply_op(np.less_equal, other) @@ -448,6 +520,7 @@ def count(self) -> int: """ return len(self) + @_numeric_only def sum(self) -> FloatOrNaN: """ Returns the sum of the values. @@ -461,6 +534,7 @@ def sum(self) -> FloatOrNaN: return np.nan return np.sum(self.values) + @_numeric_only def mean(self) -> FloatOrNaN: """ Returns the mean of the values. @@ -474,6 +548,7 @@ def mean(self) -> FloatOrNaN: return np.nan return np.mean(self.values) + @_numeric_only def min(self) -> FloatOrNaN: """ Returns the minimum value. @@ -487,6 +562,7 @@ def min(self) -> FloatOrNaN: return np.nan return np.min(self.values) + @_numeric_only def max(self) -> FloatOrNaN: """ Returns the maximum value. diff --git a/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py b/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py index b957d96..77f8662 100644 --- a/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py +++ b/tests/impulse_query_engine/unit/model/series/points_in_time_series_test.py @@ -152,6 +152,117 @@ def test_aggregations_empty(): assert pts.count() == 0 +# --- string values ------------------------------------------------------------------------------ +# String-valued series support sampling and equality only; arithmetic, ordering +# and numeric reductions are rejected. Timestamps stay numeric regardless. + + +def test_string_values_stored_as_object_with_numeric_timestamps(): + pts = PointsInTimeSeries([1, 2, 3], ["P108B", "U0046", "P108B"]) + assert pts._is_string is True + assert pts.values.dtype == object + assert pts.tstarts.dtype == np.float64 + nptest.assert_array_equal(pts.values, ["P108B", "U0046", "P108B"]) + + +def test_empty_series_defaults_to_numeric(): + # No observed value type -> numeric (backward-compatible default). + assert PointsInTimeSeries.empty()._is_string is False + + +def test_numeric_series_is_not_string(): + assert PointsInTimeSeries([0, 1], [10, 20])._is_string is False + + +def test_string_eq_scalar_returns_points_in_time(): + pts = PointsInTimeSeries([1, 2, 3], ["P108B", "U0046", "P108B"]) + result = pts == "P108B" + assert isinstance(result, PointsInTime) + nptest.assert_array_equal(result.tstarts, [1, 3]) + + +def test_string_ne_scalar_returns_points_in_time(): + pts = PointsInTimeSeries([1, 2, 3], ["P108B", "U0046", "P108B"]) + nptest.assert_array_equal((pts != "P108B").tstarts, [2]) + + +def test_string_eq_series_matches_on_value_and_timestamp(): + p1 = PointsInTimeSeries([1, 2, 3], ["A", "B", "C"]) + p2 = PointsInTimeSeries([2, 3, 4], ["X", "C", "C"]) + # Common timestamps {2,3}; values equal only at t=3 ("C" == "C"). + nptest.assert_array_equal((p1 == p2).tstarts, [3]) + + +def test_string_synchronized_with_sample_series_samples_values(): + pts = PointsInTimeSeries([5, 15, 25], ["a", "b", "c"]) + s = SampleSeries([0, 10, 20], [10, 20, 30], [1, 2, 3]) + a, b = pts.synchronized(s) + nptest.assert_array_equal(a.tstarts, [5, 15, 25]) + nptest.assert_array_equal(a.values, ["a", "b", "c"]) + nptest.assert_array_equal(b.values, [1, 2, 3]) + + +def test_string_get_data_pairs_double_timestamp_with_string_value(): + pts = PointsInTimeSeries([1, 2], ["P108B", "U0046"]) + assert pts.get_data() == [[1.0, "P108B"], [2.0, "U0046"]] + + +def test_string_dtype_is_struct_of_double_and_string(): + pts = PointsInTimeSeries([1, 2], ["P108B", "U0046"]) + assert pts.dtype() == T.ArrayType( + T.StructType( + [ + T.StructField("tstart", T.DoubleType()), + T.StructField("value", T.StringType()), + ] + ) + ) + + +@pytest.mark.parametrize( + "op", + [ + lambda p: p + "x", + lambda p: "x" + p, + lambda p: p - 1, + lambda p: 1 - p, + lambda p: p * 2, + lambda p: p / 2, + ], +) +def test_string_arithmetic_raises(op): + pts = PointsInTimeSeries([1, 2], ["A", "B"]) + with pytest.raises(TypeError, match="string-valued"): + op(pts) + + +@pytest.mark.parametrize( + "op", + [ + lambda p: p > "A", + lambda p: p >= "A", + lambda p: p < "Z", + lambda p: p <= "Z", + ], +) +def test_string_ordering_comparison_raises(op): + pts = PointsInTimeSeries([1, 2], ["A", "B"]) + with pytest.raises(TypeError, match="string-valued"): + op(pts) + + +@pytest.mark.parametrize("reduction", ["sum", "mean", "min", "max"]) +def test_string_reductions_raise(reduction): + pts = PointsInTimeSeries([1, 2], ["A", "B"]) + with pytest.raises(TypeError, match="string-valued"): + getattr(pts, reduction)() + + +def test_string_count_is_allowed(): + # count is structural (not value-dependent), so it works for strings. + assert PointsInTimeSeries([1, 2, 3], ["A", "B", "C"]).count() == 3 + + # --- plane_sweep -------------------------------------------------------------------------------- From 0a47f3f833ecbcf721e4e4c567b5b171eded28d2 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 06:59:22 +0200 Subject: [PATCH 11/17] wip corrected github action --- .github/workflows/acceptance.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 218c38b..96819e2 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -37,10 +37,6 @@ jobs: labels: linux-ubuntu-latest permissions: id-token: write - # Fork PRs get no OIDC token / secrets from GitHub, so JFrog auth (and therefore - # dependency installation) cannot run. Skip CI for them; fork PRs are to be tested - # by the reviewer(s) / maintainer(s) before merging. - if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -67,8 +63,6 @@ jobs: needs: [ not-a-fork, lint ] permissions: id-token: write - # See the note on `lint`: fork PRs cannot authenticate to JFrog, so skip CI for them. - if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} strategy: fail-fast: false matrix: From 0af77afcc5b78bb24e0de97c79ce02afb3176f87 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 07:09:40 +0200 Subject: [PATCH 12/17] added update-api-docs to Makefile ran update-api-docs --- Makefile | 5 ++++- .../model/series/points_in_time_series.md | 16 +++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 3d13e27..445f0ab 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,9 @@ coverage: build: uv build --require-hashes --build-constraints=.build-constraints.txt +update-api-docs: + cd docs/impulse && uv run pydoc-markdown + lock-dependencies: UV_LOCKED := 0 lock-dependencies: uv lock @@ -56,4 +59,4 @@ fork-sync: ./.github/scripts/fork-sync-pr.sh $(PR) .DEFAULT: all -.PHONY: all clean dev lint fmt test coverage build lock-dependencies fork-sync +.PHONY: all clean dev lint fmt test coverage build update-api-docs lock-dependencies fork-sync diff --git a/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md b/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md index 50cbf77..ef8b12c 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md @@ -37,9 +37,14 @@ def dtype() Returns the Spark data type for PointsInTimeSeries. +For numeric values the element is a homogeneous ``[tstart, value]`` double +pair (``ArrayType(ArrayType(DoubleType))``). String-valued series cannot use +that homogeneous nested array, so their element is a ``(tstart, value)`` +struct with a double timestamp and a string value. + **Returns**: -`pyspark.sql.types.ArrayType`: Spark ArrayType for points in time series: [[tstart_1, value_1], ...]. +`pyspark.sql.types.ArrayType`: Spark ArrayType matching ``get_data``'s shape for this series' value type. #### get\_data @@ -47,11 +52,16 @@ Returns the Spark data type for PointsInTimeSeries. def get_data() -> list ``` -Returns the series as a list of [tstart, value] lists. +Returns the series as a list of ``[tstart, value]`` pairs. + +For numeric values this is a list of two-element double lists. For string +values, ``column_stack`` would coerce the timestamps to strings, so the +pairs are built explicitly as ``[float(tstart), str(value)]`` — matching the +struct element type declared by :meth:`dtype`. **Returns**: -`list`: List of [tstart, value] pairs. +`list`: List of ``[tstart, value]`` pairs. #### \_\_len\_\_ From c6285554f3f4c86273db228e01f63a09716de65f Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 14:18:30 +0200 Subject: [PATCH 13/17] added poi_series_integration.md and marked differences from design to impl --- demos/data/reporting/channel_metrics.csv | 32 +- demos/data/reporting/channel_tags.csv | 36 + demos/data/reporting/poi_channels.csv | 13 + demos/reporting_pipeline.ipynb | 95 +- .../metadata/time_series_expression.md | 77 +- .../analyze/query/query_builder.md | 34 + .../analyze/query/solvers/solver_config.md | 36 + .../model/series/points_in_time_series.md | 27 +- poi_series_integration.md | 896 ++++++++++++++++++ .../metadata/time_series_expression.py | 142 ++- .../analyze/query/query_builder.py | 44 + .../analyze/query/solvers/blob_solver.py | 7 +- .../analyze/query/solvers/default_solver.py | 127 ++- .../analyze/query/solvers/empty_cache.py | 7 +- .../analyze/query/solvers/series_cache.py | 25 +- .../analyze/query/solvers/solver_config.py | 25 + src/impulse_query_engine/measurement_db.py | 21 + .../model/series/points_in_time_series.py | 34 +- src/impulse_query_engine/schema.py | 20 + src/impulse_reporting/config/config_parser.py | 1 + tests/conftest.py | 31 + .../integration/poi_channel_solve_test.py | 243 +++++ ...default_solver_wide_column_mapping_test.py | 1 + .../solvers/default_solver_wide_only_test.py | 1 + .../query/solvers/solver_config_test.py | 12 +- .../data/basic_narrow_csv/channel_metrics.csv | 2 + .../data/unit_test_csv/1_channel_metrics.csv | 2 + .../data/unit_test_csv/1_channel_tags.csv | 2 + 28 files changed, 1892 insertions(+), 101 deletions(-) create mode 100644 demos/data/reporting/poi_channels.csv create mode 100644 poi_series_integration.md create mode 100644 tests/impulse_query_engine/integration/poi_channel_solve_test.py diff --git a/demos/data/reporting/channel_metrics.csv b/demos/data/reporting/channel_metrics.csv index 5637f58..7846174 100644 --- a/demos/data/reporting/channel_metrics.csv +++ b/demos/data/reporting/channel_metrics.csv @@ -1,13 +1,19 @@ -container_id,channel_id,sample_count,min,max,mean,begin_ms,end_ms,duration_ms,sample_rate,value_type -1,4,56667,-11.0,-3.0,-8.35955670848995,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE -1,5,56667,0.0,3385.0,1572.765489614767,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE -1,7,56667,0.0,201.0,70.67870188998889,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE -1,9,56667,-33.0,130.0,2.3332627455132617,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE -2,2,47336,-8.0,6.0,-6.105247591684975,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE -2,5,47336,0.0,3177.0,1736.7744422849416,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE -2,7,47336,0.0,188.0,83.95660807841811,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE -2,8,47336,-26.0,117.0,0.8931046138245733,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE -3,2,54775,-32.0,134.0,9.632861706983112,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE -3,7,54775,0.0,2545.0,1308.4330990415335,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE -3,9,54775,0.0,125.0,44.4629849383843,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE -3,10,54775,-3.0,2.0,-2.157334550433592,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE +container_id,channel_id,sample_count,min,max,mean,begin_ms,end_ms,duration_ms,sample_rate,value_type,series_type +1,4,56667,-11.0,-3.0,-8.35955670848995,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE +1,5,56667,0.0,3385.0,1572.765489614767,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE +1,7,56667,0.0,201.0,70.67870188998889,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE +1,9,56667,-33.0,130.0,2.3332627455132617,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE +2,2,47336,-8.0,6.0,-6.105247591684975,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE +2,5,47336,0.0,3177.0,1736.7744422849416,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE +2,7,47336,0.0,188.0,83.95660807841811,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE +2,8,47336,-26.0,117.0,0.8931046138245733,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE +3,2,54775,-32.0,134.0,9.632861706983112,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE +3,7,54775,0.0,2545.0,1308.4330990415335,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE +3,9,54775,0.0,125.0,44.4629849383843,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE +3,10,54775,-3.0,2.0,-2.157334550433592,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE +1,90,3,,,,1519629856439000,1519633356439000,3500000,,STRING,POINTS_IN_TIME +1,91,3,1.0,3.0,2.0,1519629856439000,1519633356439000,3500000,,DOUBLE,POINTS_IN_TIME +2,90,2,,,,1519756824107000,1519758824107000,2000000,,STRING,POINTS_IN_TIME +2,91,2,1.0,2.0,1.5,1519756824107000,1519758824107000,2000000,,DOUBLE,POINTS_IN_TIME +3,90,1,,,,1519926478375000,1519926478375000,0,,STRING,POINTS_IN_TIME +3,91,1,1.0,1.0,1.0,1519926478375000,1519926478375000,0,,DOUBLE,POINTS_IN_TIME diff --git a/demos/data/reporting/channel_tags.csv b/demos/data/reporting/channel_tags.csv index a35cf01..5918595 100644 --- a/demos/data/reporting/channel_tags.csv +++ b/demos/data/reporting/channel_tags.csv @@ -119,3 +119,39 @@ container_id,channel_id,key,value 3,10,model,Leon 3,10,to_city,RT 3,10,unit,C +1,90,brand,Seat +1,90,channel_name,DTC +1,90,model,Leon +1,90,experiment_id,experiment_4 +1,90,ecu,Engine_ECU +1,90,bus,CAN1 +1,90,code_system,P +1,91,brand,Seat +1,91,channel_name,DTC_count +1,91,model,Leon +1,91,experiment_id,experiment_4 +1,91,ecu,Engine_ECU +2,90,brand,Seat +2,90,channel_name,DTC +2,90,model,Leon +2,90,experiment_id,experiment_4 +2,90,ecu,Engine_ECU +2,90,bus,CAN1 +2,90,code_system,P +2,91,brand,Seat +2,91,channel_name,DTC_count +2,91,model,Leon +2,91,experiment_id,experiment_4 +2,91,ecu,Engine_ECU +3,90,brand,Seat +3,90,channel_name,DTC +3,90,model,Leon +3,90,experiment_id,experiment_4 +3,90,ecu,Body_ECU +3,90,bus,CAN2 +3,90,code_system,U +3,91,brand,Seat +3,91,channel_name,DTC_count +3,91,model,Leon +3,91,experiment_id,experiment_4 +3,91,ecu,Body_ECU diff --git a/demos/data/reporting/poi_channels.csv b/demos/data/reporting/poi_channels.csv new file mode 100644 index 0000000..8435727 --- /dev/null +++ b/demos/data/reporting/poi_channels.csv @@ -0,0 +1,13 @@ +container_id,channel_id,timestamp,value_double,value_string,dtype +1,90,1519629856439000,,P0301,string +1,90,1519631856439000,,P0301,string +1,90,1519633356439000,,P0135,string +1,91,1519629856439000,1.0,,double +1,91,1519631856439000,2.0,,double +1,91,1519633356439000,3.0,,double +2,90,1519756824107000,,P0420,string +2,90,1519758824107000,,P0128,string +2,91,1519756824107000,1.0,,double +2,91,1519758824107000,2.0,,double +3,90,1519926478375000,,U0100,string +3,91,1519926478375000,1.0,,double diff --git a/demos/reporting_pipeline.ipynb b/demos/reporting_pipeline.ipynb index 8c3e5eb..edc9c41 100644 --- a/demos/reporting_pipeline.ipynb +++ b/demos/reporting_pipeline.ipynb @@ -16,21 +16,21 @@ } }, "source": [ - "# Impulse — Reporting Pipeline Demo\n", + "# Impulse \u2014 Reporting Pipeline Demo\n", "\n", "The **Impulse Framework** is a Python library that enables\n", "automotive and industrial engineers to process, aggregate,\n", "and analyze petabytes of time-series measurement data on\n", - "Databricks — without requiring Apache Spark expertise.\n", + "Databricks \u2014 without requiring Apache Spark expertise.\n", "\n", "It provides **TSAL** (Time Series Analytics Language),\n", "a Pythonic expression language for defining signals,\n", "events, and aggregations.\n", "\n", "**What this notebook builds:**\n", - "A complete reporting pipeline — RPM histograms,\n", + "A complete reporting pipeline \u2014 RPM histograms,\n", "RPM-vs-speed heatmaps, per-distance-bin statistics, and\n", - "channel values sampled at every 10 km milestone — across\n", + "channel values sampled at every 10 km milestone \u2014 across\n", "3 test drives, persisted as a Gold-layer star schema and\n", "visualized inline with matplotlib.\n", "\n", @@ -61,9 +61,9 @@ "\n", "Impulse sits between a governed silver layer and a gold-layer star schema in Unity Catalog and provides three components:\n", "\n", - "- **TSAL (Time Series Analytics Language)** — a declarative Python DSL for expressing signals, events, and aggregations in natural Python, without requiring Spark expertise.\n", - "- **Query Engine** — pluggable and distributed; compiles TSAL expressions into Spark execution plans and adapts to any silver-layer layout via interchangeable solvers.\n", - "- **Aggregations** — domain-aware physical aggregations, including duration- and distance-weighted 1D/2D histograms and event-scoped statistics." + "- **TSAL (Time Series Analytics Language)** \u2014 a declarative Python DSL for expressing signals, events, and aggregations in natural Python, without requiring Spark expertise.\n", + "- **Query Engine** \u2014 pluggable and distributed; compiles TSAL expressions into Spark execution plans and adapts to any silver-layer layout via interchangeable solvers.\n", + "- **Aggregations** \u2014 domain-aware physical aggregations, including duration- and distance-weighted 1D/2D histograms and event-scoped statistics." ] }, { @@ -229,7 +229,7 @@ " (e.g., one test drive)\n", "- **Channel** = one sensor signal within a container\n", " (e.g., Engine RPM), stored as raw\n", - " `(timestamp, value)` samples — the framework\n", + " `(timestamp, value)` samples \u2014 the framework\n", " automatically converts these to intervals on the fly" ] }, @@ -263,7 +263,7 @@ "SILVER = [\n", " \"container_metrics\", \"container_tags\",\n", " \"channel_metrics\", \"channel_tags\",\n", - " \"channels\",\n", + " \"channels\", \"poi_channels\",\n", "]\n", "for t in SILVER:\n", " pdf = pd.read_csv(f\"{csv_dir}/{t}.csv\")\n", @@ -344,13 +344,13 @@ "# 2. Initialize the Report\n", "\n", "The `Report` orchestrator takes a config specifying:\n", - "- **`source`** — Silver layer tables\n", - "- **`unity_sink`** — Gold layer output\n", - "- **`query_engine.solver`** — `DefaultSolver` for\n", + "- **`source`** \u2014 Silver layer tables\n", + "- **`unity_sink`** \u2014 Gold layer output\n", + "- **`query_engine.solver`** \u2014 `DefaultSolver` for\n", " parallel per-container execution\n", - "- **`query_engine.data_type`** — `RAW` for raw\n", + "- **`query_engine.data_type`** \u2014 `RAW` for raw\n", " timestamp data (auto-converted to intervals)\n", - "- **`measurement_dimensions`** — container metadata\n", + "- **`measurement_dimensions`** \u2014 container metadata\n", " to carry into Gold layer" ] }, @@ -391,6 +391,7 @@ " \"container_metrics_table\": f\"{pfx}_container_metrics\",\n", " \"channel_metrics_table\": f\"{pfx}_channel_metrics\",\n", " \"channels_uri\": f\"{pfx}_channels\",\n", + " \"poi_channels_uri\": f\"{pfx}_poi_channels\",\n", " \"container_tags_table\": f\"{pfx}_container_tags\",\n", " \"channel_tags_table\": f\"{pfx}_channel_tags\",\n", " },\n", @@ -440,7 +441,7 @@ "source": [ "# 3. Select Physical Channels\n", "\n", - "Channels are selected by **metadata tags** —\n", + "Channels are selected by **metadata tags** \u2014\n", "no column names, no SQL, no joins.\n", "These are **lazy expressions**: no data is read yet." ] @@ -500,7 +501,7 @@ "# 4. Define Virtual Signals & Events\n", "\n", "**TSAL** uses Python operators to build lazy\n", - "expression trees — no Spark knowledge needed.\n", + "expression trees \u2014 no Spark knowledge needed.\n", "\n", "**Virtual signals** derive from physical channels.\n", "**Events** are time windows where a condition holds." @@ -536,7 +537,7 @@ ")\n", "\n", "# Instant the trip odometer crosses each additional\n", - "# 10 km — a set of points in time, not an interval.\n", + "# 10 km \u2014 a set of points in time, not an interval.\n", "distance_milestones = (distance_km % 10).falling_edges()" ] }, @@ -558,9 +559,9 @@ "source": [ "# 5. Register Events\n", "\n", - "- **BasicEvent** — from a TSAL boolean expression\n", - "- **ContainerEvent** — spans the entire recording\n", - "- **PointsInTimeEvent** — a set of instants (e.g. each 10 km milestone)" + "- **BasicEvent** \u2014 from a TSAL boolean expression\n", + "- **ContainerEvent** \u2014 spans the entire recording\n", + "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone)" ] }, { @@ -628,10 +629,10 @@ "source": [ "# 6. Define Aggregations\n", "\n", - "- **Histogram** — 1D duration-weighted distribution\n", - "- **Histogram2D** — 2D heatmap of two signals\n", - "- **StatisticsAggregator** — min, median, mean, max per event\n", - "- **PointValueAggregator** — channel value sampled at each instant of a points-in-time event" + "- **Histogram** \u2014 1D duration-weighted distribution\n", + "- **Histogram2D** \u2014 2D heatmap of two signals\n", + "- **StatisticsAggregator** \u2014 min, median, mean, max per event\n", + "- **PointValueAggregator** \u2014 channel value sampled at each instant of a points-in-time event" ] }, { @@ -712,7 +713,7 @@ "))\n", "\n", "# Sample Vehicle Speed & Engine RPM at each 10 km\n", - "# milestone — one value per channel per instant.\n", + "# milestone \u2014 one value per channel per instant.\n", "page.add_aggregation(PointValueAggregator(\n", " name=\"values_at_distance_milestones\",\n", " input_expressions=[veh_spd, eng_rpm],\n", @@ -771,8 +772,8 @@ "source": [ "# 7. Compute & Persist\n", "\n", - "- `determine_report()` — parallel execution\n", - "- `persist_results()` — writes star schema" + "- `determine_report()` \u2014 parallel execution\n", + "- `persist_results()` \u2014 writes star schema" ] }, { @@ -819,11 +820,11 @@ "Read the Gold-layer tables back and render the\n", "results inline with **matplotlib**:\n", "\n", - "- **Bar** — RPM histogram\n", - "- **Heatmap** — RPM vs Speed\n", - "- **Table** — per-container statistics\n", - "- **Scatter** — Speed & RPM at each 10 km milestone\n", - " (markers only — values exist only *at* each instant)" + "- **Bar** \u2014 RPM histogram\n", + "- **Heatmap** \u2014 RPM vs Speed\n", + "- **Table** \u2014 per-container statistics\n", + "- **Scatter** \u2014 Speed & RPM at each 10 km milestone\n", + " (markers only \u2014 values exist only *at* each instant)" ] }, { @@ -846,12 +847,12 @@ "source": [ "import matplotlib.pyplot as plt\n", "\n", - "# ─── Table prefix ───\n", + "# \u2500\u2500\u2500 Table prefix \u2500\u2500\u2500\n", "T = f\"{pfx}\"\n", "\n", - "# ════════════════════════════════════════════════════════════════════\n", - "# 1. BAR — RPM Histogram (aggregated across all containers)\n", - "# ════════════════════════════════════════════════════════════════════\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 1. BAR \u2014 RPM Histogram (aggregated across all containers)\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", "hist_df = (\n", " spark.read.table(f\"{T}_histogram_fact\")\n", " .join(\n", @@ -869,14 +870,14 @@ "ax.bar(hist_df[\"bin_name\"], hist_df[\"duration_s\"], color=\"steelblue\", edgecolor=\"white\")\n", "ax.set_xlabel(\"Engine RPM bin\")\n", "ax.set_ylabel(\"Duration (s)\")\n", - "ax.set_title(\"RPM Histogram — Duration in Each RPM Band (all containers)\")\n", + "ax.set_title(\"RPM Histogram \u2014 Duration in Each RPM Band (all containers)\")\n", "plt.xticks(rotation=45, ha=\"right\", fontsize=8)\n", "plt.tight_layout()\n", "plt.show()\n", "\n", - "# ════════════════════════════════════════════════════════════════════\n", - "# 2. HEATMAP — RPM vs Speed\n", - "# ════════════════════════════════════════════════════════════════════\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 2. HEATMAP \u2014 RPM vs Speed\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", "heat_df = (\n", " spark.read.table(f\"{T}_histogram2d_fact\")\n", " .groupBy(\"x_bin_id\", \"y_bin_id\", \"x_bin_name\", \"y_bin_name\",\n", @@ -913,14 +914,14 @@ "ax.set_yticklabels([lbl[1] for lbl in y_labels], fontsize=7)\n", "ax.set_xlabel(\"Engine RPM\")\n", "ax.set_ylabel(\"Vehicle Speed (km/h)\")\n", - "ax.set_title(\"RPM vs Speed Heatmap — Duration (s)\")\n", + "ax.set_title(\"RPM vs Speed Heatmap \u2014 Duration (s)\")\n", "plt.colorbar(im, ax=ax, label=\"Duration (s)\")\n", "plt.tight_layout()\n", "plt.show()\n", "\n", - "# ════════════════════════════════════════════════════════════════════\n", - "# 3. TABLE — Per-container Statistics (container_stats)\n", - "# ════════════════════════════════════════════════════════════════════\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 3. TABLE \u2014 Per-container Statistics (container_stats)\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", "stats_df = (\n", " spark.read.table(f\"{T}_stats_aggregator_fact\")\n", " .join(\n", @@ -948,9 +949,9 @@ " ),\n", ")\n", "\n", - "# ════════════════════════════════════════════════════════════════════\n", - "# 4. SCATTER — Speed & RPM at Each 10 km Milestone\n", - "# ════════════════════════════════════════════════════════════════════\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 4. SCATTER \u2014 Speed & RPM at Each 10 km Milestone\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", "milestone_df = (\n", " spark.read.table(f\"{T}_stats_aggregator_fact\")\n", " .join(\n", diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md index 253c03d..67c0c08 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/metadata/time_series_expression.md @@ -3,6 +3,39 @@ sidebar_label: time_series_expression title: impulse_query_engine.analyze.metadata.time_series_expression --- +## SeriesType + +```python +class SeriesType(StrEnum) +``` + +How a channel's samples are interpreted (mirrors :class:`RawEncoder`). + +``SAMPLE`` — the default; ``[tstart, tend)`` intervals over which the value is +*valid* (reconstructed by an interpolation method, zero-order hold today), +backed by :class:`SampleSeries`. + +``POINTS_IN_TIME`` — ``(tᵢ, vᵢ)`` points valid *only at* their timestamps, no +between-point validity, backed by :class:`PointsInTimeSeries`. + + +## PoiValueType + +```python +class PoiValueType(StrEnum) +``` + +The value data type of a POI channel — selects its ``poi_channels`` value + +column and which in-memory :class:`PointsInTimeSeries` variant is built. + +``DOUBLE`` — numeric points (``poi_channels.value_double``); the full +arithmetic / ordering / reduction operator set applies. + +``STRING`` — string points (``poi_channels.value_string``, e.g. DTC codes); +only sampling and equality apply (see :class:`PointsInTimeSeries`). + + ## TimeSeriesSelector ```python @@ -12,7 +45,10 @@ class TimeSeriesSelector(TimeSeriesExpression, RequiresDeserialization) #### \_\_init\_\_ ```python -def __init__(expr, uses_alias: bool = False) +def __init__(expr, + uses_alias: bool = False, + series_type: SeriesType = SeriesType.SAMPLE, + value_type: PoiValueType = PoiValueType.DOUBLE) ``` Initialize a TimeSeriesSelector. @@ -20,6 +56,18 @@ Initialize a TimeSeriesSelector. **Arguments**: - `expr` (`TagExpression`): Tag expression to select. +- `uses_alias` (`bool`): Whether the channel resolves via the channel-alias table. +- `series_type` (`SeriesType`): How the selected channel's samples are interpreted. ``SAMPLE`` +(default) builds a :class:`SampleSeries` — today's behavior, +unchanged. ``POINTS_IN_TIME`` builds a :class:`PointsInTimeSeries` +(values valid only at their timestamps); identification / matching is +identical, only the built object and its result dtype differ. This is +the plan-time source of truth for the series type (so ``dtype()`` is +correct for a bare POI selection with no per-channel metadata lookup). +- `value_type` (`PoiValueType`): For a ``POINTS_IN_TIME`` selection, the declared value data type +(``DOUBLE`` / ``STRING``). Ignored for ``SAMPLE``. Drives plan-time +typing and string-op gating; validated against the silver +``poi_channels.dtype`` at solve time (assertion contract). #### dtype @@ -31,7 +79,10 @@ Returns the Spark data type. **Returns**: -`pyspark.sql.types.DataType`: Data type (BinaryType). +`pyspark.sql.types.DataType`: ``BinaryType`` for a SAMPLE selection (serialized ``SampleSeries``), +or the value-type-aware ``PointsInTimeSeries.dtype()`` for a +POINTS_IN_TIME selection (``array>`` for numeric, +``array>`` for string). #### deserialize @@ -39,7 +90,11 @@ Returns the Spark data type. def deserialize(d) ``` -Deserialize sample series after collection/toPandas. +Deserialize a SAMPLE result after collection/toPandas. + +POINTS_IN_TIME results are serialized by ``get_data()`` (a plain +``[[t, v], ...]`` list) and need no deserialization, so they are returned +as-is; only a SAMPLE (binary) blob is decoded to a :class:`SampleSeries`. **Arguments**: @@ -47,23 +102,21 @@ Deserialize sample series after collection/toPandas. **Returns**: -`SampleSeries`: Deserialized sample series. +`SampleSeries or Any`: Deserialized sample series (SAMPLE), else *d* unchanged. #### build ```python -def build(cache: SeriesCache) -> SampleSeries +def build(cache: SeriesCache) ``` -Instantiate a SampleSeries from given cache data. +Instantiate the selected series from cache data. -**Arguments**: - -- `cache` (`SeriesCache`): Cache containing time series data. +Resolution is identical regardless of series type — resolve the matching +candidates, take the first ``(container_id, channel_id)``, and let the +cache build the right object. The **data** is authoritative for the built +type: :meth:`TimeSeriesCache.load_blob` returns a -**Returns**: - -`SampleSeries`: Built sample series. #### get\_required\_tag\_exprs diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md index 40fffeb..1f5fb82 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/query_builder.md @@ -119,6 +119,40 @@ Create a time series selector for the given channel tags. `TimeSeriesSelector`: Time series selector object. +#### poi\_channel + +```python +def poi_channel(dtype: PoiValueType = PoiValueType.DOUBLE, + **kwargs) -> TimeSeriesSelector +``` + +Create a Points-in-Time (POI) channel selector. + +Parallel to :meth:`channel` — it builds the **same** ``TimeSeriesSelector`` +from a tag/column match on ``**kwargs`` (e.g. +``poi_channel(channel_name="DTC")``), differing only in that it is stamped +``series_type=POINTS_IN_TIME`` (so it solves to a +:class:`~impulse_query_engine.model.series.points_in_time_series.PointsInTimeSeries` +— a value valid only *at* each timestamp — rather than a ``SampleSeries``) +and carries the declared value ``dtype``. + +Channel *identification* (tag/column match, ``get_selector_expr``, +``required_tags``, ``selector_id``) is identical to :meth:`channel`; only +the built object and its result dtype differ. + +**Arguments**: + +- `dtype` (`PoiValueType`): The POI channel's value data type: ``DOUBLE`` (default, numeric) or +``STRING`` (e.g. DTC codes — only sampling and equality apply). This +declared type drives plan-time result typing and string-op gating; it +is validated against the silver ``poi_channels.dtype`` at solve time +(an actual/declared mismatch raises). +- `**kwargs` (`dict`): Channel tag-value pairs, matched exactly like :meth:`channel`'s. + +**Returns**: + +`TimeSeriesSelector`: A selector stamped ``series_type=POINTS_IN_TIME`` with the given value type. + #### select ```python diff --git a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md index 8d65a48..e16b99c 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/analyze/query/solvers/solver_config.md @@ -233,6 +233,42 @@ def value_col() -> str Internal column name for the signal value on the channels table. +#### poi\_timestamp\_col + +```python +def poi_timestamp_col() -> str +``` + +Internal column name for the point timestamp on the poi_channels table. + + +#### poi\_value\_double\_col + +```python +def poi_value_double_col() -> str +``` + +Internal column name for the numeric value on the poi_channels table. + + +#### poi\_value\_string\_col + +```python +def poi_value_string_col() -> str +``` + +Internal column name for the string value on the poi_channels table. + + +#### poi\_dtype\_col + +```python +def poi_dtype_col() -> str +``` + +Internal column name for the per-row value-dtype discriminator on poi_channels. + + #### tag\_key\_col ```python diff --git a/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md b/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md index ef8b12c..bdf55c3 100644 --- a/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md +++ b/docs/impulse/docs/references/api/impulse_query_engine/model/series/points_in_time_series.md @@ -24,6 +24,11 @@ A PointsInTimeSeries associates a value to each timestamp. Unlike a SampleSeries a value is only defined *at* its timestamp and is not considered valid in between consecutive timestamps. +The value type (numeric vs string) is inferred from *values*. An **empty** +series has no values to infer from and therefore defaults to numeric; use +:meth:`empty_string` when an explicitly string-typed empty series is needed +(e.g. plan-time result typing of a bare string-POI selection). + **Arguments**: - `tstarts` (`Sized`): Array-like of time points. @@ -420,9 +425,27 @@ Returns a string representation for debugging. def empty() -> PointsInTimeSeries ``` -Returns an empty PointsInTimeSeries. +Returns an empty (numeric) PointsInTimeSeries. + +**Returns**: + +`PointsInTimeSeries`: Empty numeric PointsInTimeSeries object. + +#### empty\_string + +```python +def empty_string() -> PointsInTimeSeries +``` + +Returns an empty **string-valued** PointsInTimeSeries. + +An empty series has no values to infer a type from, so the constructor +defaults to numeric; this factory forces the string value type. Used for +plan-time result typing of a bare string-POI selection, where the empty +series must report the string ``dtype()`` and reject numeric-only ops +(e.g. ``mean()``) before any data is read. **Returns**: -`PointsInTimeSeries`: Empty PointsInTimeSeries object. +`PointsInTimeSeries`: Empty string-valued PointsInTimeSeries object. diff --git a/poi_series_integration.md b/poi_series_integration.md new file mode 100644 index 0000000..5c975be --- /dev/null +++ b/poi_series_integration.md @@ -0,0 +1,896 @@ +--- +sidebar_position: 1 +title: POI Series Integration +--- + +# Design: Integrating Points-in-Time (POI) Series into the Silver Layer + +**Status:** Proposed  ·  **Scope:** `impulse_query_engine` silver-layer +data model + `DefaultSolver` solve stage  ·  **Non-goal:** changing the +6-stage filter pipeline. + +## 1. Summary + +Impulse currently models every channel as a **sample series** — a sequence of +`[tstart, tend)` intervals over which the series is assumed to be **valid** (this +validity is what channel synchronization relies on). It does *not* intrinsically +assume the value is *held constant* over the interval: how a value is reconstructed +within `[tstart, tend)` is an **interpolation** choice. Today the only interpolation +used is **zero-order hold** (the value at `tstart` carries forward), but additional +interpolation methods could be added in the future without changing the underlying +validity model. We want to add a second kind of channel, a **Points-in-Time (POI) +Series**: a list of `(tᵢ, vᵢ)` pairs where each value is defined **only at its +timestamp** and **no assumption of validity (and hence no interpolation) is made +between two consecutive timestamps**. + +The backend model class already exists — +[`PointsInTimeSeries`](../references/api/impulse_query_engine/model/series/points_in_time_series.md) +— and already implements arithmetic, comparisons, `synchronized` / `synchronized_all`, +and the reducing aggregations (`count`, `sum`, `mean`, `min`, `max`). The +integration work is therefore **not** about series math; it is about: + +1. **Where POI samples live in silver** (a new `poi_channels` table), and +2. **How the solver knows a channel is POI** — table membership (data in + `poi_channels` ⇒ POI) plus the query author's `poi_channel(...)` selector; no + explicit `series_type` column is needed (see [§3.2](#32-discriminator-table-membership-which-table-holds-the-channels-data)), and +3. **How the solve step builds a `PointsInTimeSeries` instead of a `SampleSeries`** + for those channels. + +The central design observation is that the entire metadata **filter pipeline is +already series-type-agnostic**, so POI support drops into the *solve* stage only. + +:::note Terminology + +- **Sample series** — the existing channel type; `[tstart, tend)` intervals over + which the series is *valid*, with values reconstructed by an interpolation method + (zero-order hold today). Backed by `SampleSeries`. +- **POI series** — the new channel type; `(tᵢ, vᵢ)` points valid *only at* their + timestamps, with no between-point validity or interpolation. Backed by + `PointsInTimeSeries`. + +::: + +### 1.1 Motivating example: ECU defect / error codes (DTCs) + +The canonical real-world POI series in vehicle testing is the stream of **defect +codes** (a.k.a. error codes, or **Diagnostic Trouble Codes — DTCs**) emitted by a +vehicle's Electronic Control Units (ECUs). When an ECU's diagnostic monitor detects +a fault — a misfire, a sensor reading out of range, a lost CAN message — it emits a +code at the **instant the fault is registered**. In a test fleet these are captured +off the CAN/UDS bus (e.g. via the `ReadDTCInformation` service, UDS `0x19`) and +logged with the timestamp at which the ECU reported them. + +A DTC event stream is a **textbook POI series**, and specifically a **string-valued** +one: + +- **Event-driven, not continuous.** A code exists *at* the moment the ECU raised it + and says **nothing** about the time between two codes. Interpolating "the value + between two error codes" is meaningless — which is exactly the POI validity model + (no between-point validity), and exactly what the held-over-interval `SampleSeries` + model would get *wrong*. +- **String values.** The standardized code is a short alphanumeric string in the + `P0301` form (1 letter for the system — **P**owertrain / **C**hassis / **B**ody / + **U**network — a generic/OEM digit, a subsystem family digit, and a 2-digit fault + index; e.g. `P0301` = cylinder-1 misfire). This is why POI channels need the + string `value_type` from [§3.4](#34-per-channel-value-dtype-double-vs-string): the + natural analysis is *equality* ("when did `P0301` occur?"), never arithmetic or + ordering on the code — matching the equality-only operator set we implement for + string POI series. + +This use case also motivates **mix-and-match** ([§2](#2-background-why-this-fits-so-cleanly)), +because DTCs are almost always analyzed **together with the continuous signals** +recorded in the same container: + +- **"Freeze-frame"-style analysis.** ECUs snapshot continuous PIDs (engine RPM, + vehicle speed, coolant temperature, …) at the instant a code is set. In Impulse + this is the exact shape of `PointValueAggregator` / a `PointsInTimeEvent`: sample a + `SampleSeries` channel (`Engine_RPM`) **at the timestamps of** a POI channel + (`DTC == "P0301"`). The POI channel supplies the instants; the sample channel + supplies the values valid at those instants — one query, one container, both series + types in the same pandas UDF. +- **Counting / windowing.** "How many `P0301` events occurred while + `Engine_RPM > 4000`?" combines a string-POI equality filter with an interval + derived from a sample series — again both series types in one expression. + +Sketched in the query API this design proposes ([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)), +the string DTC channel is selected with the dedicated `poi_channel(...)` method +and its `dtype`, and mixes freely with an ordinary `channel(...)` sample selection: + +```python +dtc = query.poi_channel(channel_name="DTC", dtype="string") # string POI series +rpm = query.channel(channel_name="Engine_RPM") # sample series + +# "freeze-frame": RPM at the instants DTC == "P0301" +rpm.where(dtc == "P0301") + +# equality is the only comparator defined on a string POI series (§3.4) +``` + +:::note Timestamp caveat (informative) + +DTCs do not universally carry an absolute wall-clock timestamp in the ECU fault +memory — the reliable instant usually comes from the logger/gateway that timestamps +the event when it reads the code (GPS/NTP-synced), and any per-DTC snapshot/extended +records are OEM-dependent. For Impulse this is an **ingestion** concern: whatever +timestamp the silver pipeline lands on `poi_channels.timestamp` is the instant the +engine treats as the point's `tᵢ`. It does not affect the data-model or solver +design below. + +::: + +## 2. Background: why this fits so cleanly + +The [`DefaultSolver` filter pipeline](../references/query_engine/query_solvers.md) +runs six stages, but only ever passes **identity + selector metadata** between +them: + +``` +filter_container_tags → filter_container_metrics → filter_channel_tags → +filter_channel_metrics → (alias resolution) → solve +``` + +Every stage up to `solve` produces at most +`(container_id, channel_id, selector_ids)` (plus optional unit columns). **None of +these stages read `tstart` / `tend` / `value` or make any interval-validity or +interpolation assumption.** The validity-and-interpolation semantics enter the +system in exactly one place: + +- `DefaultSolver.solve` reads the `channels` table, joins it to the channel-match + frame, and runs a grouped-map UDF (`_solve_udf`). +- Inside the UDF, `TimeSeriesCache.load_blob(...)` constructs a **`SampleSeries`** + from the `(ts, te, val)` columns. +- `TimeSeriesSelector.build(cache)` calls `cache.load_blob(...)` and returns that + `SampleSeries` to the expression tree. + +So a channel becomes a `SampleSeries` at `TimeSeriesCache.load_blob`, and nowhere +else. If we can make that one call return a `PointsInTimeSeries` for POI channels, +the rest of the engine — expression evaluation, events, aggregations — already +works, because `PointsInTimeSeries` and `SampleSeries` share the operator and +synchronization protocol, and `SampleSeries.where(PointsInTime)` / +`PointsInTimeSeries.plane_sweep` already bridge the two representations. + +**Mixing is the common case, and it is a per-container concern.** POI and sample +channels live in the **same containers**, and users routinely combine them in one +expression (e.g. `poi_channel - sample_channel`). Cross-type alignment +(`synchronized`) happens **inside** the per-container pandas UDF, on the in-memory +series objects. That imposes a hard requirement: **both series types for a container +must be present in the same UDF invocation.** A design that solved sample and POI +channels in two *separate* UDFs and unioned the results would break mixing — each +UDF would see only half of a container's channels and could not evaluate a +cross-type expression. The design below therefore feeds **one unified pandas frame +per container** (sample + POI channel data together) into a **single** grouped-map +UDF, and a unified cache builds the correct series type per channel. + +```mermaid +flowchart TB + subgraph pipeline["Filter pipeline (UNCHANGED — series-type-agnostic)"] + direction LR + A[container tags] --> B[container metrics] --> C[channel tags] --> D[channel metrics] + end + D -->|"(container_id, channel_id, selector_ids, series_type)"| J + + subgraph solve["solve stage (the ONLY place that touches series semantics)"] + direction TB + RS["read channels
(SAMPLE rows)"] --> J + RP["read poi_channels
(POI rows)"] --> J + J["union sample + POI sample data
keyed by (container_id, channel_id)
→ ONE frame per container"] + J --> U["grouped-map UDF, grouped by container_id
(both series types in the same pandas frame)"] + U --> C2["unified cache builds per channel:
SampleSeries (valid over interval; ZOH today)
or PointsInTimeSeries (valid only at points)"] + C2 --> EV["evaluate expression tree
cross-type ops align via synchronized"] + end + EV --> OUT([one wide row per container]) +``` + +## 3. Chosen design + +### 3.1 Storage: a separate `poi_channels` table + +POI samples are stored in a **new silver table `poi_channels`**, parallel to +`channels` but carrying a single timestamp (no derived `tend`, because a POI point +has no notion of a validity interval) and **two typed value columns** plus a +per-row `dtype` discriminator, since a POI value may be numeric **or** a string: + +| Column | Type | Nullable | Description | +|----------------|----------|----------|-------------------------------------------------------------------| +| `container_id` | `long` | No | Parent container identifier (join key). | +| `channel_id` | `int` | No | Channel identifier. | +| `timestamp` | `long` | No | Point timestamp (microseconds). | +| `value_double` | `double` | Yes | Value at this timestamp **when `dtype = double`**; else null. | +| `value_string` | `string` | Yes | Value at this timestamp **when `dtype = string`**; else null. | +| `dtype` | `string` | No | Value data type: `double` or `string`. Selects the value column. | + +For any given row exactly one of `value_double` / `value_string` is populated, +chosen by `dtype`. `dtype` is expected to be **constant per +`(container_id, channel_id)`** — a channel is either a numeric POI channel or a +string POI channel, not a mix (see [§3.4](#34-per-channel-value-dtype-double-vs-string)). + +`container_id` follows the same +[type rules as the rest of the silver layer](../data_model/silver_layer_schema.md) +— it may be `long` / `int` / `string`, but must be **consistent across all silver +tables** since the engine joins on it. This matches the CLAUDE.md invariant that +`container_id` / `channel_id` types are derived dynamically and never hardcoded. + +**Why a separate table rather than reusing `channels`:** + +- **No semantic overloading of `tend`.** The `channels` RLE format treats a + trailing zero-duration `[t, t)` row as a *closed endpoint* of a sample series (an + interval of validity that has collapsed to a single instant). Reusing that row + shape for a *whole* POI channel would require every + reader (the solve UDF, the RLE/interval encoders, `SampleSeries` construction) to + disambiguate "closed endpoint of a sample series" from "a genuine point". A + dedicated table keeps the two data shapes physically and semantically distinct. +- **Cleaner ingestion contract.** Producers write POI points as `(timestamp, value)` + with no obligation to synthesize a `tend`, which they cannot do correctly for POI + data anyway. +- **Minimal disturbance to the sample-series path.** The existing `channels` read + and RLE/interval encoding are untouched, and a `SAMPLE` channel still builds the + identical `SampleSeries`. The cache does gain a per-channel series-type dispatch + (required so sample and POI channels can be mixed in one UDF — see + [§4.3](#43-one-unified-per-container-frame-one-udf-one-dispatching-cache)), but the + sample branch's behavior is unchanged. + +The cost is a new configured table + a new read path + a branch in the solve +prelude — all localized to `DefaultSolver.solve` / `MeasurementDB` (see §4). + +### 3.2 Discriminator: a `series_type` column on `channel_metrics` + +:::note Implemented differently — see [§9](#9-aspects-which-differ-from-the-design) +The `series_type` column described below was **not** added. Table membership +(`channels` vs `poi_channels`) is the discriminator instead. See [§9](#9-aspects-which-differ-from-the-design). +::: + +A channel is marked POI by a **new `series_type` column on `channel_metrics`**: + +| Column | Type | Nullable | Description | +|---------------|----------|----------|--------------------------------------------------------------------| +| `series_type` | `string` | Yes | `SAMPLE` (default) or `POINTS_IN_TIME`. Null/absent ⇒ `SAMPLE`. | + +Design points: + +- **Backward compatible.** Existing tables without the column, or with `NULL`, + resolve to `SAMPLE`, so every current deployment behaves exactly as today. +- **Rides the pipeline as pass-through metadata.** `channel_metrics` is already + read in `filter_channel_metrics`; `series_type` is just one more column carried + on the channel-match rows through to `solve`. It participates in **no** filtering + decision. +- **Not `value_type`.** `channel_metrics.value_type` already exists but describes + the *value's data type* (`double`, `int`, …). Overloading it to also encode + *series semantics* would conflate two orthogonal concepts and is rejected. A new, + purpose-specific column keeps the discriminator explicit and self-documenting. +- **Introduce a `SeriesType` enum** (mirroring `RawEncoder`) so the string literals + live in one place and are referenced by `SolverConfig.series_type_col` / + the solve branch rather than being sprinkled as bare strings. + +`series_type` is added to `SolverConfig` as an internal column name property +(`series_type_col`, default `"series_type"`), so a physical layout that names the +column differently maps it via `channel_metrics.column_name_mapping` exactly like +every other column. + +### 3.3 Data model after the change + +```mermaid +erDiagram + container_metrics { + long container_id PK + } + channel_metrics { + long container_id FK + int channel_id FK + string series_type "SAMPLE | POINTS_IN_TIME (null ⇒ SAMPLE)" + } + channels { + long container_id FK + int channel_id FK + long tstart + long tend + double value + } + poi_channels { + long container_id FK + int channel_id FK + long timestamp + double value_double "when dtype = double" + string value_string "when dtype = string" + string dtype "double | string" + } + + container_metrics ||--o{ channel_metrics : container_id + channel_metrics ||--o{ channels : "SAMPLE channels" + channel_metrics ||--o{ poi_channels : "POINTS_IN_TIME channels" +``` + +A given `(container_id, channel_id)` has its samples in **exactly one** of +`channels` or `poi_channels`, selected by its `series_type` row in +`channel_metrics`. + +### 3.4 Per-channel value dtype: double vs string + +The `poi_channels.dtype` column determines which value column +(`value_double` / `value_string`) carries the point value. We treat `dtype` as a +**per-channel** property: all rows of a `(container_id, channel_id)` share one +`dtype`. This keeps a channel's value type stable, matches how measurement channels +behave in practice, and lets the solve step pick the value column **once** per +channel rather than per row. + +The two dtypes are **not** symmetric, because the backend model represents them +differently. `PointsInTimeSeries` **cannot represent string values today** — its +constructor hardcodes `np.array(values, dtype=np.float64)`, which would coerce +strings to `NaN`. We close this gap by extending the **single** +[`PointsInTimeSeries`](../references/api/impulse_query_engine/model/series/points_in_time_series.md) +class to hold values of either kind, rather than adding a second class. + +**Chosen model change — dual value arrays + a `value_type` property:** + +- **Keep the existing `float64` value array** for numeric values (unchanged; today's + numeric behavior is preserved bit-for-bit). +- **Add a second value array of dtype `object`** to hold string values. + *(Implemented differently — a single value array whose type is inferred at + construction; see [§9](#9-aspects-which-differ-from-the-design).)* +- **Add a `value_type` property on the class** distinguishing a **numeric** from a + **string** POI series. This is the single source of truth for which value array is + populated and which operations are legal. (Constructors/factories set it; a + numeric series leaves the object array empty and vice-versa.) +- **Spark `dtype()` becomes `value_type`-aware:** `ArrayType(ArrayType(DoubleType))` + for numeric (unchanged), `ArrayType(ArrayType(StringType))` for string. + +**Operations on a string POI series (this iteration):** + +- **Only the equality comparator (`==`) is implemented.** It matches the numeric + behavior — synchronize on shared timestamps, compare values, return the + `PointsInTime` where values are equal — but over string values. + *(Implemented more permissively — both `==` and `!=` are supported for strings; + see [§9](#9-aspects-which-differ-from-the-design).)* +- **All other comparators (`<`, `<=`, `>`, `>=`) return a + `NotImplementedError`** for a string series, as do the numeric-only reductions and + arithmetic (`sum`, `mean`, `min`, `max`, `+`, `-`, `*`, `/`). These raise a clear, + explicit error rather than silently coercing to `NaN`. +- Value-type-independent operations remain valid regardless of `value_type`: + `count`, `start_time` / `end_time`, `to_points_in_time`, `plane_sweep`, and the + timestamp side of `synchronized`. + +:::note "series type" appears on three distinct axes — keep them straight + +| Where | Values | Meaning | +|-------|--------|---------| +| table membership (`channels` / `poi_channels`) | sample vs POI | Which table holds the channel's data — *this* is the sample-vs-POI discriminator (no `series_type` column; see [§9](#9-aspects-which-differ-from-the-design)). | +| `poi_channels.dtype` (silver column) | `double` / `string` | A POI channel's value type — selects `value_double` vs `value_string`. | +| `PointsInTimeSeries.value_type` (class property) | numeric / string | Which in-memory value array is active and which operations are legal. | + +The middle and bottom rows are the same distinction on two sides of the Arrow +boundary: `poi_channels.dtype` on a channel becomes `PointsInTimeSeries.value_type` +on the object the cache builds for it. + +::: + +The **selectable operations are gated by `value_type`** so that, e.g., +`string_poi.mean()` fails up front (via `evaluation_type()` — see [§4.4](#44-result-typing)) +rather than producing `NaN`. + +:::note Scope check + +String POI support is the one part of this design that requires touching the +backend model (`PointsInTimeSeries`). Everything else — storage, discriminator, +pipeline, solve branch — is additive. If string POI is not needed in the first +iteration, the numeric (`double`) path can ship alone: the solver simply routes +only `dtype = double` channels and rejects (or ignores, per config) `string` +channels until the model work lands. + +::: + +### 3.5 Example: tag & metric entries for DTC POI channels + +Concrete rows for the [DTC example](#11-motivating-example-ecu-defect--error-codes-dtcs), +on an existing recording `container_id = 1`. Two POI channels are added on +`channel_id`s not used by any sample channel in that container: a **string** DTC-code +channel (`channel_id = 90`) and a **numeric** fault-occurrence-count channel +(`channel_id = 91`). + +#### Channel level — where POI-specific entries naturally live + +**Channel selection metadata.** In the EAV layout these are `channel_tags` rows +(`container_id, channel_id, key, value`); in the wide layout the same facts are +columns on `channel_metrics`. A DTC channel is selected by its `channel_name` and +described by ECU/bus context: + +| container_id | channel_id | key | value | +|--------------|------------|----------------|--------------| +| 1 | 90 | `channel_name` | `DTC` | +| 1 | 90 | `ecu` | `Engine_ECU` | +| 1 | 90 | `bus` | `CAN1` | +| 1 | 90 | `code_system` | `P` (powertrain) | +| 1 | 91 | `channel_name` | `DTC_count` | +| 1 | 91 | `ecu` | `Engine_ECU` | + +**Channel metrics** (`channel_metrics`). The **new `series_type`** marks the channel +as POI; the **existing `value_type`** records the value data type. Crucially, the +numeric statistic columns behave differently by value type — they are **undefined +(null) for a string POI channel**, and meaningful (computed over the point values, +**unweighted** — there are no durations) for a numeric one: + +| Column | DTC string channel (90) | DTC count numeric channel (91) | Notes | +|----------------|-------------------------|--------------------------------|-------| +| `series_type` | `POINTS_IN_TIME` | `POINTS_IN_TIME` | new discriminator (§3.2) | +| `value_type` | `STRING` | `DOUBLE` | pre-existing data-type column | +| `channel_name` | `DTC` | `DTC_count` | selection key (wide layout) | +| `sample_count` | `3` (three events) | `3` | number of points | +| `begin_s`/`end_s` | first/last event time | first/last event time | point extent, not a validity span | +| `min`/`max`/`mean`/`std` | **null** | computed over point values | undefined for strings; unweighted for numeric POI | +| `pz1`/`pz10`/`pz90`/`pz99` | **null** | optional | percentiles undefined for strings | +| `nan_ratio` | **null** | **null** | duration-weighted → N/A for POI | + +The per-row **`dtype`** (`string` / `double`) lives on `poi_channels`, not here (§3.1); +`series_type` on `channel_metrics` is what routes the channel to `poi_channels`. + +#### Container level — optional summaries for pre-filtering + +A container is a whole recording and owns **both** sample and POI channels, so +container-level tags/metrics are **not** POI-specific — the usual `vehicle_key`, +`brand`, `model`, `project` entries are unchanged. What POI *optionally* adds here is +**summary metadata that lets you pre-filter containers** without scanning +`poi_channels` (the same role the percentile columns play for sample channels): + +EAV `container_tags` (`container_id, key, value`): + +| container_id | key | value | Purpose | +|--------------|------------------|-------------|---------| +| 1 | `vehicle_key` | `Seat_Leon` | existing — unchanged | +| 1 | `has_dtc` | `true` | optional — "recordings that logged any fault" | +| 1 | `ecu_sw_version` | `4.11.2` | optional — correlate faults with firmware | + +Wide `container_metrics` can carry the analogous optional column +`num_dtc_events = 3` for the same pre-filtering purpose. + +These container-level additions are **purely optional and additive**: omit them and +POI channels still work; add them only to enable "find recordings where a `P0301` +occurred"-style container filters before the channel stage. A query like +`query.havingTag(has_dtc="true")` then narrows containers exactly as any other +container tag does — no POI-specific pipeline behavior. + + +## 4. Implementation plan + +The change is localized. Nothing in stages 1–5 of the pipeline changes. + +### 4.1 Config & schema + +1. `SolverConfig`: add `poi_channels: TableConfig`, add the `series_type_col` + property (`"series_type"`), and add a `poi_channels_uri` slot to + `MeasurementDBConfig` (+ `for_unity_catalog` / `for_debug` wiring, mirroring + `channels_uri`). `poi_channels_uri = None` means "no POI channels configured". +2. `MeasurementDB.poi_channels(spark)` reader, mirroring `channels(...)`. +3. `schema.py`: add a reference `POI_CHANNELS_SCHEMA` (`container_id`, `channel_id`, + `timestamp`, `value_double`, `value_string`, `dtype`) and add `series_type` to + `CHANNEL_METRICS`. As documented in CLAUDE.md these are **reference** schemas, + not enforced on read. +4. Add a `SeriesType` StrEnum (`SAMPLE`, `POINTS_IN_TIME`) next to `RawEncoder`, and + a `PoiValueType` StrEnum (`double`, `string`) for the per-row `dtype`. +5. Add `SolverConfig` internal-name properties for the new POI columns + (`poi_timestamp_col`, `poi_value_double_col`, `poi_value_string_col`, + `poi_dtype_col`) so physical layouts remap them via + `poi_channels.column_name_mapping` like every other table. +6. Extend `SolverConfig.col_map` (the short-key → column-name map handed to the UDF + cache, today `cid/ch/ts/te/val/conv`) with `series_type`, `value_string`, and + `dtype` keys so the unified cache (§4.3) can locate them in the pandas frame. +7. Add two optional fields to `TimeSeriesSelector` (`series_type`, `value_type`), + defaulting to `SAMPLE` / numeric so existing `channel(...)` selectors are + unchanged, and add `QueryBuilder.poi_channel(*, dtype=PoiValueType.double, + **kwargs)` (see [§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)). + +### 4.2 Query API and carrying the discriminators to solve + +#### `QueryBuilder.poi_channel(...)` + +POI channels are selected through a dedicated **`poi_channel(...)` factory method** +on `QueryBuilder`, parallel to the existing `channel(...)` / `channel_with_alias(...)`: + +```python +def poi_channel(self, *, dtype: PoiValueType = PoiValueType.double, **kwargs) -> TimeSeriesSelector: + # same tag/column matching as channel(...) — builds the selector expr from **kwargs + return TimeSeriesSelector(expr, series_type=POINTS_IN_TIME, value_type=dtype) +``` + +Design points: + +- **No new selector class.** `poi_channel` returns the **same `TimeSeriesSelector`** + that `channel(...)` returns; channel *identification* (tag/column match, + `get_selector_expr`, `required_tags`, `selector_id`, the direct/aliased split) is + identical for POI and sample channels, so there is nothing to override. The method + is a **factory**, not a subclass — it just stamps the selector with its + `series_type` (`POINTS_IN_TIME`) and the caller-declared value `dtype`. +- **Explicit intent at the call site.** `query.poi_channel(channel_name="DTC")` + reads as "this is an event stream, not a signal," and gives POI-only knobs + (the `dtype`) a natural home. `dtype` defaults to `double`, so the common numeric + case stays terse; a string DTC channel is `poi_channel(channel_name="DTC", dtype=string)`. +- **The selector now carries `series_type` + `value_type`.** `TimeSeriesSelector` + gains two optional fields (defaulting to `SAMPLE` / numeric so `channel(...)` is + unchanged). This makes the selector the **plan-time** source of truth for the + series type — which is what simplifies result typing (see [§4.4](#44-result-typing)): + `evaluation_type()` / `dtype()` and the string-op gating work **without** any + pre-pipeline `channel_metrics` lookup, and `string_poi.mean()` can be rejected at + **build time** before Spark is involved. + +:::caution Declared `dtype` is validated against the data, not trusted over it + +The user-declared `dtype` and the silver data are **two sources that must agree**. +The contract is **assertion, not authority**: + +> The check validates against the **data itself**, not a `channel_metrics.series_type` +> column (which was dropped — see [§9](#9-aspects-which-differ-from-the-design)): a POI +> point row carries a null `tend`, so a `poi_channel(...)` that resolves to +> interval-shaped rows is a SAMPLE channel, and an all-null value column exposes a +> declared/actual `dtype` mismatch. + +- The declared `series_type` / `dtype` drive **plan-time** typing and op-gating. +- At **solve time** the data remains authoritative: if the resolved channel's actual + shape **disagrees** with what the selector declared, the solver **raises a clear + error** (mirroring the existing unit-conversion conflict check), rather than silently + reading the wrong value column or overriding the data. + +This keeps the ergonomic win (no plan-time lookup, early validation) without letting +a wrong declaration silently mis-read a channel (e.g. a `dtype=double` hint on a +string channel yielding all-null `value_double`). + +::: + +#### Carrying the discriminators through the pipeline + +`filter_channel_metrics` already reads and column-maps `channel_metrics`. Include +`series_type` in the projected channel-match columns (defaulting null → `SAMPLE` +via `F.coalesce`). It travels alongside `selector_ids` with no effect on any +filter, exactly like the existing per-channel metadata. This solve-time +`series_type` (and, for POI, `dtype`) is what the **assertion check above** +validates the selector's declared values against. + +### 4.3 One unified per-container frame, one UDF, one dispatching cache + +Because sample and POI channels share containers and are mixed in a single +expression, they **must be solved together in one grouped-map UDF per container** +(see the requirement established in [§2](#2-background-why-this-fits-so-cleanly)). +The design keeps the existing single-UDF shape and makes the *cache* series-type +aware, rather than forking the UDF. + +**Step 1 — normalize both sample sources into one Spark frame.** In +`_prepare_channels_join`, read and column-map **both** tables and project them into +a common superset schema keyed by `(container_id, channel_id)`, carrying a +`series_type` discriminator (and, for POI, `dtype`): + +| Column | SAMPLE row source | POI row source | +|----------------|-----------------------|---------------------------------------| +| `container_id` | `channels` | `poi_channels` | +| `channel_id` | `channels` | `poi_channels` | +| `series_type` | `SAMPLE` | `POINTS_IN_TIME` | +| `tstart` | `channels.tstart` | `poi_channels.timestamp` | +| `tend` | `channels.tend` | `null` (POI has no validity interval) | +| `value` | `channels.value` | `poi_channels.value_double` | +| `value_string` | `null` | `poi_channels.value_string` | +| `dtype` | `null` (⇒ numeric) | `poi_channels.dtype` | + +`unionByName` the two projections into a single DataFrame, join it to the +channel-match frame on `(container_id, channel_id)`, then — exactly as today — +`groupBy(container_id).apply(udf)`. Only channels that survived the filter pipeline +are shipped, so the union stays small. A container's sample and POI rows now land in +the **same** pandas frame. + +**Step 2 — a unified cache that dispatches per channel.** Generalize +`TimeSeriesCache` (or add a `UnifiedSeriesCache` that subsumes it) so `load_blob` +inspects the channel slice's `series_type` and builds the right object: + +- `series_type == SAMPLE` → `SampleSeries(tstart, tend, value)` (today's behavior, + unchanged). +- `series_type == POINTS_IN_TIME` and `dtype == double` → numeric + `PointsInTimeSeries(tstart, value)` (the POI timestamp lives in the `tstart` + column of the unified frame). +- `series_type == POINTS_IN_TIME` and `dtype == string` → the string point series + from [§3.4](#34-per-channel-value-dtype-double-vs-string), built from + `(tstart, value_string)`. + +The cache keeps the same `(cid, ch) → (start, stop)` range-index over the sorted +frame; the only change is which columns each slice reads and which class it +instantiates. Because `series_type` and `dtype` are constant per channel, the +dispatch is decided **once** per `(cid, ch)` slice, not per row. + +**Step 3 — expression evaluation is unchanged.** `TimeSeriesSelector.build(cache)` +still just calls `cache.load_blob(...)`; it now transparently gets a `SampleSeries` +or a point series. A mixed expression such as `poi_channel - sample_channel` is +evaluated on the two in-memory objects, and `PointsInTimeSeries._apply_basic_op` +already handles the cross-type case by aligning against the `SampleSeries` at the +POI timestamps via `synchronized`. **No new math and no second UDF.** + +The `series_type` / `dtype` discriminators are carried the same pass-through way as +the existing per-channel metadata (they originate on `channel_metrics` / +`poi_channels`; see [§8](#8-open-questions)), so both the cache and the result-typing +step (§4.4) know each channel's kind without scanning its data. + +:::note Why not two UDFs? + +Splitting SAMPLE and POI into two grouped-map UDFs and unioning their **outputs** +would be simpler to write but is **incorrect** for the common mix-and-match case: +each UDF would receive only a subset of a container's channels, so an expression +referencing one channel of each type could not be evaluated — one operand would +always be missing from that UDF's frame. Unifying the **input** frame and keeping a +single UDF is what makes cross-type expressions work. + +::: + +### 4.4 Result typing + +`QueryBuilder._determine_result_objects_dtypes` builds each selection against an +`EmptyTimeSeriesCache` to learn its result `dtype`. Today `EmptyTimeSeriesCache.load_blob` +always returns an empty `SampleSeries`, so a bare POI selection would be mistyped +as `BinaryType` (the `SampleSeries` serialization dtype) instead of +`PointsInTimeSeries.dtype()` (`ArrayType(ArrayType(DoubleType))`). + +**Because the selector now carries its own `series_type` / `value_type` +([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)), this resolves with +no plan-time metadata lookup.** `EmptyTimeSeriesCache.load_blob` simply consults the +calling selector and returns an empty series of the matching kind: + +- a `SAMPLE` selector → empty `SampleSeries` (today's behavior); +- a numeric POI selector → empty numeric `PointsInTimeSeries`; +- a string POI selector → empty `PointsInTimeSeries` with `value_type = string`. + +`evaluation_type()` / `dtype()` are then correct for bare POI selections and for +expressions whose output type depends on the input type — and the string-op gating +fires **at build time**: `string_poi.mean()` builds an empty string point series +whose `mean()` raises `NotImplementedError`, so the selection is rejected up front +rather than producing a silent `NaN`, before Spark is involved. + +This removes the earlier need to pre-resolve each selector's type from +`channel_metrics` and inject it into the empty cache: the declared type on the +selector *is* the plan-time source. (The silver metadata still has the final say at +solve time via the [§4.2 assertion check](#42-query-api-and-carrying-the-discriminators-to-solve).) +It also mirrors how `PointsInTimeEvent` and `PointValueAggregator` already validate +`evaluation_type()` up front, so the mechanism is consistent with existing code. + +### 4.5 `PointsInTimeSeries` model change + +The one backend-model change (see [§3.4](#34-per-channel-value-dtype-double-vs-string)): + +- Add a second value array (dtype `object`) alongside the existing `float64` array, + and a `value_type` property (numeric / string) selecting which is active. +- Constructors/factories set `value_type`: the numeric path keeps today's + `np.array(values, dtype=np.float64)`; the string path stores values as an `object` + array and leaves the numeric array empty. +- Make `dtype()` return `ArrayType(ArrayType(StringType))` when `value_type` is + string (numeric unchanged). +- Implement **`__eq__` for string series** (synchronize on timestamps → compare + string values → `PointsInTime`). Have `__ne__`, `__lt__`, `__le__`, `__gt__`, + `__ge__`, the arithmetic operators, and the numeric reductions (`sum`, `mean`, + `min`, `max`) **raise `NotImplementedError`** when `value_type` is string. +- Leave `count`, `start_time` / `end_time`, `to_points_in_time`, `plane_sweep`, and + the timestamp handling in `synchronized` value-type-independent (they already are). + +### 4.6 Extend the existing test dataset with DTC POI channels + +Rather than build a bespoke POI fixture, **extend the existing session-scoped silver +dataset** so POI channels live alongside the current sample channels in the **same +containers** — this is what exercises the mix-and-match path (§4.3) end to end and +mirrors the [DTC motivating example](#11-motivating-example-ecu-defect--error-codes-dtcs). +The guiding constraint is **additive, non-destructive**: every existing test must +keep passing untouched. + +The `setup_basic_db` fixture (autouse, session-scoped) loads +`tests/unit/data/basic_narrow_csv/` into `spark_catalog.silver.*`. Use the concrete +rows from [§3.5](#35-example-tag--metric-entries-for-dtc-poi-channels) (DTC string +channel `channel_id = 90`, numeric count channel `channel_id = 91` on +`container_id = 1`) as the fixture data. The plan: + +1. **New `poi_channels` data file.** Add + `basic_narrow_csv/poi_channels.csv` with + `container_id, channel_id, timestamp, value_double, value_string, dtype` and a + couple of **DTC channels** on **existing** `container_id`s (e.g. a `DTC` string + channel with points like `(t₁, "P0301")`, `(t₂, "P0420")`, and a numeric POI + channel such as a fault-occurrence counter). Choose `channel_id`s **not already + used** by that container in `channel_data.csv` so the two sample sources stay + disjoint per the design invariant (a channel lives in exactly one of + `channels` / `poi_channels`). +2. **Append POI rows to `channel_metrics.csv`.** Add one row per new POI channel + carrying the new `series_type = POINTS_IN_TIME` column. **Backfill existing rows + with `series_type = SAMPLE`** (or leave blank and rely on the null ⇒ `SAMPLE` + default — pick one and be consistent). Existing sample channels are unaffected. +3. **Load `poi_channels` in the fixture.** Extend `setup_basic_db` to read the new + CSV and write `spark_catalog.silver.poi_channels`, and add its slot to the + `MeasurementDBConfig` used by the basic-db fixtures (`poi_channels_uri`). Because + `poi_channels_uri` defaults to `None`, **any db config that does not opt in is + unchanged**, so unrelated fixtures/tests see no difference. +4. **EAV + wide tag/metric parity.** So POI channels are *selectable* the same way + in both channel-selection modes: + - **EAV fixtures** (`setup_narrow_db`, `unit_test_csv/`): append POI rows to + `1_channel_tags.csv` (e.g. `channel_name = "DTC"`) and `1_channel_metrics.csv`, + plus any container-level tags/metrics needed, so a + `query.poi_channel(channel_name="DTC", dtype="string")` resolves the POI channel + through the pivot path (identification is identical to `channel(...)`; only the + selector's declared `series_type` / `value_type` differ — [§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)). + - **Wide fixtures** (`basic_narrow_csv`): the `channel_name` column already on + `channel_metrics` covers direct selection; just ensure the appended POI rows + carry a distinct `channel_name` (e.g. `"DTC"`). + +:::caution Two different columns both once called "value type" + +`basic_narrow_csv/channel_metrics.csv` **already** has a `value_type` column holding +values like `DOUBLE` (and the EAV `1_channel_metrics.csv` has `numerical`). That is +the **pre-existing** per-channel data-type column and is **not** the discriminator +this design adds. Keep them separate: + +- **existing `channel_metrics.value_type`** — untouched; describes the value data + type and is not read by the solver for routing. +- **new `channel_metrics.series_type`** — `SAMPLE` / `POINTS_IN_TIME`; routes to + `channels` vs `poi_channels` (§3.2). +- **new `poi_channels.dtype`** — `double` / `string`; selects `value_double` / + `value_string` (§3.4). + +Do **not** overload the existing `value_type` column for either new purpose — the +column names in the fixtures must stay distinct, and existing tests that read +`value_type` must be left as-is. + +::: + +**Regression guard.** Run the full existing suite after extending the fixtures and +confirm it is green *before* adding POI-specific tests (§7). Because the changes are +purely additive — new file, appended rows with a defaulting column, an opt-in table +slot — no existing assertion (row counts, computed means, dimension contents) should +move. If any does, the extension was not additive and must be corrected. + +## 5. What explicitly does **not** change + +- **The 6-stage filter pipeline.** `filter_container_tags` → + `filter_container_metrics` → `filter_channel_tags` → `filter_channel_metrics` → + alias resolution are untouched. POI channels are identified by the *same* + `TimeSeriesSelector` class, tag/column matching, and tag/metric filters as sample + channels — `poi_channel(...)` is a factory over the same selector, not a new + selection path ([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)). +- **`channels` table and RLE/interval encoders.** The sample-series read and + raw→interval encoding are untouched. +- **The sample-series *behavior* in the cache.** `TimeSeriesCache` gains a + per-channel dispatch (§4.3), but for a `SAMPLE` channel it builds the exact same + `SampleSeries(tstart, tend, value)` as today — the sample path's semantics and + output are unchanged. (This is a behavior guarantee, not a "no code changed" + claim: the cache does gain POI-aware branching.) +- **The single grouped-map UDF per container.** The solve stage still groups by + `container_id` and applies one UDF; POI does **not** add a second UDF or a + post-hoc union of two result sets. The input frame is widened to carry both series + types, not the execution model. +- **Persistence / gold layer.** Aggregations over POI series already reduce to + scalars (`mean`, `sum`, `count`, …) or `PointsInTime` events, which the existing + fact/dimension tables already accept (`PointsInTimeEvent`, `PointValueAggregator`). +- **`PointsInTimeSeries` for numeric (`double`) channels.** No new methods needed; + it already implements the full operator/sync/aggregation protocol for `float64` + values. (String POI is the exception — it requires the model change in + [§3.4](#34-per-channel-value-dtype-double-vs-string).) +- **`SampleSeries` interpolation semantics.** This design does not change how + sample-series values are reconstructed within `[tstart, tend)`. Zero-order hold + remains the only interpolation today; adding further interpolation methods later + is an **orthogonal** effort. The distinction that matters for POI is *validity* + (does a value exist between two timestamps at all?), not *which* interpolation is + applied where validity holds. + +## 6. Alternatives considered + +| Alternative | Why not chosen | +|-------------|----------------| +| **Store POI in `channels` with `tend == tstart`** | Overloads the "closed endpoint" meaning of zero-duration rows; forces every reader/encoder to disambiguate a whole POI channel from a sample-series endpoint. | +| **Store POI in the RAW `channels` (timestamp, value) format + a skip-encoding flag** | Couples POI to RAW mode and to the raw→interval encoder; a channel's storage shape would depend on an unrelated `data_type` setting. | +| **Overload the existing `value_type` column as the discriminator** | Conflates value *data type* with *series semantics*; two orthogonal concerns in one column, harder to reason about and to validate. | +| **A new dedicated POI solver class** | Unnecessary — the filter pipeline is shared and identical; only `load_blob` differs. A per-channel branch inside `DefaultSolver.solve` is far less code than a parallel solver. | +| **Two grouped-map UDFs (one SAMPLE, one POI), union the outputs** | **Incorrect** for the common mix-and-match case: each UDF sees only a subset of a container's channels, so an expression combining a POI and a sample channel (e.g. `poi - sample`) has a missing operand. Cross-type `synchronized` must run on both in-memory series inside **one** UDF. | + +## 7. Testing strategy + +Following the repo's fixture-reuse convention (CLAUDE.md → *Testing patterns*). +The POI tests run against the **extended shared dataset from [§4.6](#46-extend-the-existing-test-dataset-with-dtc-poi-channels)** +(DTC channels added to the existing `spark_catalog.silver.*` fixtures) rather than a +throwaway db, so they cover the real read path and the mix-and-match case: + +- Assert on **real computed values**, not row counts: e.g. a numeric POI `mean()` + equals the unweighted mean of the point values (contrast with the duration-weighted + `SampleSeries.mean()`, whose weighting follows from interval validity), proving the + between-point validity is genuinely absent. +- A **string POI** test: `query.poi_channel(channel_name="DTC", dtype="string")` + builds a `PointsInTimeSeries` with `value_type = string`; the **equality comparator** + (`== "P0301"` → `PointsInTime` on matching timestamps) and value-type-independent ops + (`count`, `to_points_in_time`, point sampling) work, while every **other comparator** + (`!=`, `<`, `<=`, `>`, `>=`), the arithmetic operators, and the numeric reductions + (`mean`, `sum`, `min`, `max`) raise `NotImplementedError` — asserted both directly on + the series object and, for a reduction inside a selection, at `evaluation_type()` + **build time** (not as a silent `NaN`, and before Spark runs). +- A **mix-and-match test (the primary correctness case)**: a single container owning + both a SAMPLE channel and a numeric POI channel, selected with `query.channel(...)` + and `query.poi_channel(...)` respectively, with **one expression referencing both** + (`rpm.where(dtc == "P0301")`, and `poi - sample`). This asserts both series land in + the *same* per-container pandas frame, are built by the unified cache, and align via + `synchronized` — the behavior a two-UDF design would break. Assert the computed + values, not just that it runs. +- A **declared-vs-actual `dtype` assertion test** ([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)): + `query.poi_channel(channel_name="DTC", dtype="double")` on a channel whose silver + `dtype` is `string` (or a `poi_channel` on a `SAMPLE` channel) raises a clear error + at solve time — the data stays authoritative, the wrong declaration is not silently + honored. +- A backward-compat test: a `channel_metrics` with no `series_type` column still + solves as SAMPLE, and existing `channel(...)` selections are unaffected by the new + optional selector fields. + +## 8. Open questions + +- **Should `series_type` be validated against the presence of data in the matching + table?** (e.g. a POI-marked channel with rows only in `channels`.) Proposed: + no hard validation initially; document that the marker is authoritative and the + non-matching table is not read for that channel. +- **~~Where should the POI value `dtype` be resolved for planning?~~ (Resolved.)** + The user declares `dtype` on `query.poi_channel(...)` and the selector carries it + ([§4.2](#42-query-api-and-carrying-the-discriminators-to-solve)), so plan-time + result typing (§4.4) needs **no** pre-pipeline `poi_channels` / `channel_metrics` + scan. The silver `poi_channels.dtype` remains authoritative at solve time and is + validated against the declared value (assertion contract). *Remaining sub-question:* + should the engine also support **inferring** `dtype` when the user omits it (rather + than defaulting to `double`) — e.g. a cheap `distinct` on `channel_metrics` — for + callers who prefer not to declare it? Proposed: keep the explicit `double` default + for now; add inference only if a concrete need appears. +- **Enforcing the constant-`dtype` invariant.** A channel is entirely numeric or + entirely string — `dtype` is constant per `(container_id, channel_id)` by + contract. This is a settled invariant, not an open question; the only decision is + whether to *defend* it. Proposed: an optional validate-and-raise (like the + unit-conversion conflict check) that flags any channel carrying more than one + distinct `dtype`, so a malformed ingest fails loudly instead of picking an + arbitrary value column. +- **String value column when scaling.** If more non-numeric dtypes appear later + (e.g. `bool`, `int`), revisit whether a typed-column-per-dtype layout still scales + or whether a single `value` string column + cast is preferable. +- **Calculated channels producing POI output.** `solve_calculated_channels` emits a + narrow `[container_id, channel_id, tstart, tend, value]` frame. Emitting a POI + *calculated* channel would need a narrow POI shape (`timestamp, value`). Deferred — + out of scope for ingesting POI *input* series. + +## 9. Aspects which differ from the design + +A few things landed differently than sections 3–4 describe. The shipped code is the +source of truth; those sections are left as the original proposal, and each spot that +changed points here. None of these change what the feature does — they mostly remove +machinery the design added that turned out to be unnecessary once the selector became +the source of truth for a channel's series type. + +### 9.1 No `series_type` column on `channel_metrics` (§3.2) + +The design added a `series_type` marker to `channel_metrics` so the solver could tell a +POI channel from a sample channel. We dropped it. A channel's data lives in exactly one +of `channels` or `poi_channels`, so **which table it comes from already tells us the +series type** — the extra column was redundant, and nothing ever read it at solve time. +Today the only way to get a `PointsInTimeSeries` is to read from `poi_channels`, so the +table membership is a complete answer. + +### 9.2 One value array, type inferred at construction (§3.4, §4.5) + +The design proposed keeping the numeric `float64` array and adding a *second* `object` +array for strings, with a `value_type` property choosing between them. In practice +`PointsInTimeSeries` keeps a **single** value array and infers whether it's string or +numeric from the values at construction time (an `_is_string` flag). It's less +bookkeeping — there's no pair of arrays to keep in sync, one always empty — and it +reads more naturally: you build the series from whatever values you have and it figures +out its own type. An explicit `empty_string()` factory covers the one case inference +can't (an empty series has nothing to infer from). + +### 9.3 String POI also supports `!=` (§3.4, §4.5) + +The design limited string POI series to equality (`==`) and had `!=` raise alongside +the ordering and arithmetic operators. We kept `!=` too. + +### 9.4 The declared-vs-actual check reads the data shape, not a marker (§4.2) + +The design validated the selector's declared `series_type` / `dtype` against the +`channel_metrics.series_type` column. With that column gone (9.1), the solve-time check +instead looks at the **data it resolved to** + +### 9.5 Series-type dispatch is driven by the selector, not a per-row column (§4.3) + +The design's solve stage stamped `series_type` (and `dtype`) onto every channel-data +row so the cache could inspect each slice. Since the selector already knows its own +type, we pass that into `load_blob` instead and drop the per-row markers from the frame +that crosses into the pandas UDF. Only `value_string` still rides along, because that's +real data a string channel needs, not a discriminator. The result is the same object +per channel with a bit less shipped across the Arrow boundary. + +### 9.6 Enum placement (§4.1) + +Minor: the design suggested putting `SeriesType` next to `RawEncoder` in +`solver_config.py`. It lives in `time_series_expression.py` instead, next to +`TimeSeriesSelector` (which carries it) and alongside the new `PoiValueType` enum. That's +where the selector-as-source-of-truth logic reads most naturally. diff --git a/src/impulse_query_engine/analyze/metadata/time_series_expression.py b/src/impulse_query_engine/analyze/metadata/time_series_expression.py index 0977dc3..87bd166 100644 --- a/src/impulse_query_engine/analyze/metadata/time_series_expression.py +++ b/src/impulse_query_engine/analyze/metadata/time_series_expression.py @@ -4,18 +4,50 @@ import operator import zlib from collections.abc import Callable, Iterable +from enum import StrEnum from typing import TYPE_CHECKING, Any import pyspark.sql.types as T import impulse_query_engine.util as U from impulse_query_engine.analyze.metadata.tag_expression import TagExpression +from impulse_query_engine.model.series.points_in_time_series import PointsInTimeSeries from impulse_query_engine.model.series.sample_series import SampleSeries if TYPE_CHECKING: from impulse_query_engine.analyze.query.solvers.series_cache import SeriesCache +class SeriesType(StrEnum): + """How a channel's samples are interpreted (mirrors :class:`RawEncoder`). + + ``SAMPLE`` — the default; ``[tstart, tend)`` intervals over which the value is + *valid* (reconstructed by an interpolation method, zero-order hold today), + backed by :class:`SampleSeries`. + + ``POINTS_IN_TIME`` — ``(tᵢ, vᵢ)`` points valid *only at* their timestamps, no + between-point validity, backed by :class:`PointsInTimeSeries`. + """ + + SAMPLE = "SAMPLE" + POINTS_IN_TIME = "POINTS_IN_TIME" + + +class PoiValueType(StrEnum): + """The value data type of a POI channel — selects its ``poi_channels`` value + column and which in-memory :class:`PointsInTimeSeries` variant is built. + + ``DOUBLE`` — numeric points (``poi_channels.value_double``); the full + arithmetic / ordering / reduction operator set applies. + + ``STRING`` — string points (``poi_channels.value_string``, e.g. DTC codes); + only sampling and equality apply (see :class:`PointsInTimeSeries`). + """ + + DOUBLE = "double" + STRING = "string" + + class RequiresDeserialization: pass @@ -619,7 +651,13 @@ def from_dict(obj: dict) -> TimeSeriesExpression: class TimeSeriesSelector(TimeSeriesExpression, RequiresDeserialization): - def __init__(self, expr, uses_alias: bool = False): + def __init__( + self, + expr, + uses_alias: bool = False, + series_type: SeriesType = SeriesType.SAMPLE, + value_type: PoiValueType = PoiValueType.DOUBLE, + ): """ Initialize a TimeSeriesSelector. @@ -627,18 +665,48 @@ def __init__(self, expr, uses_alias: bool = False): ---------- expr : TagExpression Tag expression to select. + uses_alias : bool, optional + Whether the channel resolves via the channel-alias table. + series_type : SeriesType, optional + How the selected channel's samples are interpreted. ``SAMPLE`` + (default) builds a :class:`SampleSeries` — today's behavior, + unchanged. ``POINTS_IN_TIME`` builds a :class:`PointsInTimeSeries` + (values valid only at their timestamps); identification / matching is + identical, only the built object and its result dtype differ. This is + the plan-time source of truth for the series type (so ``dtype()`` is + correct for a bare POI selection with no per-channel metadata lookup). + value_type : PoiValueType, optional + For a ``POINTS_IN_TIME`` selection, the declared value data type + (``DOUBLE`` / ``STRING``). Ignored for ``SAMPLE``. Drives plan-time + typing and string-op gating; validated against the silver + ``poi_channels.dtype`` at solve time (assertion contract). """ self._expr = expr self._uses_alias = uses_alias + self._series_type = series_type + self._value_type = value_type TimeSeriesExpression.__init__(self, is_single_signal=True) @property def uses_alias(self) -> bool: return self._uses_alias + @property + def series_type(self) -> SeriesType: + return self._series_type + + @property + def value_type(self) -> PoiValueType: + return self._value_type + @property def selector_id(self) -> int: - return zlib.crc32(str(self._expr).encode()) + # Include series_type so a SAMPLE and a POINTS_IN_TIME selection of the + # same tag expression resolve as distinct channels. SAMPLE keeps the + # historical id (bare ``str(expr)`` hash) for backward compatibility. + if self._series_type is SeriesType.SAMPLE: + return zlib.crc32(str(self._expr).encode()) + return zlib.crc32(f"{self._series_type}|{self._expr}".encode()) def dtype(self): """ @@ -647,13 +715,33 @@ def dtype(self): Returns ------- pyspark.sql.types.DataType - Data type (BinaryType). + ``BinaryType`` for a SAMPLE selection (serialized ``SampleSeries``), + or the value-type-aware ``PointsInTimeSeries.dtype()`` for a + POINTS_IN_TIME selection (``array>`` for numeric, + ``array>`` for string). """ + if self._series_type is SeriesType.POINTS_IN_TIME: + return self._empty_points_in_time().dtype() return T.BinaryType() + def _empty_points_in_time(self) -> PointsInTimeSeries: + """Empty POI series carrying this selector's declared value type. + + A string selector must build a string-typed empty series so ``dtype()`` + and the string-op gating (e.g. ``.mean()`` raising) reflect the declared + type before any data is read. + """ + if self._value_type is PoiValueType.STRING: + return PointsInTimeSeries.empty_string() + return PointsInTimeSeries.empty() + def deserialize(self, d): """ - Deserialize sample series after collection/toPandas. + Deserialize a SAMPLE result after collection/toPandas. + + POINTS_IN_TIME results are serialized by ``get_data()`` (a plain + ``[[t, v], ...]`` list) and need no deserialization, so they are returned + as-is; only a SAMPLE (binary) blob is decoded to a :class:`SampleSeries`. Parameters ---------- @@ -662,14 +750,26 @@ def deserialize(self, d): Returns ------- - SampleSeries - Deserialized sample series. + SampleSeries or Any + Deserialized sample series (SAMPLE), else *d* unchanged. """ + if self._series_type is SeriesType.POINTS_IN_TIME: + return d return SampleSeries.deserialize(d) - def build(self, cache: SeriesCache) -> SampleSeries: + def build(self, cache: SeriesCache): """ - Instantiate a SampleSeries from given cache data. + Instantiate the selected series from cache data. + + Resolution is identical regardless of series type — resolve the matching + candidates, take the first ``(container_id, channel_id)``, and let the + cache build the right object. The **data** is authoritative for the built + type: :meth:`TimeSeriesCache.load_blob` returns a + :class:`PointsInTimeSeries` for a ``POINTS_IN_TIME`` slice and a + :class:`SampleSeries` otherwise. The selector's own :attr:`series_type` / + :attr:`value_type` are used only for **plan-time** typing (:meth:`dtype` + against an empty cache), so a bare POI selection types correctly and a + string-only op is rejected before Spark runs. Parameters ---------- @@ -678,16 +778,25 @@ def build(self, cache: SeriesCache) -> SampleSeries: Returns ------- - SampleSeries - Built sample series. + SampleSeries or PointsInTimeSeries """ candidates = cache.resolve(self) if len(candidates) == 0: + if self._series_type is SeriesType.POINTS_IN_TIME: + return self._empty_points_in_time() return SampleSeries.empty() # TODO: select candidate mid = candidates.container_id.iloc[0] cid = candidates.channel_id.iloc[0] - return cache.load_blob(mid, cid, uses_alias=self.uses_alias) + # The selector is the source of truth for the series type: pass it to the + # cache so load_blob builds the right object without a per-row discriminator. + return cache.load_blob( + mid, + cid, + uses_alias=self.uses_alias, + series_type=self._series_type, + value_type=self._value_type, + ) def get_required_tag_exprs(self) -> set[TagExpression]: """ @@ -765,6 +874,8 @@ def as_dict(self) -> dict[str, Any]: obj["type"] = U.name_of(TimeSeriesSelector) obj["expr"] = self._expr.as_dict() obj["uses_alias"] = self._uses_alias + obj["series_type"] = str(self._series_type) + obj["value_type"] = str(self._value_type) return obj @staticmethod @@ -783,7 +894,14 @@ def from_dict(obj: dict): Selector instance. """ expr = TimeSeriesExpression.from_dict(obj["expr"]) - m = TimeSeriesSelector(expr, uses_alias=obj.get("uses_alias", False)) + # Default to SAMPLE / DOUBLE so selectors serialized before POI support + # (no series_type / value_type keys) deserialize unchanged. + m = TimeSeriesSelector( + expr, + uses_alias=obj.get("uses_alias", False), + series_type=SeriesType(obj.get("series_type", SeriesType.SAMPLE)), + value_type=PoiValueType(obj.get("value_type", PoiValueType.DOUBLE)), + ) if "alias" in obj and obj["alias"] is not None: m.alias(obj["alias"]) return m diff --git a/src/impulse_query_engine/analyze/query/query_builder.py b/src/impulse_query_engine/analyze/query/query_builder.py index 26ac47e..a97b465 100644 --- a/src/impulse_query_engine/analyze/query/query_builder.py +++ b/src/impulse_query_engine/analyze/query/query_builder.py @@ -7,7 +7,9 @@ from impulse_query_engine.analyze.metadata.metric_expression import MetricSelector from impulse_query_engine.analyze.metadata.tag_expression import TagSelector from impulse_query_engine.analyze.metadata.time_series_expression import ( + PoiValueType, RequiresDeserialization, + SeriesType, TimeSeriesExpression, TimeSeriesSelector, ) @@ -161,6 +163,48 @@ def channel_with_alias(self, **kwargs) -> TimeSeriesSelector: expr = expr & (TagSelector(k) == str(arg)) return TimeSeriesSelector(expr, uses_alias=True) + def poi_channel( + self, dtype: PoiValueType = PoiValueType.DOUBLE, **kwargs + ) -> TimeSeriesSelector: + """ + Create a Points-in-Time (POI) channel selector. + + Parallel to :meth:`channel` — it builds the **same** ``TimeSeriesSelector`` + from a tag/column match on ``**kwargs`` (e.g. + ``poi_channel(channel_name="DTC")``), differing only in that it is stamped + ``series_type=POINTS_IN_TIME`` (so it solves to a + :class:`~impulse_query_engine.model.series.points_in_time_series.PointsInTimeSeries` + — a value valid only *at* each timestamp — rather than a ``SampleSeries``) + and carries the declared value ``dtype``. + + Channel *identification* (tag/column match, ``get_selector_expr``, + ``required_tags``, ``selector_id``) is identical to :meth:`channel`; only + the built object and its result dtype differ. + + Parameters + ---------- + dtype : PoiValueType, optional + The POI channel's value data type: ``DOUBLE`` (default, numeric) or + ``STRING`` (e.g. DTC codes — only sampling and equality apply). This + declared type drives plan-time result typing and string-op gating; it + is validated against the silver ``poi_channels.dtype`` at solve time + (an actual/declared mismatch raises). + **kwargs : dict + Channel tag-value pairs, matched exactly like :meth:`channel`'s. + + Returns + ------- + TimeSeriesSelector + A selector stamped ``series_type=POINTS_IN_TIME`` with the given value type. + """ + expr = None + for k, arg in kwargs.items(): + if not expr: + expr = TagSelector(k) == str(arg) + else: + expr = expr & (TagSelector(k) == str(arg)) + return TimeSeriesSelector(expr, series_type=SeriesType.POINTS_IN_TIME, value_type=dtype) + def select(self, *args) -> Self: """ Set the selection expressions for the query. diff --git a/src/impulse_query_engine/analyze/query/solvers/blob_solver.py b/src/impulse_query_engine/analyze/query/solvers/blob_solver.py index 6e5bced..833f749 100644 --- a/src/impulse_query_engine/analyze/query/solvers/blob_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/blob_solver.py @@ -49,10 +49,15 @@ def resolve(self, selection): idx = selection._expr.build_pandas(self.df) return self.df[idx] - def load_blob(self, container_id, channel_id, uses_alias: bool = False): + def load_blob( + self, container_id, channel_id, uses_alias: bool = False, series_type=None, value_type=None + ): """ Load a time series blob from disk. + ``series_type`` / ``value_type`` are accepted for interface compatibility + with :class:`SeriesCache`; this blob cache serves only SAMPLE series. + Parameters ---------- container_id : Any diff --git a/src/impulse_query_engine/analyze/query/solvers/default_solver.py b/src/impulse_query_engine/analyze/query/solvers/default_solver.py index 5e2e5d7..ef0a527 100644 --- a/src/impulse_query_engine/analyze/query/solvers/default_solver.py +++ b/src/impulse_query_engine/analyze/query/solvers/default_solver.py @@ -11,6 +11,11 @@ from impulse_query_engine.analyze.metadata.metric_expression import MetricExpression from impulse_query_engine.analyze.metadata.tag_expression import TagExpression +from impulse_query_engine.analyze.metadata.time_series_expression import ( + PoiValueType, + SeriesType, +) +from impulse_query_engine.model.series.points_in_time_series import PointsInTimeSeries from impulse_query_engine.model.series.sample_series import SampleSeries from .query_solver import QuerySolver @@ -43,7 +48,11 @@ def __init__(self, pdf, col_map: dict[str, str]): col_map : dict[str, str] Mapping with keys ``"cid"``, ``"ch"``, ``"ts"``, ``"te"``, ``"val"``, ``"conv"`` to the actual column names in *pdf*. The - ``"conv"`` column is optional in *pdf*. + ``"conv"`` column is optional in *pdf*. For a POI (``POINTS_IN_TIME``) + selector, :meth:`load_blob` builds a :class:`PointsInTimeSeries` — the + **selector** (not a per-row column) chooses the series type; the + ``"value_string"`` key names the string value column that a string POI + slice reads. """ self._cid_col = col_map["cid"] self._ch_col = col_map["ch"] @@ -52,6 +61,9 @@ def __init__(self, pdf, col_map: dict[str, str]): self._val_col = col_map["val"] self._conv_col = col_map.get("conv") self._has_conversion = self._conv_col is not None and self._conv_col in pdf.columns + # String POI slices read their value from this column; series-type dispatch + # is driven by the selector passed to load_blob, not a per-row marker. + self._value_string_col = col_map.get("value_string") # *pdf* holds channel data for a whole container, so avoid creating unnecessary copies of the data. meta_cols = [ @@ -98,13 +110,20 @@ def resolve(self, selection): idx = selection._expr.build_pandas(self.mdf) return self.mdf[idx] - def load_blob(self, mid, cid, uses_alias: bool = False): + def load_blob(self, mid, cid, uses_alias: bool = False, series_type=None, value_type=None): """ Load a time series blob from the DataFrame. + The **calling selector** chooses the series type (via *series_type* / + *value_type*), so no per-row discriminator column is needed: a + ``POINTS_IN_TIME`` selector yields a :class:`PointsInTimeSeries` (string- + valued when *value_type* is ``STRING``, else numeric), otherwise a + :class:`SampleSeries`. The declared type is validated against the silver + metadata in the solve prelude, so the data stays authoritative. + When the underlying *pdf* carries a conversion-factor column (the column named by ``col_map["conv"]``) **and** the caller is an - aliased selector (``uses_alias=True``), the returned values are + aliased selector (``uses_alias=True``), the returned SAMPLE values are multiplied by that factor. Direct selectors on the same physical channel always receive raw values — unit conversion is a property of the alias, not of the channel. @@ -118,14 +137,28 @@ def load_blob(self, mid, cid, uses_alias: bool = False): uses_alias : bool, optional ``True`` when the calling selector resolved via channel_mapping. Gates the per-channel conversion factor; defaults to ``False``. + series_type : SeriesType, optional + The calling selector's series type; ``POINTS_IN_TIME`` builds a + :class:`PointsInTimeSeries`. ``None`` (default) => SAMPLE. + value_type : PoiValueType, optional + For a POI selector, its declared value type; ``STRING`` reads the + string value column, otherwise the numeric one. Returns ------- - SampleSeries - The loaded sample series object. + SampleSeries or PointsInTimeSeries """ lo, hi = self._ranges.get((mid, cid), (0, 0)) s = self.pdf.iloc[lo:hi] + + if series_type == SeriesType.POINTS_IN_TIME: + self._assert_poi_data(s, value_type) + if value_type == PoiValueType.STRING: + # value_string is a populated string column, so the constructor + # infers the string value type from it. + return PointsInTimeSeries(s[self._ts_col], s[self._value_string_col]) + return PointsInTimeSeries(s[self._ts_col], s[self._val_col]) + values = s[self._val_col] if self._has_conversion and len(s) > 0 and uses_alias: factor = s[self._conv_col].iloc[0] @@ -133,6 +166,52 @@ def load_blob(self, mid, cid, uses_alias: bool = False): values = values * factor return SampleSeries(s[self._ts_col], s[self._te_col], values) + def _assert_poi_data(self, s, value_type) -> None: + """Validate a POI selector against the data it resolved to. + + The selector drives series-type dispatch, but the silver data stays + authoritative: a ``poi_channel(...)`` selector must land on genuine POI + rows. POI rows carry a null ``tend`` (a point has no validity interval), + whereas SAMPLE rows always carry a real ``tend`` (non-nullable in + ``channels``); so a non-null ``tend`` on a POI-declared slice means the + selector was pointed at a SAMPLE channel. A ``STRING`` declaration + additionally requires a populated ``value_string``. Either mismatch raises + rather than silently reading the wrong column (mirrors the unit-conversion + conflict check). + """ + if len(s) == 0: + return + if pd.notna(s[self._te_col].iloc[0]): + raise ValueError( + "POI channel series-type mismatch: poi_channel(...) resolved to a SAMPLE " + "channel (its rows carry a validity interval). Use channel(...) for SAMPLE " + "channels and poi_channel(...) for POINTS_IN_TIME channels." + ) + + has_string_col = self._value_string_col is not None and self._value_string_col in s.columns + string_all_null = has_string_col and s[self._value_string_col].isna().all() + double_all_null = s[self._val_col].isna().all() + + if value_type == PoiValueType.STRING: + # A string POI channel must carry string values; all-null means the + # channel is actually numeric (declared the wrong dtype). + if not has_string_col or string_all_null: + raise ValueError( + "POI channel dtype mismatch: poi_channel(dtype=string) resolved to a channel " + "with no string values (it is a numeric POI channel). Pass dtype=double to " + "poi_channel(...)." + ) + else: + # A numeric POI channel must carry numeric values; all-null numeric + # with populated string values means the channel is actually a string + # channel (declared the wrong dtype). + if double_all_null and has_string_col and not string_all_null: + raise ValueError( + "POI channel dtype mismatch: poi_channel(dtype=double) resolved to a channel " + "whose numeric values are all null (it is a string POI channel). Pass " + "dtype=string to poi_channel(...)." + ) + class DefaultSolver(QuerySolver): """ @@ -1026,6 +1105,15 @@ def _prepare_channels_join(self, query, channels_df) -> tuple[DataFrame, DataFra self.config.value_col, ) + # POI channel data is unioned in AFTER RLE encoding above, so its + # zero-duration points are never run-length merged. The inner join to + # channels_df below drops any POI rows whose channel was not selected, so + # unioning whenever a poi_channels table is configured is correct (a + # pure-SAMPLE query simply matches no POI channel_ids). Which object each + # channel builds is decided by the selector (passed to load_blob), not a + # per-row marker — SAMPLE rows just lack value_string. + q = self._union_poi_channel_data(query, q) + joined_df = q.join( F.broadcast(channels_df), on=[self.config.container_id_col, self.config.channel_id_col], @@ -1033,6 +1121,35 @@ def _prepare_channels_join(self, query, channels_df) -> tuple[DataFrame, DataFra container_count = channels_df.select(self.config.container_id_col).distinct().count() return q, joined_df, container_count + def _union_poi_channel_data(self, query, channels_q: DataFrame) -> DataFrame: + """Union POI channel-data rows into the (already-encoded) channel-data frame. + + Reads ``poi_channels``, column-maps it, and projects it into the SAMPLE + channel-data superset — POI ``timestamp`` becomes ``tstart`` (``tend`` + **null**, since a point has no validity interval, which is also the signal + the cache validates a POI selector against), ``value_double`` becomes the + numeric ``value`` column, and ``value_string`` rides alongside for a string + POI channel. No per-row ``series_type`` / ``dtype`` marker is shipped: the + selector drives series-type dispatch in :meth:`TimeSeriesCache.load_blob`. + Returns *channels_q* unchanged when no ``poi_channels`` table is configured. + """ + db = query.db + if not (hasattr(db, "has_poi_channels") and db.has_poi_channels()): + return channels_q + + cfg = self.config + poi = db.poi_channels(self.spark) + poi = self._apply_column_mapping(poi, cfg.poi_channels.column_name_mapping) + poi_proj = poi.select( + F.col(cfg.container_id_col), + F.col(cfg.channel_id_col), + F.col(cfg.poi_timestamp_col).alias(cfg.tstart_col), + F.lit(None).cast(T.LongType()).alias(cfg.tend_col), + F.col(cfg.poi_value_double_col).alias(cfg.value_col), + F.col(cfg.poi_value_string_col).alias(cfg.poi_value_string_col), + ) + return channels_q.unionByName(poi_proj, allowMissingColumns=True) + def _apply_grouped_map(self, joined_df, container_count, schema, solve_udf) -> DataFrame: """Run *solve_udf* per container, or return an empty frame when none match.""" if container_count == 0: diff --git a/src/impulse_query_engine/analyze/query/solvers/empty_cache.py b/src/impulse_query_engine/analyze/query/solvers/empty_cache.py index 32ca0d1..9664126 100644 --- a/src/impulse_query_engine/analyze/query/solvers/empty_cache.py +++ b/src/impulse_query_engine/analyze/query/solvers/empty_cache.py @@ -25,7 +25,7 @@ def resolve(self, selection): """ return [] - def load_blob(self, mid, cid, uses_alias: bool = False): + def load_blob(self, mid, cid, uses_alias: bool = False, series_type=None, value_type=None): """ Return an empty SampleSeries for any container and channel ID. @@ -38,6 +38,11 @@ def load_blob(self, mid, cid, uses_alias: bool = False): uses_alias : bool, optional Unused by this cache; accepted for interface compatibility with :class:`SeriesCache`. + series_type, value_type : optional + Accepted for interface compatibility. The empty-series typing for a + POI selector is handled by ``TimeSeriesSelector.build`` (its length-0 + branch), which returns the correctly typed empty series without + reaching this method. Returns ------- diff --git a/src/impulse_query_engine/analyze/query/solvers/series_cache.py b/src/impulse_query_engine/analyze/query/solvers/series_cache.py index 081d53b..8debf8d 100644 --- a/src/impulse_query_engine/analyze/query/solvers/series_cache.py +++ b/src/impulse_query_engine/analyze/query/solvers/series_cache.py @@ -24,7 +24,14 @@ def resolve(self, selection) -> pd.DataFrame: pass @abstractmethod - def load_blob(self, mid, cid, uses_alias: bool = False) -> SampleSeries: + def load_blob( + self, + mid, + cid, + uses_alias: bool = False, + series_type=None, + value_type=None, + ) -> SampleSeries: """ Resolve given mid and cid to a series. @@ -41,10 +48,22 @@ def load_blob(self, mid, cid, uses_alias: bool = False) -> SampleSeries: conversion factor when this is ``True``, so a direct selector on the same physical channel always returns raw values. Defaults to ``False`` (direct / no-conversion semantics). + series_type : SeriesType, optional + The calling selector's series type. The selector — not a per-row + data column — is the source of truth for which object to build: + ``POINTS_IN_TIME`` builds a :class:`PointsInTimeSeries`, otherwise a + :class:`SampleSeries`. ``None`` (default) means SAMPLE, so callers + that predate POI are unchanged. + value_type : PoiValueType, optional + For a ``POINTS_IN_TIME`` selector, its declared value type + (``DOUBLE`` / ``STRING``) — selects the numeric vs string value + column. Ignored for SAMPLE. The declared type is validated against + the silver metadata in the solve prelude, so the data stays + authoritative. Returns ------- - SampleSeries - The loaded sample series object. + SampleSeries or PointsInTimeSeries + The loaded series object. """ pass diff --git a/src/impulse_query_engine/analyze/query/solvers/solver_config.py b/src/impulse_query_engine/analyze/query/solvers/solver_config.py index 0ca7113..0eb1122 100644 --- a/src/impulse_query_engine/analyze/query/solvers/solver_config.py +++ b/src/impulse_query_engine/analyze/query/solvers/solver_config.py @@ -143,6 +143,7 @@ class SolverConfig(BaseModel): channel_metrics: TableConfig = TableConfig() channel_mapping: ChannelMappingConfig = ChannelMappingConfig() channels: TableConfig = TableConfig() + poi_channels: TableConfig = TableConfig() unit_conversion: TableConfig = TableConfig() # ------------------------------------------------------------------ @@ -231,6 +232,26 @@ def value_col(self) -> str: """Internal column name for the signal value on the channels table.""" return "value" + @property + def poi_timestamp_col(self) -> str: + """Internal column name for the point timestamp on the poi_channels table.""" + return "timestamp" + + @property + def poi_value_double_col(self) -> str: + """Internal column name for the numeric value on the poi_channels table.""" + return "value_double" + + @property + def poi_value_string_col(self) -> str: + """Internal column name for the string value on the poi_channels table.""" + return "value_string" + + @property + def poi_dtype_col(self) -> str: + """Internal column name for the per-row value-dtype discriminator on poi_channels.""" + return "dtype" + @property def tag_key_col(self) -> str: """Internal column name for the attribute key on the container_tags (EAV) table.""" @@ -379,4 +400,8 @@ def col_map(self) -> dict[str, str]: "te": self.tend_col, "val": self.value_col, "conv": self.conversion_factor_col, + # String POI slices read their value from this column. Series-type + # dispatch is driven by the selector (passed to load_blob), so no + # per-row series_type / dtype marker column is needed in the frame. + "value_string": self.poi_value_string_col, } diff --git a/src/impulse_query_engine/measurement_db.py b/src/impulse_query_engine/measurement_db.py index c0ba27c..3ebdcde 100644 --- a/src/impulse_query_engine/measurement_db.py +++ b/src/impulse_query_engine/measurement_db.py @@ -14,6 +14,7 @@ def __init__( channel_tags_table=None, channel_metrics_table=None, channels_uri=None, + poi_channels_uri=None, channel_mapping_table=None, unit_conversion_table=None, table_locations: str = "external_locations", @@ -23,6 +24,9 @@ def __init__( self.channel_tags_table = channel_tags_table self.channel_metrics_table = channel_metrics_table self.channels_uri = channels_uri + # Optional Points-in-Time (POI) channel-data table. ``None`` means no POI + # channels are configured, so POI-unaware deployments are unchanged. + self.poi_channels_uri = poi_channels_uri self.channel_mapping_table = channel_mapping_table self.unit_conversion_table = unit_conversion_table self.table_locations = table_locations @@ -34,6 +38,7 @@ def for_unity_catalog( core_schema_name: str = "core", channel_mapping_table: str | None = None, unit_conversion_table: str | None = None, + poi_channels_uri: str | None = None, ): return MeasurementDBConfig( container_tags_table=f"{catalog_name}.{core_schema_name}.container_tags", @@ -41,6 +46,7 @@ def for_unity_catalog( channel_tags_table=f"{catalog_name}.{core_schema_name}.channel_tags", channel_metrics_table=f"{catalog_name}.{core_schema_name}.channel_metrics", channels_uri=f"{catalog_name}.{core_schema_name}.channels", + poi_channels_uri=poi_channels_uri, channel_mapping_table=channel_mapping_table, unit_conversion_table=unit_conversion_table, table_locations="unity_catalog", @@ -58,6 +64,7 @@ def for_debug(debug_tables): "channel_metrics" if "channel_metrics" in debug_tables else None ), channels_uri="channels" if "channels" in debug_tables else None, + poi_channels_uri="poi_channels" if "poi_channels" in debug_tables else None, channel_mapping_table=( "channel_mapping" if "channel_mapping" in debug_tables else None ), @@ -103,6 +110,20 @@ def channel_metrics(self, spark) -> DataFrame: def channels(self, spark) -> DataFrame: return self._read_table(spark, self.config.channels_uri) + def has_poi_channels(self) -> bool: + """Whether a Points-in-Time (POI) channel-data table is configured.""" + return getattr(self.config, "poi_channels_uri", None) is not None + + def poi_channels(self, spark) -> DataFrame: + """Read the Points-in-Time (POI) channel-data table. + + Parallel to :meth:`channels`. Raises if no ``poi_channels_uri`` is + configured — callers should gate on :meth:`has_poi_channels` first. + """ + if not self.has_poi_channels(): + raise ValueError("poi_channels_uri is not configured") + return self._read_table(spark, self.config.poi_channels_uri) + def channel_mapping(self, spark) -> DataFrame: if self.config.channel_mapping_table is None: raise ValueError("channel_mapping_table is not configured") diff --git a/src/impulse_query_engine/model/series/points_in_time_series.py b/src/impulse_query_engine/model/series/points_in_time_series.py index 192a73d..44d8dcd 100644 --- a/src/impulse_query_engine/model/series/points_in_time_series.py +++ b/src/impulse_query_engine/model/series/points_in_time_series.py @@ -53,6 +53,11 @@ def __init__(self, tstarts: Sized, values: Sized): a value is only defined *at* its timestamp and is not considered valid in between consecutive timestamps. + The value type (numeric vs string) is inferred from *values*. An **empty** + series has no values to infer from and therefore defaults to numeric; use + :meth:`empty_string` when an explicitly string-typed empty series is needed + (e.g. plan-time result typing of a bare string-POI selection). + Parameters ---------- tstarts : Sized @@ -65,8 +70,6 @@ def __init__(self, tstarts: Sized, values: Sized): # string-valued series support sampling (``synchronized`` / ``.where``) # and equality comparisons (``==`` / ``!=``) only — arithmetic, ordering # and numeric reductions are rejected (see the ``@_numeric_only`` methods). - # An empty series has no observed value type, so it defaults to numeric - # (the safe, backward-compatible case). self.tstarts = np.array(tstarts, dtype=np.float64) self._is_string = np.asarray(values).dtype.kind in ("U", "S", "O") if self._is_string: @@ -604,11 +607,34 @@ def __repr__(self) -> str: @staticmethod def empty() -> PointsInTimeSeries: """ - Returns an empty PointsInTimeSeries. + Returns an empty (numeric) PointsInTimeSeries. Returns ------- PointsInTimeSeries - Empty PointsInTimeSeries object. + Empty numeric PointsInTimeSeries object. """ return PointsInTimeSeries([], []) + + @staticmethod + def empty_string() -> PointsInTimeSeries: + """ + Returns an empty **string-valued** PointsInTimeSeries. + + An empty series has no values to infer a type from, so the constructor + defaults to numeric; this factory forces the string value type. Used for + plan-time result typing of a bare string-POI selection, where the empty + series must report the string ``dtype()`` and reject numeric-only ops + (e.g. ``mean()``) before any data is read. + + Returns + ------- + PointsInTimeSeries + Empty string-valued PointsInTimeSeries object. + """ + # A single-element object array makes the constructor infer string, then + # slice back to empty so no value is retained. + series = PointsInTimeSeries([], []) + series._is_string = True + series.values = np.asarray([], dtype=object) + return series diff --git a/src/impulse_query_engine/schema.py b/src/impulse_query_engine/schema.py index 8323182..f64465b 100644 --- a/src/impulse_query_engine/schema.py +++ b/src/impulse_query_engine/schema.py @@ -52,6 +52,26 @@ ] ) +# Points-in-Time (POI) channel samples: a value defined only *at* its timestamp +# (no derived tend / validity interval). Two typed value columns plus a per-row +# dtype discriminator, since a POI value may be numeric or a string; exactly one of +# value_double / value_string is populated per row, selected by dtype. +# +# A channel is a POI channel iff its data lives here rather than in ``channels`` — +# table membership *is* the series-type discriminator, so no ``series_type`` column +# is needed on ``channel_metrics``. A given (container_id, channel_id) lives in +# exactly one of ``channels`` / ``poi_channels``. +POI_CHANNELS_SCHEMA = T.StructType( + [ + T.StructField("container_id", T.LongType(), nullable=False), + T.StructField("channel_id", T.IntegerType(), nullable=False), + T.StructField("timestamp", T.LongType(), nullable=False), + T.StructField("value_double", T.DoubleType()), + T.StructField("value_string", T.StringType()), + T.StructField("dtype", T.StringType(), nullable=False), + ] +) + CHANNELS_SCHEMA = T.StructType( [ T.StructField("container_id", T.LongType(), nullable=False), diff --git a/src/impulse_reporting/config/config_parser.py b/src/impulse_reporting/config/config_parser.py index f8e3d0b..84f4915 100644 --- a/src/impulse_reporting/config/config_parser.py +++ b/src/impulse_reporting/config/config_parser.py @@ -151,6 +151,7 @@ class Source(BaseModel): channel_mapping_table: Annotated[str, AfterValidator(is_valid_table_name)] | None = None unit_conversion_table: Annotated[str, AfterValidator(is_valid_table_name)] | None = None + #todo probably add poi here as well so users can configure it class UnitySink(BaseModel): """ diff --git a/tests/conftest.py b/tests/conftest.py index 0ea8731..d571329 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,6 +47,7 @@ def basic_narrow_db(spark, mock_workspace_client) -> MeasurementDB: tables["container_metrics"] = spark.read.table("spark_catalog.silver.container_metrics") tables["channel_metrics"] = spark.read.table("spark_catalog.silver.channel_metrics") tables["channels"] = spark.read.table("spark_catalog.silver.channels") + tables["poi_channels"] = spark.read.table("spark_catalog.silver.poi_channels") cfg = MeasurementDBConfig.for_debug(tables) return MeasurementDB(cfg, ws=mock_workspace_client) @@ -96,6 +97,20 @@ def setup_narrow_db(spark): ), schema=S.CHANNELS_SCHEMA, ) + poi_channels = spark.createDataFrame( + pd.read_csv( + f"{base_path}/tests/unit/data/unit_test_csv/1_poi_channels.csv", + dtype={ + "container_id": np.int64, + "channel_id": np.int32, + "timestamp": np.longlong, + "value_double": np.float64, + "value_string": "object", + "dtype": "object", + }, + ), + schema=S.POI_CHANNELS_SCHEMA, + ) container_tags.write.format("delta").mode("overwrite").saveAsTable( "spark_catalog.silver_narrow_db.container_tags" @@ -112,6 +127,9 @@ def setup_narrow_db(spark): channels.write.format("delta").mode("overwrite").saveAsTable( "spark_catalog.silver_narrow_db.channels" ) + poi_channels.write.format("delta").mode("overwrite").saveAsTable( + "spark_catalog.silver_narrow_db.poi_channels" + ) @pytest.fixture(scope="session", autouse=True) @@ -131,11 +149,20 @@ def setup_basic_db(spark): container_metric_path = f"{base_path}/tests/unit/data/basic_narrow_csv/container_metrics.csv" channel_metric_path = f"{base_path}/tests/unit/data/basic_narrow_csv/channel_metrics.csv" channels_path = f"{base_path}/tests/unit/data/basic_narrow_csv/channel_data.csv" + poi_channels_path = f"{base_path}/tests/unit/data/basic_narrow_csv/poi_channels.csv" options = {"header": "True", "delimiter": ",", "inferSchema": "True"} container_metrics = spark.read.options(**options).csv(container_metric_path) channel_metrics = spark.read.options(**options).csv(channel_metric_path) channels = spark.read.options(**options).csv(channels_path) + # POI channel data: explicit schema so empty value columns keep their nullable + # typed shape (value_double double / value_string string) rather than being + # inferred as all-null strings. + poi_channels = ( + spark.read.schema(S.POI_CHANNELS_SCHEMA) + .options(header="True", delimiter=",") + .csv(poi_channels_path) + ) container_metrics.write.format("delta").mode("overwrite").saveAsTable( "spark_catalog.silver.container_metrics" @@ -150,6 +177,9 @@ def setup_basic_db(spark): "spark_catalog.silver.channel_metrics" ) channels.write.format("delta").mode("overwrite").saveAsTable("spark_catalog.silver.channels") + poi_channels.write.format("delta").mode("overwrite").saveAsTable( + "spark_catalog.silver.poi_channels" + ) @pytest.fixture(scope="session") @@ -236,6 +266,7 @@ def narrow_db(spark, setup_narrow_db, mock_workspace_client) -> MeasurementDB: "spark_catalog.silver_narrow_db.channel_metrics" ) debug_tables["channels"] = spark.read.table("spark_catalog.silver_narrow_db.channels") + debug_tables["poi_channels"] = spark.read.table("spark_catalog.silver_narrow_db.poi_channels") cfg = MeasurementDBConfig.for_debug(debug_tables) return MeasurementDB(cfg, ws=mock_workspace_client) diff --git a/tests/impulse_query_engine/integration/poi_channel_solve_test.py b/tests/impulse_query_engine/integration/poi_channel_solve_test.py new file mode 100644 index 0000000..8b032ff --- /dev/null +++ b/tests/impulse_query_engine/integration/poi_channel_solve_test.py @@ -0,0 +1,243 @@ +"""End-to-end integration tests for Points-in-Time (POI) channels. + +Exercises the full solve pipeline for ``query.poi_channel(...)`` against the shared +``basic_narrow_db`` (wide) and ``narrow_db`` (EAV) fixtures, which carry POI channels +on ``container_id = 1`` alongside the existing sample channels (see conftest / +``poi_channels.csv``): + +- ``channel_id = 90`` — a **string** DTC-code channel (``P0301`` / ``P0420`` / ``P0301``) +- ``channel_id = 91`` — a **numeric** DTC-count channel (values ``1, 2, 3``) + +Covers: numeric POI unweighted reductions, string POI equality + op gating, the +mix-and-match case (a SAMPLE and a POI channel in one expression), the declared-vs-actual +dtype/series-type assertion, and SAMPLE backward-compatibility. +""" + +import math + +import pytest +import pyspark.sql.types as T +from pyspark.sql import SparkSession + +from impulse_query_engine.analyze.metadata.time_series_expression import PoiValueType +from impulse_query_engine.analyze.query.solvers.default_solver import DefaultSolver +from impulse_query_engine.measurement_db import MeasurementDB + + +class TestNumericPoi: + def test_numeric_poi_mean_is_unweighted(self, spark: SparkSession, basic_narrow_db): + """A numeric POI ``mean()`` is the plain (unweighted) mean of the point values — + POI points have no duration to weight by, unlike ``SampleSeries.mean()``.""" + solver = DefaultSolver(spark) + q = basic_narrow_db.query + dtc_count = q.poi_channel(channel_name="DTC_count") # values 1, 2, 3 + + result = q.select(dtc_count.mean().alias("m")).solve(spark=spark, solver=solver) + + rows = {r.container_id: r.m for r in result.collect()} + assert rows[1] == 2.0 # unweighted mean of (1, 2, 3) + + def test_numeric_poi_sum_and_count(self, spark: SparkSession, basic_narrow_db): + solver = DefaultSolver(spark) + q = basic_narrow_db.query + dtc_count = q.poi_channel(channel_name="DTC_count") + + result = q.select( + dtc_count.sum().alias("s"), + dtc_count.count().alias("c"), + ).solve(spark=spark, solver=solver) + + row = {r.container_id: r for r in result.collect()}[1] + assert row.s == 6.0 # 1 + 2 + 3 + assert row.c == 3 + + def test_bare_numeric_poi_selection_types_as_points_in_time( + self, spark: SparkSession, basic_narrow_db + ): + """A bare numeric POI selection serializes as ``array>`` + (PointsInTimeSeries), not the SAMPLE ``binary`` blob type.""" + solver = DefaultSolver(spark) + q = basic_narrow_db.query + dtc_count = q.poi_channel(channel_name="DTC_count").alias("pit") + + result = q.select(dtc_count).solve(spark=spark, solver=solver) + + assert result.schema["pit"].dataType == T.ArrayType(T.ArrayType(T.DoubleType())) + rows = {r.container_id: r.pit for r in result.collect()} + # three points [t, v], values 1..3 (unweighted, in timestamp order) + assert [pt[1] for pt in rows[1]] == [1.0, 2.0, 3.0] + + +class TestStringPoi: + def test_string_poi_equality_selects_matching_instants( + self, spark: SparkSession, basic_narrow_db + ): + """``string_poi == "P0301"`` yields the instants where the code equals P0301. + + Sampling the count channel at those instants (via ``.where``) picks out the two + P0301 occurrences, proving the string equality drove the point selection. + """ + solver = DefaultSolver(spark) + q = basic_narrow_db.query + dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + + # DTC == "P0301" is a PointsInTime; serialize it directly. + result = q.select((dtc == "P0301").alias("hits")).solve(spark=spark, solver=solver) + + rows = {r.container_id: r.hits for r in result.collect()} + # P0301 occurs at the 1st and 3rd of the three DTC timestamps. + assert len(rows[1]) == 2 + + def test_string_poi_count_and_sampling_allowed(self, spark: SparkSession, basic_narrow_db): + solver = DefaultSolver(spark) + q = basic_narrow_db.query + dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + + result = q.select(dtc.count().alias("c")).solve(spark=spark, solver=solver) + + assert {r.container_id: r.c for r in result.collect()}[1] == 3 + + @pytest.mark.parametrize("reduction", ["mean", "sum", "min", "max"]) + def test_string_poi_numeric_reduction_rejected_at_build( + self, spark: SparkSession, basic_narrow_db, reduction + ): + """A numeric reduction on a string POI selection is rejected at plan/build time + (before Spark runs), not as a silent NaN.""" + q = basic_narrow_db.query + dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + selection = getattr(dtc, reduction)().alias("bad") + with pytest.raises(TypeError, match="string-valued"): + q.select(selection)._determine_result_objects_dtypes() + + +class TestMixAndMatch: + """The primary correctness case: a SAMPLE and a POI channel in one expression, both + in the same per-container pandas frame, aligned via ``synchronized``.""" + + def test_sample_channel_sampled_at_poi_instants(self, spark: SparkSession, narrow_db): + """Sample the ``seed`` SAMPLE channel at the instants of the numeric POI channel. + + narrow_db container 1: ``seed`` sample channel has values 1..10 over t=0..10; the + numeric POI channel (91) has points at t = 2, 5, 8. Sampling seed at those instants + picks the seed values valid there. + """ + solver = DefaultSolver(spark) + q = narrow_db.query + seed = q.channel(seed="0") + dtc_count = q.poi_channel(channel_name="DTC_count") + + # Sample the sample-series at the POI points (cross-type synchronize). + result = q.select(seed.where(dtc_count.to_points_in_time()).alias("sampled")).solve( + spark=spark, solver=solver + ) + + rows = {r.container_id: r.sampled for r in result.collect()} + # three sampled points at the POI instants t = 2, 5, 8 + assert [pt[0] for pt in rows[1]] == [2.0, 5.0, 8.0] + for pt in rows[1]: + assert not math.isnan(pt[1]) + + def test_string_poi_and_sample_freeze_frame(self, spark: SparkSession, narrow_db): + """Freeze-frame: sample the seed channel at the instants where DTC == "P0301".""" + solver = DefaultSolver(spark) + q = narrow_db.query + seed = q.channel(seed="0") + dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + + result = q.select(seed.where(dtc == "P0301").alias("frozen")).solve( + spark=spark, solver=solver + ) + + rows = {r.container_id: r.frozen for r in result.collect()} + # P0301 at t = 2 and 8 (EAV fixture); seed sampled there. + assert [pt[0] for pt in rows[1]] == [2.0, 8.0] + + def test_sample_channel_sampled_at_poi_instants_wide( + self, spark: SparkSession, basic_narrow_db + ): + """Wide-mode counterpart of the mix-and-match case (``basic_narrow_db``). + + Uses a POI numeric channel to filter/sample a real sample channel. Only + "Ambient Air Temperature" (channel 6) spans all three POI instants in the + ``basic_narrow_csv`` fixture (the other channels end earlier), so it is the + channel sampled here. Proves POI-drives-channel-selection works through the + wide (columns-on-channel_metrics) path, not just the EAV pivot path. + """ + solver = DefaultSolver(spark) + q = basic_narrow_db.query + amb = q.channel(channel_name="Ambient Air Temperature") + dtc_count = q.poi_channel(channel_name="DTC_count") # points at 3 POI instants + + result = q.select( + amb.where(dtc_count.to_points_in_time()).alias("sampled") + ).solve(spark=spark, solver=solver) + + rows = {r.container_id: r.sampled for r in result.collect()} + # The three POI instants (microsecond epochs) from basic_narrow_csv/poi_channels.csv. + poi_instants = [1499929300000000.0, 1499931000000000.0, 1499933000000000.0] + assert [pt[0] for pt in rows[1]] == poi_instants + # Each instant sampled a real Ambient-Air-Temp value (not a miss / NaN). + for pt in rows[1]: + assert not math.isnan(pt[1]) + + def test_string_poi_freeze_frame_wide(self, spark: SparkSession, basic_narrow_db): + """Wide-mode freeze-frame: sample "Ambient Air Temperature" where DTC == "P0301". + + In ``basic_narrow_csv`` P0301 occurs at the 1st and 3rd DTC instants, so the + string-POI equality predicate selects exactly those two instants of the + sample channel — the freeze-frame case resolved via the wide channel path. + """ + solver = DefaultSolver(spark) + q = basic_narrow_db.query + amb = q.channel(channel_name="Ambient Air Temperature") + dtc = q.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + + result = q.select(amb.where(dtc == "P0301").alias("frozen")).solve( + spark=spark, solver=solver + ) + + rows = {r.container_id: r.frozen for r in result.collect()} + # P0301 at the 1st and 3rd instants (basic_narrow_csv/poi_channels.csv). + assert [pt[0] for pt in rows[1]] == [1499929300000000.0, 1499933000000000.0] + for pt in rows[1]: + assert not math.isnan(pt[1]) + + +class TestDeclaredVsActual: + def test_poi_channel_declared_double_on_string_channel_raises( + self, spark: SparkSession, basic_narrow_db + ): + """Declaring ``dtype=double`` on a channel whose silver dtype is ``string`` raises + at solve time — the data stays authoritative.""" + solver = DefaultSolver(spark) + q = basic_narrow_db.query + # DTC is a numeric-less (string) channel; declaring double resolves rows + # whose value_double is all null → dtype mismatch raised in the solve UDF. + bad = q.poi_channel(channel_name="DTC", dtype=PoiValueType.DOUBLE) + with pytest.raises(Exception, match="dtype mismatch"): + q.select(bad.count().alias("c")).solve(spark=spark, solver=solver).collect() + + def test_poi_channel_on_sample_channel_raises(self, spark: SparkSession, basic_narrow_db): + """``poi_channel`` on a SAMPLE channel raises the series-type mismatch. + + The SAMPLE channel's rows carry a real (non-null) validity interval, which + is the signal load_blob validates a POI-declared selector against. + """ + solver = DefaultSolver(spark) + q = basic_narrow_db.query + bad = q.poi_channel(channel_name="Engine RPM") + with pytest.raises(Exception, match="series-type mismatch"): + q.select(bad.count().alias("c")).solve(spark=spark, solver=solver).collect() + + +class TestBackwardCompat: + def test_sample_channel_unaffected_by_poi(self, spark: SparkSession, basic_narrow_db): + """An ordinary SAMPLE ``channel(...)`` selection is unchanged by POI support.""" + solver = DefaultSolver(spark) + q = basic_narrow_db.query + rpm = q.channel(channel_name="Engine RPM") + + result = q.select(rpm.mean().alias("rpm_mean")).solve(spark=spark, solver=solver) + + rows = {r.container_id for r in result.collect()} + assert rows == {1, 2, 3} diff --git a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_column_mapping_test.py b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_column_mapping_test.py index 0e53a80..b236f0e 100644 --- a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_column_mapping_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_column_mapping_test.py @@ -611,6 +611,7 @@ def test_col_map_always_returns_internal_names(self, spark): "te": "tend", "val": "value", "conv": "conversion_factor", + "value_string": "value_string", } def test_mapping_entries_stored_correctly(self, spark): diff --git a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_only_test.py b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_only_test.py index 9ab2b68..b74b1cd 100644 --- a/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_only_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/solvers/default_solver_wide_only_test.py @@ -505,6 +505,7 @@ def test_col_map_always_returns_internal_names(self, spark: SparkSession): "te": "tend", "val": "value", "conv": "conversion_factor", + "value_string": "value_string", } def test_config_properties_return_internal_names(self, spark: SparkSession): diff --git a/tests/impulse_query_engine/unit/analyze/query/solvers/solver_config_test.py b/tests/impulse_query_engine/unit/analyze/query/solvers/solver_config_test.py index c692c2e..524b97d 100644 --- a/tests/impulse_query_engine/unit/analyze/query/solvers/solver_config_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/solvers/solver_config_test.py @@ -38,6 +38,7 @@ "te": "tend", "val": "value", "conv": "conversion_factor", + "value_string": "value_string", } @@ -148,7 +149,15 @@ class TestColMap: def test_col_map_keys(self, cfg: SolverConfig): """col_map should contain exactly the expected short keys.""" - assert set(cfg.col_map.keys()) == {"cid", "ch", "ts", "te", "val", "conv"} + assert set(cfg.col_map.keys()) == { + "cid", + "ch", + "ts", + "te", + "val", + "conv", + "value_string", + } def test_col_map_default_config(self): """Default SolverConfig col_map should match hardcoded defaults.""" @@ -160,6 +169,7 @@ def test_col_map_default_config(self): "te": "tend", "val": "value", "conv": "conversion_factor", + "value_string": "value_string", } def test_col_map_consistent_with_properties(self, cfg: SolverConfig): diff --git a/tests/unit/data/basic_narrow_csv/channel_metrics.csv b/tests/unit/data/basic_narrow_csv/channel_metrics.csv index d43f432..887cb03 100644 --- a/tests/unit/data/basic_narrow_csv/channel_metrics.csv +++ b/tests/unit/data/basic_narrow_csv/channel_metrics.csv @@ -11,3 +11,5 @@ container_id,channel_id,channel_name,group_idx,channel_idx,unit,sample_count,min 2,6,Ambient Air Temperature,2,2,C,57240,21,33,28.793081761006288,1499367269349000,1499372240481000,4971132000,11.514480001738036,DOUBLE 1,7,Vehicle Speed Sensor,3,1,km/h,59625,0,217,68.22906498951782,1499929242072000,1499934640063000,5397991000,11.045776104480352,DOUBLE 1,5,Engine RPM,2,1,RPM,59625,0,3658,1490.707790356394,1499929242072000,1499934640063000,5397991000,11.045776104480352,DOUBLE +1,90,DTC,0,0,,3,,,,1499929300000000,1499933000000000,3700000,,STRING +1,91,DTC_count,0,0,,3,1,3,2.0,1499929300000000,1499933000000000,3700000,,DOUBLE diff --git a/tests/unit/data/unit_test_csv/1_channel_metrics.csv b/tests/unit/data/unit_test_csv/1_channel_metrics.csv index 46dc975..59749d5 100644 --- a/tests/unit/data/unit_test_csv/1_channel_metrics.csv +++ b/tests/unit/data/unit_test_csv/1_channel_metrics.csv @@ -1,2 +1,4 @@ container_id,channel_id,value_type,sample_count,nan_ratio,begin_s,end_s,duration_ms,original_sample_count,original_sr,min,max,mean,std,pz1,pz10,pz90,pz99 1,1,numerical,1,1.0,0.0,100.0,1,1,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0 +1,90,string,3,,2.0,8.0,0,3,,,,,,,,, +1,91,numerical,3,,2.0,8.0,0,3,,1.0,3.0,2.0,,,,, diff --git a/tests/unit/data/unit_test_csv/1_channel_tags.csv b/tests/unit/data/unit_test_csv/1_channel_tags.csv index b572465..4120877 100644 --- a/tests/unit/data/unit_test_csv/1_channel_tags.csv +++ b/tests/unit/data/unit_test_csv/1_channel_tags.csv @@ -1,2 +1,4 @@ container_id,channel_id,key,value 1,1,seed,0 +1,90,channel_name,DTC +1,91,channel_name,DTC_count From 74a3cb9a01589aa5547dd7448c5ad4a7d3d40f71 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 15:07:10 +0200 Subject: [PATCH 14/17] added poi example to notebook Added missing poi in Source Basemodel --- demos/data/reporting/channel_metrics.csv | 38 ++--- demos/reporting_pipeline.ipynb | 144 +++++++++++++++++- .../analyze/query/query_builder.py | 12 +- src/impulse_reporting/config/config_parser.py | 6 +- .../unit/analyze/query/query_builder_test.py | 55 +++++++ .../unit/config/config_parser_test.py | 51 +++++++ 6 files changed, 278 insertions(+), 28 deletions(-) diff --git a/demos/data/reporting/channel_metrics.csv b/demos/data/reporting/channel_metrics.csv index 7846174..46fe67c 100644 --- a/demos/data/reporting/channel_metrics.csv +++ b/demos/data/reporting/channel_metrics.csv @@ -1,19 +1,19 @@ -container_id,channel_id,sample_count,min,max,mean,begin_ms,end_ms,duration_ms,sample_rate,value_type,series_type -1,4,56667,-11.0,-3.0,-8.35955670848995,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE -1,5,56667,0.0,3385.0,1572.765489614767,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE -1,7,56667,0.0,201.0,70.67870188998889,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE -1,9,56667,-33.0,130.0,2.3332627455132617,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE,SAMPLE -2,2,47336,-8.0,6.0,-6.105247591684975,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE -2,5,47336,0.0,3177.0,1736.7744422849416,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE -2,7,47336,0.0,188.0,83.95660807841811,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE -2,8,47336,-26.0,117.0,0.8931046138245733,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE,SAMPLE -3,2,54775,-32.0,134.0,9.632861706983112,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE -3,7,54775,0.0,2545.0,1308.4330990415335,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE -3,9,54775,0.0,125.0,44.4629849383843,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE -3,10,54775,-3.0,2.0,-2.157334550433592,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE,SAMPLE -1,90,3,,,,1519629856439000,1519633356439000,3500000,,STRING,POINTS_IN_TIME -1,91,3,1.0,3.0,2.0,1519629856439000,1519633356439000,3500000,,DOUBLE,POINTS_IN_TIME -2,90,2,,,,1519756824107000,1519758824107000,2000000,,STRING,POINTS_IN_TIME -2,91,2,1.0,2.0,1.5,1519756824107000,1519758824107000,2000000,,DOUBLE,POINTS_IN_TIME -3,90,1,,,,1519926478375000,1519926478375000,0,,STRING,POINTS_IN_TIME -3,91,1,1.0,1.0,1.0,1519926478375000,1519926478375000,0,,DOUBLE,POINTS_IN_TIME +container_id,channel_id,sample_count,min,max,mean,begin_ms,end_ms,duration_ms,sample_rate,value_type +1,4,56667,-11.0,-3.0,-8.35955670848995,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE +1,5,56667,0.0,3385.0,1572.765489614767,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE +1,7,56667,0.0,201.0,70.67870188998889,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE +1,9,56667,-33.0,130.0,2.3332627455132617,1519629356439000,1519634174599000,4818160000,11.76112872963953,DOUBLE +2,2,47336,-8.0,6.0,-6.105247591684975,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE +2,5,47336,0.0,3177.0,1736.7744422849416,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE +2,7,47336,0.0,188.0,83.95660807841811,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE +2,8,47336,-26.0,117.0,0.8931046138245733,1519755824107000,1519760198065000,4373958000,10.822234689953586,DOUBLE +3,2,54775,-32.0,134.0,9.632861706983112,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE +3,7,54775,0.0,2545.0,1308.4330990415335,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE +3,9,54775,0.0,125.0,44.4629849383843,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE +3,10,54775,-3.0,2.0,-2.157334550433592,1519924978375000,1519929815329000,4836954000,11.32427556681333,DOUBLE +1,90,3,,,,1519629856439000,1519633356439000,3500000,,STRING +1,91,3,1.0,3.0,2.0,1519629856439000,1519633356439000,3500000,,DOUBLE +2,90,2,,,,1519756824107000,1519758824107000,2000000,,STRING +2,91,2,1.0,2.0,1.5,1519756824107000,1519758824107000,2000000,,DOUBLE +3,90,1,,,,1519926478375000,1519926478375000,0,,STRING +3,91,1,1.0,1.0,1.0,1519926478375000,1519926478375000,0,,DOUBLE diff --git a/demos/reporting_pipeline.ipynb b/demos/reporting_pipeline.ipynb index edc9c41..cb9b31c 100644 --- a/demos/reporting_pipeline.ipynb +++ b/demos/reporting_pipeline.ipynb @@ -482,6 +482,51 @@ ")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3a. Select POI Channels (Diagnostic Trouble Codes)\n", + "\n", + "Not every channel is a continuous signal. A **Points-in-Time (POI)** channel is\n", + "an *event stream*: each value exists **only at its timestamp**, with no validity\n", + "in between. The textbook example is **DTCs** (Diagnostic Trouble Codes) \u2014 the\n", + "fault codes an ECU emits at the instant it detects a problem (`P0301` = cylinder-1\n", + "misfire, \u2026).\n", + "\n", + "POI channels are selected with **`poi_channel(...)`** instead of `channel(...)`.\n", + "Identification is identical (same metadata tags); only the semantics differ \u2014\n", + "a POI channel solves to a `PointsInTimeSeries`, not a `SampleSeries`.\n", + "\n", + "| | `channel(...)` | `poi_channel(...)` |\n", + "|---|---|---|\n", + "| Shape | `[tstart, tend)` intervals | `(t\u1d62, v\u1d62)` points |\n", + "| Valid between points? | yes (interpolated) | **no** |\n", + "| Backed by | `SampleSeries` | `PointsInTimeSeries` |" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "# String POI channel: the DTC code emitted at each fault instant.\n", + "# dtype=\"string\" -> equality is the natural operation (\"when did P0301 occur?\"),\n", + "# never arithmetic or ordering on a code.\n", + "dtc = db.query.poi_channel(\n", + " channel_name=\"DTC\",\n", + " dtype=\"string\",\n", + " brand=\"Seat\", model=\"Leon\",\n", + ")\n", + "\n", + "# Numeric POI channel: a running fault-occurrence counter.\n", + "dtc_count = db.query.poi_channel(\n", + " channel_name=\"DTC_count\",\n", + " brand=\"Seat\", model=\"Leon\",\n", + ")" + ] + }, { "cell_type": "markdown", "metadata": { @@ -561,7 +606,8 @@ "\n", "- **BasicEvent** \u2014 from a TSAL boolean expression\n", "- **ContainerEvent** \u2014 spans the entire recording\n", - "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone)" + "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone)\n", + "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone, or each **P0301 misfire** from the DTC channel)" ] }, { @@ -608,7 +654,17 @@ " expr=distance_milestones,\n", " desc=\"Each 10 km driven (instant)\",\n", ")\n", - "report.add_event(milestone_event)" + "report.add_event(milestone_event)\n", + "\n", + "# POI freeze-frame: the instants a P0301 misfire was logged.\n", + "# `dtc == \"P0301\"` is a PointsInTime \u2014 the set of timestamps where the\n", + "# string code equals P0301 \u2014 exactly what a PointsInTimeEvent wants.\n", + "p0301_event = PointsInTimeEvent(\n", + " name=\"p0301_misfires\",\n", + " expr=(dtc == \"P0301\"),\n", + " desc=\"Each instant a P0301 misfire code was set\",\n", + ")\n", + "report.add_event(p0301_event)" ] }, { @@ -721,6 +777,19 @@ " event=milestone_event,\n", " desc=\"Speed & RPM at each 10 km milestone\",\n", "))\n", + "\n", + "# \u2500\u2500 POI aggregations \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n", + "# Freeze-frame: Engine RPM & Vehicle Speed at each P0301 misfire instant.\n", + "# The POI event supplies the timestamps; the sample channels supply the\n", + "# values valid there \u2014 both series types in one aggregation.\n", + "page.add_aggregation(PointValueAggregator(\n", + " name=\"values_at_p0301\",\n", + " input_expressions=[eng_rpm, veh_spd],\n", + " channel_names=[\"Engine RPM\", \"Vehicle Speed\"],\n", + " event=p0301_event,\n", + " desc=\"RPM & Speed at each P0301 misfire\",\n", + "))\n", + "\n", "print(f\"{len(page.aggregations)} aggregations added\")" ] }, @@ -754,6 +823,34 @@ "print(\"Calculated channel 'avg_temp' registered\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Numeric POI: fault counts per recording\n", + "\n", + "A numeric POI channel reduces like any signal \u2014 but the reductions are\n", + "**unweighted** (points have no duration). `count()` / `max()` on `dtc_count`\n", + "answer \"how many faults did each recording log?\" directly from the query engine.\n", + "\n", + "(The report-level `StatsAggregator` is designed for continuous `SampleSeries`\n", + "inputs, so a per-container POI count is shown here as a direct query instead.)" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "dtc_summary = db.query.select(\n", + " dtc_count.count().alias(\"n_faults\"),\n", + " dtc_count.max().alias(\"peak_count\"),\n", + ").solve(spark=spark, solver=report.get_solver())\n", + "\n", + "display(dtc_summary.orderBy(\"container_id\"))" + ] + }, { "cell_type": "markdown", "metadata": { @@ -824,7 +921,9 @@ "- **Heatmap** \u2014 RPM vs Speed\n", "- **Table** \u2014 per-container statistics\n", "- **Scatter** \u2014 Speed & RPM at each 10 km milestone\n", - " (markers only \u2014 values exist only *at* each instant)" + " (markers only \u2014 values exist only *at* each instant)\n", + "\n", + "Includes two **POI** views: Engine RPM sampled at each P0301 misfire (freeze-frame), and fault-code counts per recording." ] }, { @@ -980,7 +1079,42 @@ "ax.set_title(\"Speed & RPM at Each 10 km Milestone\")\n", "ax.legend()\n", "plt.tight_layout()\n", - "plt.show()" + "plt.show()\n", + "\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 5. POI \u2014 Engine RPM at Each P0301 Misfire (freeze-frame)\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# PointValueAggregator writes to the shared stats_aggregator_fact table;\n", + "# select its visual by name, like the milestone scatter above.\n", + "p0301_df = (\n", + " spark.read.table(f\"{T}_stats_aggregator_fact\")\n", + " .join(\n", + " spark.read.table(f\"{T}_stats_aggregator_dimension\")\n", + " .filter(\"name = 'values_at_p0301'\"),\n", + " on=\"visual_id\",\n", + " )\n", + " .select(\"container_id\", \"channel_name\", \"event_instance_id\", \"statistic_value\")\n", + " .toPandas()\n", + ")\n", + "\n", + "if not p0301_df.empty:\n", + " rpm_hits = p0301_df[p0301_df[\"channel_name\"] == \"Engine RPM\"]\n", + " fig, ax = plt.subplots(figsize=(10, 4))\n", + " for cid, grp in rpm_hits.groupby(\"container_id\"):\n", + " ax.scatter(\n", + " grp[\"event_instance_id\"], grp[\"statistic_value\"],\n", + " label=f\"Container {cid}\", s=90, alpha=0.85,\n", + " edgecolors=\"k\", linewidths=0.5,\n", + " )\n", + " ax.set_xlabel(\"P0301 occurrence #\")\n", + " ax.set_ylabel(\"Engine RPM at fault instant\")\n", + " ax.set_title(\"Freeze-frame: Engine RPM at each P0301 misfire\")\n", + " ax.legend()\n", + " plt.tight_layout()\n", + " plt.show()\n", + "else:\n", + " print(\"No P0301 misfires in the demo data.\")\n", + "" ] }, { @@ -1204,4 +1338,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} +} \ No newline at end of file diff --git a/src/impulse_query_engine/analyze/query/query_builder.py b/src/impulse_query_engine/analyze/query/query_builder.py index a97b465..26469b8 100644 --- a/src/impulse_query_engine/analyze/query/query_builder.py +++ b/src/impulse_query_engine/analyze/query/query_builder.py @@ -183,9 +183,10 @@ def poi_channel( Parameters ---------- - dtype : PoiValueType, optional + dtype : PoiValueType or str, optional The POI channel's value data type: ``DOUBLE`` (default, numeric) or - ``STRING`` (e.g. DTC codes — only sampling and equality apply). This + ``STRING`` (e.g. DTC codes — only sampling and equality apply). Accepts + either the enum or its string value (``"double"`` / ``"string"``). This declared type drives plan-time result typing and string-op gating; it is validated against the silver ``poi_channels.dtype`` at solve time (an actual/declared mismatch raises). @@ -197,13 +198,18 @@ def poi_channel( TimeSeriesSelector A selector stamped ``series_type=POINTS_IN_TIME`` with the given value type. """ + # Accept a plain string ("string" / "double") as well as the enum, so + # poi_channel(..., dtype="string") behaves identically to the enum form. + value_type = PoiValueType(dtype) expr = None for k, arg in kwargs.items(): if not expr: expr = TagSelector(k) == str(arg) else: expr = expr & (TagSelector(k) == str(arg)) - return TimeSeriesSelector(expr, series_type=SeriesType.POINTS_IN_TIME, value_type=dtype) + return TimeSeriesSelector( + expr, series_type=SeriesType.POINTS_IN_TIME, value_type=value_type + ) def select(self, *args) -> Self: """ diff --git a/src/impulse_reporting/config/config_parser.py b/src/impulse_reporting/config/config_parser.py index 84f4915..634b215 100644 --- a/src/impulse_reporting/config/config_parser.py +++ b/src/impulse_reporting/config/config_parser.py @@ -128,6 +128,10 @@ class Source(BaseModel): Full Unity Catalog path to the channel metrics table. channels_uri : str Full Unity Catalog path to the channels data table. + poi_channels_uri : str, optional + Full Unity Catalog path to the Points-in-Time (POI) channel data table. + Required only when the report selects POI channels via ``poi_channel()``; + omit it for sample-only data models. channel_mapping_table : str, optional Full Unity Catalog path to the channel mapping table. Required when using ``channel_with_alias()`` for logical alias resolution. @@ -148,10 +152,10 @@ class Source(BaseModel): container_metrics_table: Annotated[str, AfterValidator(is_valid_table_name)] channel_metrics_table: Annotated[str, AfterValidator(is_valid_table_name)] channels_uri: Annotated[str, AfterValidator(is_valid_table_name)] + poi_channels_uri: Annotated[str, AfterValidator(is_valid_table_name)] | None = None channel_mapping_table: Annotated[str, AfterValidator(is_valid_table_name)] | None = None unit_conversion_table: Annotated[str, AfterValidator(is_valid_table_name)] | None = None - #todo probably add poi here as well so users can configure it class UnitySink(BaseModel): """ diff --git a/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py b/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py index 008b811..ef361f9 100644 --- a/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py +++ b/tests/impulse_query_engine/unit/analyze/query/query_builder_test.py @@ -4,6 +4,8 @@ from impulse_query_engine.analyze.metadata.tag_expression import TagSelector from impulse_query_engine.analyze.metadata.time_series_expression import ( + PoiValueType, + SeriesType, TimeSeriesSelector, ) from impulse_query_engine.model.series import Intervals @@ -195,3 +197,56 @@ def test_timeseries_selector_dtype_matches_sample_series_dtype(): ts = TimeSeriesSelector(TagSelector("name") == "test") ss = SampleSeries.empty() assert ts.dtype() == ss.dtype() + + +# --------------------------------------------------------------------------- +# QueryBuilder.poi_channel — dtype accepts the enum OR its string value +# --------------------------------------------------------------------------- +class TestPoiChannelDtypeArg: + """``poi_channel(dtype=...)`` must accept both ``PoiValueType`` and the plain + string value (``"double"`` / ``"string"``). A regression guard: a plain + ``dtype="string"`` used to be stored verbatim (a ``str``, not the enum), so the + ``is PoiValueType.STRING`` identity checks silently fell through and a string + POI channel behaved as numeric — blowing up on the first string comparison. + """ + + def test_default_dtype_is_double(self, narrow_db): + sel = narrow_db.query.poi_channel(channel_name="DTC_count") + assert sel.series_type is SeriesType.POINTS_IN_TIME + assert sel.value_type is PoiValueType.DOUBLE + + def test_enum_string_dtype(self, narrow_db): + sel = narrow_db.query.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + assert sel.value_type is PoiValueType.STRING + + def test_plain_string_dtype_coerced_to_enum(self, narrow_db): + # the design-doc form: poi_channel(..., dtype="string") + sel = narrow_db.query.poi_channel(channel_name="DTC", dtype="string") + assert sel.value_type is PoiValueType.STRING + + def test_plain_string_double_dtype_coerced_to_enum(self, narrow_db): + sel = narrow_db.query.poi_channel(channel_name="DTC_count", dtype="double") + assert sel.value_type is PoiValueType.DOUBLE + + def test_invalid_dtype_raises(self, narrow_db): + with pytest.raises(ValueError): + narrow_db.query.poi_channel(channel_name="DTC", dtype="int") + + def test_string_poi_types_as_struct_regardless_of_arg_form(self, narrow_db): + # both arg forms must produce an identical string-typed result dtype + enum_sel = narrow_db.query.poi_channel(channel_name="DTC", dtype=PoiValueType.STRING) + str_sel = narrow_db.query.poi_channel(channel_name="DTC", dtype="string") + assert enum_sel.dtype() == str_sel.dtype() + # string POI serializes as array>, not array> + assert isinstance(str_sel.dtype(), T.ArrayType) + assert isinstance(str_sel.dtype().elementType, T.StructType) + + def test_string_poi_equality_evaluates_to_points_in_time(self, narrow_db): + # dtype="string" must yield a string series so `== "code"` works at plan time + dtc = narrow_db.query.poi_channel(channel_name="DTC", dtype="string") + assert (dtc == "P0301").evaluation_type() is PointsInTime + + def test_string_poi_mean_rejected_at_build_time(self, narrow_db): + dtc = narrow_db.query.poi_channel(channel_name="DTC", dtype="string") + with pytest.raises(TypeError, match="string-valued"): + dtc.mean().evaluation_type() diff --git a/tests/impulse_reporting/unit/config/config_parser_test.py b/tests/impulse_reporting/unit/config/config_parser_test.py index 64189af..34c9992 100644 --- a/tests/impulse_reporting/unit/config/config_parser_test.py +++ b/tests/impulse_reporting/unit/config/config_parser_test.py @@ -113,6 +113,57 @@ def test_impulse_config_drop_implausible_data_enabled(): assert config.query_engine.drop_implausible_data is True +# --------------------------------------------------------------------------- +# Source.poi_channels_uri — must survive parsing AND reach the MeasurementDB. +# Regression: the field was missing from the Source model, so pydantic silently +# dropped it and the whole reporting-layer POI path was inert (has_poi_channels +# always False) even when the config supplied a poi_channels_uri. +# --------------------------------------------------------------------------- +def test_source_poi_channels_uri_parsed(): + config_json = { + **impulse_config_JSON, + "source": { + **impulse_config_JSON["source"], + "poi_channels_uri": "impulse_demo.silver.poi_channels", + }, + } + config = ImpulseConfig.model_validate(config_json) + assert config.source.poi_channels_uri == "impulse_demo.silver.poi_channels" + + +def test_source_poi_channels_uri_defaults_to_none(): + config = ImpulseConfig.model_validate(impulse_config_JSON) + assert config.source.poi_channels_uri is None + + +def test_poi_channels_uri_reaches_measurement_db(): + """End-to-end passthrough: a poi_channels_uri in the config makes the built + MeasurementDB POI-aware (this is what was silently broken).""" + from unittest.mock import create_autospec + + from databricks.sdk import WorkspaceClient + + from impulse_reporting.core.report import Report + + with_poi = ImpulseConfig.model_validate( + { + **impulse_config_JSON, + "source": { + **impulse_config_JSON["source"], + "poi_channels_uri": "impulse_demo.silver.poi_channels", + }, + } + ) + db = Report.create_measurement_db(with_poi, create_autospec(WorkspaceClient)) + assert db.has_poi_channels() + assert db.config.poi_channels_uri == "impulse_demo.silver.poi_channels" + + # ...and a config without it stays POI-unaware. + without_poi = ImpulseConfig.model_validate(impulse_config_JSON) + db2 = Report.create_measurement_db(without_poi, create_autospec(WorkspaceClient)) + assert not db2.has_poi_channels() + + def test_impulse_config_drop_implausible_data_rejects_rle(): """drop_implausible_data=True with RLE data must raise ValidationError. From 7eb0af31704d45472a3b35859c0147860e3c11c4 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 15:27:21 +0200 Subject: [PATCH 15/17] added more poi features to the demo notebook --- demos/reporting_pipeline.ipynb | 77 ++++++++++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 4 deletions(-) diff --git a/demos/reporting_pipeline.ipynb b/demos/reporting_pipeline.ipynb index cb9b31c..fee38e6 100644 --- a/demos/reporting_pipeline.ipynb +++ b/demos/reporting_pipeline.ipynb @@ -583,7 +583,17 @@ "\n", "# Instant the trip odometer crosses each additional\n", "# 10 km \u2014 a set of points in time, not an interval.\n", - "distance_milestones = (distance_km % 10).falling_edges()" + "distance_milestones = (distance_km % 10).falling_edges()\n", + "\n", + "# POI as a time-window anchor. A POI instant has no duration, but we often\n", + "# want the signal *around* it. `.expand(w)` turns each P0301 instant into a\n", + "# [t - w, t + w] interval (w in the data's time unit \u2014 microseconds here),\n", + "# so `\u00b110 s` is 10e6. Overlapping windows are merged.\n", + "WINDOW_US = 10e6 # \u00b110 seconds\n", + "p0301_window = (dtc == \"P0301\").expand(WINDOW_US)\n", + "\n", + "# All Engine RPM samples recorded within \u00b110 s of a misfire.\n", + "rpm_around_p0301 = eng_rpm.where(p0301_window)" ] }, { @@ -607,7 +617,8 @@ "- **BasicEvent** \u2014 from a TSAL boolean expression\n", "- **ContainerEvent** \u2014 spans the entire recording\n", "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone)\n", - "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone, or each **P0301 misfire** from the DTC channel)" + "- **PointsInTimeEvent** \u2014 a set of instants (e.g. each 10 km milestone, or each **P0301 misfire** from the DTC channel)\n", + "- **BasicEvent on a POI window** \u2014 `(dtc == \"P0301\").expand(\u00b110 s)` turns each fault instant into an interval, so you can aggregate the signal *around* each event" ] }, { @@ -664,7 +675,16 @@ " expr=(dtc == \"P0301\"),\n", " desc=\"Each instant a P0301 misfire code was set\",\n", ")\n", - "report.add_event(p0301_event)" + "report.add_event(p0301_event)\n", + "\n", + "# The \u00b110 s misfire windows as an interval event, so aggregations can run\n", + "# \"within 10 s of a P0301\" the same way they run within any other event.\n", + "p0301_window_event = BasicEvent(\n", + " name=\"p0301_window\",\n", + " expr=p0301_window,\n", + " desc=\"Within \u00b110 s of a P0301 misfire\",\n", + ")\n", + "report.add_event(p0301_window_event)" ] }, { @@ -790,6 +810,19 @@ " desc=\"RPM & Speed at each P0301 misfire\",\n", "))\n", "\n", + "\n", + "# Derived-value-around-POI: min/mean/max of the continuous signals in the\n", + "# \u00b110 s window around each misfire. `eng_rpm.where(p0301_window)` is a\n", + "# SampleSeries (values valid over the window), so StatsAggregator applies.\n", + "page.add_aggregation(StatsAggregator(\n", + " name=\"signals_around_p0301\",\n", + " input_expressions=[eng_rpm, veh_spd],\n", + " channel_names=[\"Engine RPM\", \"Vehicle Speed\"],\n", + " statistics=[\"min\", \"mean\", \"max\"],\n", + " event=p0301_window_event,\n", + " desc=\"Signal stats within \u00b110 s of a P0301 misfire\",\n", + "))\n", + "\n", "print(f\"{len(page.aggregations)} aggregations added\")" ] }, @@ -1114,7 +1147,43 @@ " plt.show()\n", "else:\n", " print(\"No P0301 misfires in the demo data.\")\n", - "" + "\n", + "\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "# 6. POI WINDOW \u2014 Engine RPM min/mean/max within \u00b110 s of each P0301\n", + "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", + "win_df = (\n", + " spark.read.table(f\"{T}_stats_aggregator_fact\")\n", + " .join(\n", + " spark.read.table(f\"{T}_stats_aggregator_dimension\")\n", + " .filter(\"name = 'signals_around_p0301'\"),\n", + " on=\"visual_id\",\n", + " )\n", + " .select(\"container_id\", \"channel_name\", \"aggregation_label\", \"statistic_value\")\n", + " .toPandas()\n", + ")\n", + "if not win_df.empty:\n", + " rpm_win = (\n", + " win_df[win_df[\"channel_name\"] == \"Engine RPM\"]\n", + " .pivot_table(index=\"container_id\", columns=\"aggregation_label\",\n", + " values=\"statistic_value\")\n", + " .reset_index()\n", + " )\n", + " fig, ax = plt.subplots(figsize=(9, 4))\n", + " x = range(len(rpm_win))\n", + " ax.bar(x, rpm_win[\"max\"] - rpm_win[\"min\"], bottom=rpm_win[\"min\"],\n", + " color=\"lightsteelblue\", edgecolor=\"steelblue\",\n", + " label=\"min\u2013max range\")\n", + " ax.scatter(x, rpm_win[\"mean\"], color=\"crimson\", zorder=3, label=\"mean\")\n", + " ax.set_xticks(list(x))\n", + " ax.set_xticklabels([f\"Container {c}\" for c in rpm_win[\"container_id\"]])\n", + " ax.set_ylabel(\"Engine RPM\")\n", + " ax.set_title(\"Engine RPM within \u00b110 s of a P0301 misfire (min\u2013max range + mean)\")\n", + " ax.legend()\n", + " plt.tight_layout()\n", + " plt.show()\n", + "else:\n", + " print(\"No P0301 misfire windows in the demo data.\")" ] }, { From cddb8fed78fdc213ee021c3a7c613e00c66d3e95 Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 15:34:26 +0200 Subject: [PATCH 16/17] switched statsagg to histogramm to showcase poi extension --- demos/reporting_pipeline.ipynb | 62 +++++++++++++++------------------- 1 file changed, 28 insertions(+), 34 deletions(-) diff --git a/demos/reporting_pipeline.ipynb b/demos/reporting_pipeline.ipynb index fee38e6..327a4e9 100644 --- a/demos/reporting_pipeline.ipynb +++ b/demos/reporting_pipeline.ipynb @@ -811,16 +811,18 @@ "))\n", "\n", "\n", - "# Derived-value-around-POI: min/mean/max of the continuous signals in the\n", - "# \u00b110 s window around each misfire. `eng_rpm.where(p0301_window)` is a\n", - "# SampleSeries (values valid over the window), so StatsAggregator applies.\n", - "page.add_aggregation(StatsAggregator(\n", - " name=\"signals_around_p0301\",\n", - " input_expressions=[eng_rpm, veh_spd],\n", - " channel_names=[\"Engine RPM\", \"Vehicle Speed\"],\n", - " statistics=[\"min\", \"mean\", \"max\"],\n", + "# Derived-value-around-POI: the Engine RPM distribution within \u00b110 s of each\n", + "# misfire. The window event filters the (SampleSeries) channel to those\n", + "# intervals, so a duration-weighted histogram shows *what the engine was doing*\n", + "# around the fault \u2014 richer than a single min/mean/max.\n", + "page.add_aggregation(HistogramDuration(\n", + " name=\"rpm_around_p0301\",\n", + " base_expr=eng_rpm,\n", + " bins=[float(i) for i in range(0, 5000, 250)],\n", " event=p0301_window_event,\n", - " desc=\"Signal stats within \u00b110 s of a P0301 misfire\",\n", + " desc=\"Engine RPM distribution within \u00b110 s of a P0301 misfire\",\n", + " channel_name=\"Engine RPM\",\n", + " bins_unit=\"RPM\", values_unit=\"s\",\n", "))\n", "\n", "print(f\"{len(page.aggregations)} aggregations added\")" @@ -1148,38 +1150,30 @@ "else:\n", " print(\"No P0301 misfires in the demo data.\")\n", "\n", - "\n", "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", - "# 6. POI WINDOW \u2014 Engine RPM min/mean/max within \u00b110 s of each P0301\n", + "# 6. POI WINDOW \u2014 Engine RPM distribution within \u00b110 s of each P0301\n", "# \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\n", - "win_df = (\n", - " spark.read.table(f\"{T}_stats_aggregator_fact\")\n", + "win_hist = (\n", + " spark.read.table(f\"{T}_histogram_fact\")\n", " .join(\n", - " spark.read.table(f\"{T}_stats_aggregator_dimension\")\n", - " .filter(\"name = 'signals_around_p0301'\"),\n", + " spark.read.table(f\"{T}_histogram_dimension\")\n", + " .filter(\"name = 'rpm_around_p0301'\"),\n", " on=\"visual_id\",\n", " )\n", - " .select(\"container_id\", \"channel_name\", \"aggregation_label\", \"statistic_value\")\n", + " .groupBy(\"bin_id\", \"bin_name\")\n", + " .agg(F.sum(\"hist_value\").alias(\"total_us\"))\n", + " .orderBy(\"bin_id\")\n", " .toPandas()\n", ")\n", - "if not win_df.empty:\n", - " rpm_win = (\n", - " win_df[win_df[\"channel_name\"] == \"Engine RPM\"]\n", - " .pivot_table(index=\"container_id\", columns=\"aggregation_label\",\n", - " values=\"statistic_value\")\n", - " .reset_index()\n", - " )\n", - " fig, ax = plt.subplots(figsize=(9, 4))\n", - " x = range(len(rpm_win))\n", - " ax.bar(x, rpm_win[\"max\"] - rpm_win[\"min\"], bottom=rpm_win[\"min\"],\n", - " color=\"lightsteelblue\", edgecolor=\"steelblue\",\n", - " label=\"min\u2013max range\")\n", - " ax.scatter(x, rpm_win[\"mean\"], color=\"crimson\", zorder=3, label=\"mean\")\n", - " ax.set_xticks(list(x))\n", - " ax.set_xticklabels([f\"Container {c}\" for c in rpm_win[\"container_id\"]])\n", - " ax.set_ylabel(\"Engine RPM\")\n", - " ax.set_title(\"Engine RPM within \u00b110 s of a P0301 misfire (min\u2013max range + mean)\")\n", - " ax.legend()\n", + "if not win_hist.empty:\n", + " win_hist[\"duration_s\"] = win_hist[\"total_us\"] / 1e6\n", + " fig, ax = plt.subplots(figsize=(10, 4))\n", + " ax.bar(win_hist[\"bin_name\"], win_hist[\"duration_s\"],\n", + " color=\"indianred\", edgecolor=\"white\")\n", + " ax.set_xlabel(\"Engine RPM bin\")\n", + " ax.set_ylabel(\"Duration (s)\")\n", + " ax.set_title(\"Engine RPM distribution within \u00b110 s of a P0301 misfire\")\n", + " plt.xticks(rotation=45, ha=\"right\", fontsize=8)\n", " plt.tight_layout()\n", " plt.show()\n", "else:\n", From 328c266988608ca8fb2fb2c8cf1235549c25312a Mon Sep 17 00:00:00 2001 From: "fabian.ade" Date: Wed, 12 Aug 2026 15:56:26 +0200 Subject: [PATCH 17/17] corrected formatting --- .../integration/poi_channel_solve_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/impulse_query_engine/integration/poi_channel_solve_test.py b/tests/impulse_query_engine/integration/poi_channel_solve_test.py index 8b032ff..32e194d 100644 --- a/tests/impulse_query_engine/integration/poi_channel_solve_test.py +++ b/tests/impulse_query_engine/integration/poi_channel_solve_test.py @@ -168,9 +168,9 @@ def test_sample_channel_sampled_at_poi_instants_wide( amb = q.channel(channel_name="Ambient Air Temperature") dtc_count = q.poi_channel(channel_name="DTC_count") # points at 3 POI instants - result = q.select( - amb.where(dtc_count.to_points_in_time()).alias("sampled") - ).solve(spark=spark, solver=solver) + result = q.select(amb.where(dtc_count.to_points_in_time()).alias("sampled")).solve( + spark=spark, solver=solver + ) rows = {r.container_id: r.sampled for r in result.collect()} # The three POI instants (microsecond epochs) from basic_narrow_csv/poi_channels.csv.