From f682e4052930236a840c874f91582c56b67a26dc Mon Sep 17 00:00:00 2001 From: David Hensle <51132108+dhensle@users.noreply.github.com> Date: Fri, 8 May 2026 21:43:08 -0700 Subject: [PATCH 1/3] adding nullable_nonnegative decode filter --- activitysim/core/configuration/top.py | 3 ++ activitysim/core/steps/output.py | 56 ++++++++++++++++++++------- docs/dev-guide/using-sharrow.md | 6 ++- 3 files changed, 49 insertions(+), 16 deletions(-) diff --git a/activitysim/core/configuration/top.py b/activitysim/core/configuration/top.py index 825283126c..7376a7021b 100644 --- a/activitysim/core/configuration/top.py +++ b/activitysim/core/configuration/top.py @@ -114,6 +114,9 @@ class OutputTable(PydanticBase): should be decoded, as negative values indicate an absence of choice. In these cases, the "tablename.fieldname" can be prefixed with a "nonnegative" filter, seperated by a pipe character (e.g. "nonnegative | land_use.zone_id"). + If the column may also contain null values, use "nullable_nonnegative" + instead to pass nulls through unchanged while still preserving negative + sentinel values. """ diff --git a/activitysim/core/steps/output.py b/activitysim/core/steps/output.py index ea543c8aaf..94e5552cda 100644 --- a/activitysim/core/steps/output.py +++ b/activitysim/core/steps/output.py @@ -21,6 +21,40 @@ logger = logging.getLogger(__name__) +def _decode_output_column(column, map_func, preserve_nulls=False): + series = pd.Series(column) + if preserve_nulls: + revised_col = series.copy() + notna = series.notna() + revised_col.loc[notna] = series.loc[notna].astype(int).map(map_func) + return revised_col + return series.astype(int).map(map_func) + + +def _apply_decode_filter(map_col, decode_filter): + preserve_nulls = False + + if decode_filter is None: + return map_col.__getitem__, preserve_nulls + + if decode_filter == "nonnegative": + + def map_func(x): + return x if x < 0 else map_col[x] + + return map_func, preserve_nulls + + if decode_filter == "nullable_nonnegative": + preserve_nulls = True + + def map_func(x): + return x if x < 0 else map_col[x] + + return map_func, preserve_nulls + + raise ValueError(f"unknown decode_filter {decode_filter}") + + @workflow.step def track_skim_usage(state: workflow.State) -> None: """ @@ -515,9 +549,11 @@ def write_tables(state: workflow.State) -> None: if decode_instruction == "time_period": map_col = list(state.network_settings.skim_time_periods.labels) - map_func = map_col.__getitem__ - revised_col = ( - pd.Series(dt.column(colname)).astype(int).map(map_func) + map_func, preserve_nulls = _apply_decode_filter( + map_col, decode_filter + ) + revised_col = _decode_output_column( + dt.column(colname), map_func, preserve_nulls=preserve_nulls ) dt = dt.drop([colname]).append_column( colname, pa.array(revised_col) @@ -536,18 +572,10 @@ def write_tables(state: workflow.State) -> None: except KeyError: map_col = parent_table.column(lookup_col) map_col = np.asarray(map_col) - map_func = map_col.__getitem__ - if decode_filter: - if decode_filter == "nonnegative": - - def map_func(x): - return x if x < 0 else map_col[x] - - else: - raise ValueError(f"unknown decode_filter {decode_filter}") + map_func, preserve_nulls = _apply_decode_filter(map_col, decode_filter) if colname in dt.column_names: - revised_col = ( - pd.Series(dt.column(colname)).astype(int).map(map_func) + revised_col = _decode_output_column( + dt.column(colname), map_func, preserve_nulls=preserve_nulls ) dt = dt.drop([colname]).append_column( colname, pa.array(revised_col) diff --git a/docs/dev-guide/using-sharrow.md b/docs/dev-guide/using-sharrow.md index d3924ed2ac..59e270d53d 100644 --- a/docs/dev-guide/using-sharrow.md +++ b/docs/dev-guide/using-sharrow.md @@ -111,7 +111,9 @@ decoded in various output files. Generally, for columns that are fully populate with zone id's (e.g. tour and trip ends) we can apply the `decode_columns` settings to reverse the mapping and restore the nominal zone id's globally for the entire column of data. For columns where there is some missing data flagged by negative -values, the "nonnegative" filter is prepended to the instruction. +values, the "nonnegative" filter is prepended to the instruction. If the column +may also contain nulls, use "nullable_nonnegative" to leave those values +unchanged while decoding the non-negative indexes. ```yaml output_tables: @@ -127,7 +129,7 @@ output_tables: decode_columns: home_zone_id: land_use.zone_id school_zone_id: nonnegative | land_use.zone_id - workplace_zone_id: nonnegative | land_use.zone_id + workplace_zone_id: nullable_nonnegative | land_use.zone_id - tablename: tours decode_columns: origin: land_use.zone_id From 325083706aa488966b7d990090d9b2ba6adf0c54 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Jul 2026 00:17:55 +0000 Subject: [PATCH 2/3] Initial plan From 0e66375df15d768814b006b70655366c90c10af0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 13 Jul 2026 00:23:24 +0000 Subject: [PATCH 3/3] Add unit tests for _decode_output_column and _apply_decode_filter (nullable_nonnegative) --- activitysim/core/steps/_decode.py | 40 ++++++++++++++ activitysim/core/steps/output.py | 35 +----------- activitysim/core/test/test_output.py | 83 ++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 34 deletions(-) create mode 100644 activitysim/core/steps/_decode.py create mode 100644 activitysim/core/test/test_output.py diff --git a/activitysim/core/steps/_decode.py b/activitysim/core/steps/_decode.py new file mode 100644 index 0000000000..73b8700163 --- /dev/null +++ b/activitysim/core/steps/_decode.py @@ -0,0 +1,40 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +import numpy as np +import pandas as pd + + +def _decode_output_column(column, map_func, preserve_nulls=False): + series = pd.Series(column) + if preserve_nulls: + revised_col = series.copy() + notna = series.notna() + revised_col.loc[notna] = series.loc[notna].astype(int).map(map_func) + return revised_col + return series.astype(int).map(map_func) + + +def _apply_decode_filter(map_col, decode_filter): + preserve_nulls = False + + if decode_filter is None: + return map_col.__getitem__, preserve_nulls + + if decode_filter == "nonnegative": + + def map_func(x): + return x if x < 0 else map_col[x] + + return map_func, preserve_nulls + + if decode_filter == "nullable_nonnegative": + preserve_nulls = True + + def map_func(x): + return x if x < 0 else map_col[x] + + return map_func, preserve_nulls + + raise ValueError(f"unknown decode_filter {decode_filter}") diff --git a/activitysim/core/steps/output.py b/activitysim/core/steps/output.py index 94e5552cda..950097379c 100644 --- a/activitysim/core/steps/output.py +++ b/activitysim/core/steps/output.py @@ -17,44 +17,11 @@ from activitysim.core import configuration, workflow from activitysim.core.workflow.checkpoint import CHECKPOINT_NAME from activitysim.core.estimation import estimation_enabled, EstimationConfig +from activitysim.core.steps._decode import _apply_decode_filter, _decode_output_column logger = logging.getLogger(__name__) -def _decode_output_column(column, map_func, preserve_nulls=False): - series = pd.Series(column) - if preserve_nulls: - revised_col = series.copy() - notna = series.notna() - revised_col.loc[notna] = series.loc[notna].astype(int).map(map_func) - return revised_col - return series.astype(int).map(map_func) - - -def _apply_decode_filter(map_col, decode_filter): - preserve_nulls = False - - if decode_filter is None: - return map_col.__getitem__, preserve_nulls - - if decode_filter == "nonnegative": - - def map_func(x): - return x if x < 0 else map_col[x] - - return map_func, preserve_nulls - - if decode_filter == "nullable_nonnegative": - preserve_nulls = True - - def map_func(x): - return x if x < 0 else map_col[x] - - return map_func, preserve_nulls - - raise ValueError(f"unknown decode_filter {decode_filter}") - - @workflow.step def track_skim_usage(state: workflow.State) -> None: """ diff --git a/activitysim/core/test/test_output.py b/activitysim/core/test/test_output.py new file mode 100644 index 0000000000..47f89451a9 --- /dev/null +++ b/activitysim/core/test/test_output.py @@ -0,0 +1,83 @@ +# ActivitySim +# See full license in LICENSE.txt. +from __future__ import annotations + +import numpy as np +import pandas as pd +import pandas.testing as pdt +import pytest + +from activitysim.core.steps._decode import _apply_decode_filter, _decode_output_column + + +@pytest.fixture +def zone_labels(): + """Sample zone ID lookup array (index 0..4 -> zone labels).""" + return np.array([100, 200, 300, 400, 500]) + + +def test_decode_output_column_no_nulls(zone_labels): + """Basic decode without nulls: each integer index maps to the zone label.""" + map_func = zone_labels.__getitem__ + column = pd.array([0, 1, 2, 3, 4]) + result = _decode_output_column(column, map_func) + expected = pd.Series([100, 200, 300, 400, 500]) + pdt.assert_series_equal(result, expected) + + +def test_decode_output_column_preserve_nulls(zone_labels): + """With preserve_nulls=True, NaN entries pass through unchanged.""" + map_func = zone_labels.__getitem__ + column = pd.array([0, pd.NA, 2, pd.NA, 4], dtype="Int64") + result = _decode_output_column(column, map_func, preserve_nulls=True) + # Non-null positions are decoded; null positions remain null. + assert result[0] == 100 + assert result[2] == 300 + assert result[4] == 500 + assert pd.isna(result[1]) + assert pd.isna(result[3]) + + +def test_apply_decode_filter_none(zone_labels): + """No filter: map_func is direct indexing and preserve_nulls is False.""" + map_func, preserve_nulls = _apply_decode_filter(zone_labels, None) + assert not preserve_nulls + assert map_func(2) == zone_labels[2] + + +def test_apply_decode_filter_nonnegative(zone_labels): + """nonnegative filter: negative values pass through; others are decoded.""" + map_func, preserve_nulls = _apply_decode_filter(zone_labels, "nonnegative") + assert not preserve_nulls + assert map_func(1) == 200 + assert map_func(-1) == -1 + + +def test_apply_decode_filter_nullable_nonnegative(zone_labels): + """nullable_nonnegative filter: negative values pass through; preserve_nulls is True.""" + map_func, preserve_nulls = _apply_decode_filter(zone_labels, "nullable_nonnegative") + assert preserve_nulls + assert map_func(1) == 200 + assert map_func(-1) == -1 + + +def test_apply_decode_filter_unknown(): + """Unknown filter name raises ValueError.""" + with pytest.raises(ValueError, match="unknown decode_filter"): + _apply_decode_filter([], "bad_filter") + + +def test_nullable_nonnegative_with_nulls_and_sentinels(zone_labels): + """ + End-to-end: a column with nulls and -1 sentinel values decoded via + nullable_nonnegative should pass nulls and negative values through unchanged + while mapping non-negative indexes to zone labels. + """ + map_func, preserve_nulls = _apply_decode_filter(zone_labels, "nullable_nonnegative") + column = pd.array([0, pd.NA, -1, 3, pd.NA], dtype="Int64") + result = _decode_output_column(column, map_func, preserve_nulls=preserve_nulls) + assert result[0] == 100 + assert pd.isna(result[1]) + assert result[2] == -1 + assert result[3] == 400 + assert pd.isna(result[4])