From 3b96439e6617fc974f6411dae78a4c22518587b1 Mon Sep 17 00:00:00 2001 From: stevenhsd <56357022+stevenhsd@users.noreply.github.com> Date: Tue, 11 Aug 2026 11:37:21 +0100 Subject: [PATCH 1/3] fix: ensure that format for times can be accessed and supplied in duckdb casting --- .../implementations/duckdb/duckdb_helpers.py | 42 ++++++++-------- .../implementations/spark/spark_helpers.py | 48 ++++++++++--------- src/dve/metadata_parser/utilities.py | 20 ++++++++ tests/test_parser/test_utils.py | 24 ++++++++++ 4 files changed, 92 insertions(+), 42 deletions(-) create mode 100644 tests/test_parser/test_utils.py diff --git a/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py b/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py index b54192f..bbbd3a4 100644 --- a/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py +++ b/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py @@ -16,6 +16,7 @@ from duckdb import DuckDBPyConnection, DuckDBPyRelation, StarExpression from duckdb.typing import DuckDBPyType from pandas import DataFrame +from dve.metadata_parser.utilities import resilient_get from pydantic import BaseModel from typing_extensions import Annotated, get_args, get_origin, get_type_hints @@ -451,23 +452,26 @@ def get_duckdb_cast_statement_from_annotation( raise ValueError(f"dict must be `typing.TypedDict` subclass, got {type_annotation!r}") for type_ in type_annotation.mro(): - _date_format: str = getattr( # type: ignore - type_, "DATE_FORMAT", DEFAULT_ISO_FORMATS.get(type_, DEFAULT_ISO_FORMATS.get(datetime)) - ) - dt_cast_statement = rf"CASE WHEN REGEXP_FULL_MATCH(TRIM({quoted_name}), '{datetime_format_to_regex(_date_format)}') THEN TRY_STRPTIME(TRIM({quoted_name}), '{_date_format}') ELSE NULL END" # pylint: disable=C0301 - - # datetime is subclass of date, so needs to be handled first - if issubclass(type_, datetime): - stmt = rf"TRY_CAST({dt_cast_statement} as TIMESTAMP)" - return stmt - if issubclass(type_, date): - stmt = rf"TRY_CAST({dt_cast_statement} as DATE)" - return stmt - if issubclass(type_, time): - stmt = rf"TRY_CAST({dt_cast_statement} as TIME)" - return stmt - duck_type = get_duckdb_type_from_annotation(type_) - if duck_type: - stmt = f"TRIM({quoted_name})" - return _cast_as_ddb_type(stmt, type_) if parent_element else stmt + if issubclass(type_, (date, time)): + _date_format: str = ( + resilient_get(type_, "DATE_FORMAT", "TIME_FORMAT") + or DEFAULT_ISO_FORMATS.get(type_, DEFAULT_ISO_FORMATS.get(datetime)) # type: ignore + ) + dt_cast_statement = rf"CASE WHEN REGEXP_FULL_MATCH(TRIM({quoted_name}), '{datetime_format_to_regex(_date_format)}') THEN TRY_STRPTIME(TRIM({quoted_name}), '{_date_format}') ELSE NULL END" # pylint: disable=C0301 + + # datetime is subclass of date, so needs to be handled first + if issubclass(type_, datetime): + stmt = rf"TRY_CAST({dt_cast_statement} as TIMESTAMP)" + return stmt + if issubclass(type_, date): + stmt = rf"TRY_CAST({dt_cast_statement} as DATE)" + return stmt + if issubclass(type_, time): + stmt = rf"TRY_CAST({dt_cast_statement} as TIME)" + return stmt + else: + duck_type = get_duckdb_type_from_annotation(type_) + if duck_type: + stmt = f"TRIM({quoted_name})" + return _cast_as_ddb_type(stmt, type_) if parent_element else stmt raise ValueError(f"No equivalent DuckDB type for {type_annotation!r}") diff --git a/src/dve/core_engine/backends/implementations/spark/spark_helpers.py b/src/dve/core_engine/backends/implementations/spark/spark_helpers.py index ae714a6..375135c 100644 --- a/src/dve/core_engine/backends/implementations/spark/spark_helpers.py +++ b/src/dve/core_engine/backends/implementations/spark/spark_helpers.py @@ -593,29 +593,31 @@ def get_spark_cast_statement_from_annotation( raise ValueError(f"dict must be `typing.TypedDict` subclass, got {type_annotation!r}") for type_ in type_annotation.mro(): - _date_format: str = getattr( # type: ignore - type_, - "DATE_FORMAT", - DEFAULT_ISO_FORMATS.get(type_, DEFAULT_ISO_FORMATS.get(dt.datetime)), - ) - - # pylint: disable=C0301 - dt_cast_statement = f"CASE WHEN REGEXP(TRIM({quoted_name}), '{datetime_format_to_regex(_date_format)}') THEN TRY_TO_TIMESTAMP(TRIM({quoted_name}), \"{python_to_java_datetime_format(_date_format)}\") ELSE NULL END" # pylint: disable=C0301 - # datetime is subclass of date, so needs to be handled first - if issubclass(type_, dt.datetime): - return ( - _cast_as_spark_type(dt_cast_statement, type_) - if parent_element - else dt_cast_statement - ) if issubclass(type_, dt.date): - return ( - _cast_as_spark_type(dt_cast_statement, type_) - if parent_element - else dt_cast_statement + _date_format: str = getattr( # type: ignore + type_, + "DATE_FORMAT", + DEFAULT_ISO_FORMATS.get(type_, DEFAULT_ISO_FORMATS.get(dt.datetime)), ) - spark_type = get_type_from_annotation(type_) - if spark_type: - stmt = f"TRIM({quoted_name})" - return _cast_as_spark_type(stmt, type_) if parent_element else stmt + + # pylint: disable=C0301 + dt_cast_statement = f"CASE WHEN REGEXP(TRIM({quoted_name}), '{datetime_format_to_regex(_date_format)}') THEN TRY_TO_TIMESTAMP(TRIM({quoted_name}), \"{python_to_java_datetime_format(_date_format)}\") ELSE NULL END" # pylint: disable=C0301 + # datetime is subclass of date, so needs to be handled first + if issubclass(type_, dt.datetime): + return ( + _cast_as_spark_type(dt_cast_statement, type_) + if parent_element + else dt_cast_statement + ) + if issubclass(type_, dt.date): + return ( + _cast_as_spark_type(dt_cast_statement, type_) + if parent_element + else dt_cast_statement + ) + else: + spark_type = get_type_from_annotation(type_) + if spark_type: + stmt = f"TRIM({quoted_name})" + return _cast_as_spark_type(stmt, type_) if parent_element else stmt raise ValueError(f"No equivalent Spark type for {type_annotation!r}") diff --git a/src/dve/metadata_parser/utilities.py b/src/dve/metadata_parser/utilities.py index 0efa078..eb8fe62 100644 --- a/src/dve/metadata_parser/utilities.py +++ b/src/dve/metadata_parser/utilities.py @@ -52,3 +52,23 @@ def chain_get( return result raise exc.TypeNotFoundError(f"Callable or type ({item!r}) not found") + +def resilient_get( + item: object, *attribute_names: str + ) -> Any: + """Given an iterable of attribute names, try to get attribute value + sequentially. Returns the first value found, otherwise returns None. + + Args: + item (object): The object to obtain attributes from (where possible) + attribute_names (tuple[str]): The attribute names to search for + + Returns: + Any: The first found attribute, otherwise None + """ + for attr in attribute_names: + try: + return getattr(item, attr) + except AttributeError: + continue + return None diff --git a/tests/test_parser/test_utils.py b/tests/test_parser/test_utils.py new file mode 100644 index 0000000..bf0c700 --- /dev/null +++ b/tests/test_parser/test_utils.py @@ -0,0 +1,24 @@ +import pytest +from dve.metadata_parser.utilities import resilient_get + +class MyParent: + cls_attr = "hello" + def __init__(self, my_attr:str, another_attr:str): + self.my_attr = my_attr + self.another_attr = another_attr + +class MyObject(MyParent): + sub_attr = "bye" + def __init__(self, extra_attr:int): + self.extra_attr = extra_attr + super().__init__("from", "child") + + +@pytest.mark.parametrize("obj,attrs,expected", [(MyParent, ("cls_attr",), "hello"), + (MyObject, ("cls_attr", "sub_attr"), "hello"), + (MyObject, ("sub_attr", "cls_attr"), "bye"), + (MyParent, ("my_attr",), None), + (MyParent("this", "test"), ("extra_attr", "my_attr"), "this"), + (MyObject("this"), ("daft_attr", "another_daft_attr", "yet_another", "another_attr"), "child")]) +def test_resilient_get(obj, attrs, expected): + assert resilient_get(obj, *attrs) == expected \ No newline at end of file From 9e51d1a76f78a4294a87ba72c0fe9e7289590c5c Mon Sep 17 00:00:00 2001 From: stevenhsd <56357022+stevenhsd@users.noreply.github.com> Date: Tue, 11 Aug 2026 12:00:23 +0100 Subject: [PATCH 2/3] style: resolve formatting and linting issues --- .../backends/implementations/duckdb/duckdb_helpers.py | 11 ++++++----- src/dve/metadata_parser/utilities.py | 5 ++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py b/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py index bbbd3a4..48b7d9e 100644 --- a/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py +++ b/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py @@ -16,7 +16,6 @@ from duckdb import DuckDBPyConnection, DuckDBPyRelation, StarExpression from duckdb.typing import DuckDBPyType from pandas import DataFrame -from dve.metadata_parser.utilities import resilient_get from pydantic import BaseModel from typing_extensions import Annotated, get_args, get_origin, get_type_hints @@ -25,6 +24,7 @@ from dve.core_engine.backends.utilities import DEFAULT_ISO_FORMATS, datetime_format_to_regex from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME from dve.core_engine.type_hints import URI, EntityName +from dve.metadata_parser.utilities import resilient_get from dve.parser.file_handling.service import LocalFilesystemImplementation, _get_implementation @@ -453,10 +453,11 @@ def get_duckdb_cast_statement_from_annotation( for type_ in type_annotation.mro(): if issubclass(type_, (date, time)): - _date_format: str = ( - resilient_get(type_, "DATE_FORMAT", "TIME_FORMAT") - or DEFAULT_ISO_FORMATS.get(type_, DEFAULT_ISO_FORMATS.get(datetime)) # type: ignore - ) + _date_format: str = resilient_get( + type_, "DATE_FORMAT", "TIME_FORMAT" + ) or DEFAULT_ISO_FORMATS.get( + type_, DEFAULT_ISO_FORMATS.get(datetime) + ) # type: ignore dt_cast_statement = rf"CASE WHEN REGEXP_FULL_MATCH(TRIM({quoted_name}), '{datetime_format_to_regex(_date_format)}') THEN TRY_STRPTIME(TRIM({quoted_name}), '{_date_format}') ELSE NULL END" # pylint: disable=C0301 # datetime is subclass of date, so needs to be handled first diff --git a/src/dve/metadata_parser/utilities.py b/src/dve/metadata_parser/utilities.py index eb8fe62..54e6439 100644 --- a/src/dve/metadata_parser/utilities.py +++ b/src/dve/metadata_parser/utilities.py @@ -53,9 +53,8 @@ def chain_get( raise exc.TypeNotFoundError(f"Callable or type ({item!r}) not found") -def resilient_get( - item: object, *attribute_names: str - ) -> Any: + +def resilient_get(item: object, *attribute_names: str) -> Any: """Given an iterable of attribute names, try to get attribute value sequentially. Returns the first value found, otherwise returns None. From 934f9de1cb747cb844a916dd97d3efe2308f0c8b Mon Sep 17 00:00:00 2001 From: stevenhsd <56357022+stevenhsd@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:39:20 +0100 Subject: [PATCH 3/3] docs: tweak docstring for resilient_get --- src/dve/metadata_parser/utilities.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/dve/metadata_parser/utilities.py b/src/dve/metadata_parser/utilities.py index 54e6439..b20cee1 100644 --- a/src/dve/metadata_parser/utilities.py +++ b/src/dve/metadata_parser/utilities.py @@ -55,12 +55,13 @@ def chain_get( def resilient_get(item: object, *attribute_names: str) -> Any: - """Given an iterable of attribute names, try to get attribute value - sequentially. Returns the first value found, otherwise returns None. + """Given a number of attribute names, try to get attribute value + sequentially. Returns the first value found, and if no attributes found + returns None. Args: item (object): The object to obtain attributes from (where possible) - attribute_names (tuple[str]): The attribute names to search for + attribute_names (str): The attribute names to search for Returns: Any: The first found attribute, otherwise None