diff --git a/cognite/client/_api/transformations/__init__.py b/cognite/client/_api/transformations/__init__.py index 5e565beb6b..9da907e274 100644 --- a/cognite/client/_api/transformations/__init__.py +++ b/cognite/client/_api/transformations/__init__.py @@ -3,6 +3,7 @@ from collections.abc import AsyncIterator, Sequence from typing import TYPE_CHECKING, Any, Literal, overload +from cognite.client._api.transformations.external_data import TransformationExternalDataAPI from cognite.client._api.transformations.jobs import TransformationJobsAPI from cognite.client._api.transformations.notifications import TransformationNotificationsAPI from cognite.client._api.transformations.schedules import TransformationSchedulesAPI @@ -36,6 +37,7 @@ def __init__(self, config: ClientConfig, api_version: str | None, cognite_client self.schedules = TransformationSchedulesAPI(config, api_version, cognite_client) self.schema = TransformationSchemaAPI(config, api_version, cognite_client) self.notifications = TransformationNotificationsAPI(config, api_version, cognite_client) + self.external_data_sources = TransformationExternalDataAPI(config, api_version, cognite_client) @overload def __call__( diff --git a/cognite/client/_api/transformations/external_data.py b/cognite/client/_api/transformations/external_data.py new file mode 100644 index 0000000000..bb4f482cbc --- /dev/null +++ b/cognite/client/_api/transformations/external_data.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +from collections.abc import Sequence + +from cognite.client._api_client import APIClient +from cognite.client.data_classes.transformations.external_data import ( + ExternalDataSource, + ExternalDataSourceList, + ExternalDataSourceUsability, + ExternalDataSourceWrite, +) +from cognite.client.utils._identifier import IdentifierSequence +from cognite.client.utils.useful_types import SequenceNotStr + + +class TransformationExternalDataAPI(APIClient): + """Manage Fabric OneLake external data sources for transformations. + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + """ + + _RESOURCE_PATH = "/transformations/external_data" + _CREATE_LIMIT = 1000 + _DELETE_LIMIT = 1000 + + async def list(self, limit: int | None = None) -> ExternalDataSourceList: + """List Fabric OneLake external data sources. + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + Args: + limit (int | None): Maximum number of results to return. Use ``None`` or ``-1`` to return all. + Defaults to returning all. + + Returns: + ExternalDataSourceList: All registered OneLake external data sources. + + Examples: + + List all registered external data sources: + + >>> from cognite.client import CogniteClient + >>> client = CogniteClient() + >>> sources = client.transformations.external_data_sources.list() + """ + return await self._list( + method="GET", + list_cls=ExternalDataSourceList, + resource_cls=ExternalDataSource, + limit=limit, + ) + + async def upsert( + self, + source: ExternalDataSourceWrite | Sequence[ExternalDataSourceWrite], + ) -> ExternalDataSource | ExternalDataSourceList: + """Create or update (upsert) Fabric OneLake external data sources. + + An upsert creates the source if it doesn't exist, or overwrites it entirely if it does. + Uniqueness is determined by ``externalId``. + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + Args: + source (ExternalDataSourceWrite | Sequence[ExternalDataSourceWrite]): Single source or list of sources to upsert. + + Returns: + ExternalDataSource | ExternalDataSourceList: The upserted source(s). + + Examples: + + Register a Fabric OneLake source: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.transformations.external_data import ( + ... ExternalDataSourceWrite, + ... ) + >>> client = CogniteClient() + >>> source = ExternalDataSourceWrite.onelake( + ... external_id="fabric-lakehouse-prod", + ... name="Production lakehouse", + ... client_id="", + ... tenant_id="", + ... client_secret="", + ... workspace_name="", + ... container_name="", + ... data_set_id=123456, + ... ) + >>> res = client.transformations.external_data_sources.upsert(source) + """ + return await self._create_multiple( + items=source, + list_cls=ExternalDataSourceList, + resource_cls=ExternalDataSource, + input_resource_cls=ExternalDataSourceWrite, + ) + + async def delete(self, external_id: str | SequenceNotStr[str]) -> None: + """Delete Fabric OneLake external data sources. + + Args: + external_id (str | SequenceNotStr[str]): External ID or list of external IDs to delete. + + Examples: + + Delete a source by external ID: + + >>> from cognite.client import CogniteClient + >>> client = CogniteClient() + >>> client.transformations.external_data_sources.delete("fabric-lakehouse-prod") + + Delete multiple sources: + + >>> client.transformations.external_data_sources.delete( + ... ["fabric-lakehouse-prod", "fabric-lakehouse-staging"] + ... ) + """ + await self._delete_multiple( + identifiers=IdentifierSequence.load(external_ids=external_id), + wrap_ids=True, + ) + + async def verify_usability(self, external_id: str) -> ExternalDataSourceUsability: + """Verify that a Fabric OneLake external data source is usable. + + Checks that the source exists and that the configured Azure credentials can access the specified + Fabric lakehouse. Returns a ``usable_version`` UUID if the source is accessible. + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + Args: + external_id (str): External ID of the source to verify. + + Returns: + ExternalDataSourceUsability: Contains ``usable_version`` (a UUID) if the source is accessible, + or ``None`` if the credentials are invalid or the source cannot be reached. + + Examples: + + Verify a source before running a transformation: + + >>> from cognite.client import CogniteClient + >>> client = CogniteClient() + >>> result = client.transformations.external_data_sources.verify_usability( + ... "fabric-lakehouse-prod" + ... ) + >>> assert result.usable_version is not None, ( + ... "Source not configured or credentials invalid" + ... ) + """ + res = await self._post( + url_path=self._RESOURCE_PATH + "/usability", + json={"externalId": external_id}, + semaphore=self._get_semaphore("write"), + ) + return ExternalDataSourceUsability._load(res.json()) diff --git a/cognite/client/_cognite_client.py b/cognite/client/_cognite_client.py index 8720fa9507..11f1b1fe05 100644 --- a/cognite/client/_cognite_client.py +++ b/cognite/client/_cognite_client.py @@ -97,6 +97,7 @@ ThreeDRevisionsAPI, ) from cognite.client._api.transformations import ( # type: ignore[attr-defined] + TransformationExternalDataAPI, TransformationJobsAPI, TransformationNotificationsAPI, TransformationSchedulesAPI, @@ -425,6 +426,7 @@ def _make_accessors_for_building_docs() -> None: AsyncCogniteClient.transformations.notifications = TransformationNotificationsAPI # type: ignore AsyncCogniteClient.transformations.jobs = TransformationJobsAPI # type: ignore AsyncCogniteClient.transformations.schema = TransformationSchemaAPI # type: ignore + AsyncCogniteClient.transformations.external_data_sources = TransformationExternalDataAPI # type: ignore AsyncCogniteClient.diagrams = DiagramsAPI # type: ignore AsyncCogniteClient.annotations = AnnotationsAPI # type: ignore AsyncCogniteClient.functions = FunctionsAPI # type: ignore diff --git a/cognite/client/_sync_api/transformations/__init__.py b/cognite/client/_sync_api/transformations/__init__.py index 9eb4f69eb2..0ab16f22a9 100644 --- a/cognite/client/_sync_api/transformations/__init__.py +++ b/cognite/client/_sync_api/transformations/__init__.py @@ -1,6 +1,6 @@ """ =============================================================================== -8bca411648596701cde79f2c9e6f2a88 +e545d766fa2f6a8b3d5946accd494c89 This file is auto-generated from the Async API modules, - do not edit manually! =============================================================================== """ @@ -12,6 +12,7 @@ from cognite.client import AsyncCogniteClient from cognite.client._constants import DEFAULT_LIMIT_READ +from cognite.client._sync_api.transformations.external_data import SyncTransformationExternalDataAPI from cognite.client._sync_api.transformations.jobs import SyncTransformationJobsAPI from cognite.client._sync_api.transformations.notifications import SyncTransformationNotificationsAPI from cognite.client._sync_api.transformations.schedules import SyncTransformationSchedulesAPI @@ -41,6 +42,7 @@ def __init__(self, async_client: AsyncCogniteClient) -> None: self.schedules = SyncTransformationSchedulesAPI(async_client) self.schema = SyncTransformationSchemaAPI(async_client) self.notifications = SyncTransformationNotificationsAPI(async_client) + self.external_data_sources = SyncTransformationExternalDataAPI(async_client) @overload def __call__( diff --git a/cognite/client/_sync_api/transformations/external_data.py b/cognite/client/_sync_api/transformations/external_data.py new file mode 100644 index 0000000000..5ec963f7d6 --- /dev/null +++ b/cognite/client/_sync_api/transformations/external_data.py @@ -0,0 +1,150 @@ +""" +=============================================================================== +53ec7cc4c0a0655facc509b38fedaff9 +This file is auto-generated from the Async API modules, - do not edit manually! +=============================================================================== +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from cognite.client import AsyncCogniteClient +from cognite.client._sync_api_client import SyncAPIClient +from cognite.client.data_classes.transformations.external_data import ( + ExternalDataSource, + ExternalDataSourceList, + ExternalDataSourceUsability, + ExternalDataSourceWrite, +) +from cognite.client.utils._async_helpers import run_sync +from cognite.client.utils.useful_types import SequenceNotStr + + +class SyncTransformationExternalDataAPI(SyncAPIClient): + """Auto-generated, do not modify manually.""" + + def __init__(self, async_client: AsyncCogniteClient) -> None: + self.__async_client = async_client + + def list(self, limit: int | None = None) -> ExternalDataSourceList: + """ + List Fabric OneLake external data sources. + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + Args: + limit (int | None): Maximum number of results to return. Use ``None`` or ``-1`` to return all. + Defaults to returning all. + + Returns: + ExternalDataSourceList: All registered OneLake external data sources. + + Examples: + + List all registered external data sources: + + >>> from cognite.client import CogniteClient + >>> client = CogniteClient() + >>> sources = client.transformations.external_data_sources.list() + """ + return run_sync(self.__async_client.transformations.external_data_sources.list(limit=limit)) + + def upsert( + self, source: ExternalDataSourceWrite | Sequence[ExternalDataSourceWrite] + ) -> ExternalDataSource | ExternalDataSourceList: + """ + Create or update (upsert) Fabric OneLake external data sources. + + An upsert creates the source if it doesn't exist, or overwrites it entirely if it does. + Uniqueness is determined by ``externalId``. + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + Args: + source (ExternalDataSourceWrite | Sequence[ExternalDataSourceWrite]): Single source or list of sources to upsert. + + Returns: + ExternalDataSource | ExternalDataSourceList: The upserted source(s). + + Examples: + + Register a Fabric OneLake source: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.transformations.external_data import ( + ... ExternalDataSourceWrite, + ... ) + >>> client = CogniteClient() + >>> source = ExternalDataSourceWrite.onelake( + ... external_id="fabric-lakehouse-prod", + ... name="Production lakehouse", + ... client_id="", + ... tenant_id="", + ... client_secret="", + ... workspace_name="", + ... container_name="", + ... data_set_id=123456, + ... ) + >>> res = client.transformations.external_data_sources.upsert(source) + """ + return run_sync(self.__async_client.transformations.external_data_sources.upsert(source=source)) + + def delete(self, external_id: str | SequenceNotStr[str]) -> None: + """ + Delete Fabric OneLake external data sources. + + Args: + external_id (str | SequenceNotStr[str]): External ID or list of external IDs to delete. + + Examples: + + Delete a source by external ID: + + >>> from cognite.client import CogniteClient + >>> client = CogniteClient() + >>> client.transformations.external_data_sources.delete("fabric-lakehouse-prod") + + Delete multiple sources: + + >>> client.transformations.external_data_sources.delete( + ... ["fabric-lakehouse-prod", "fabric-lakehouse-staging"] + ... ) + """ + return run_sync(self.__async_client.transformations.external_data_sources.delete(external_id=external_id)) + + def verify_usability(self, external_id: str) -> ExternalDataSourceUsability: + """ + Verify that a Fabric OneLake external data source is usable. + + Checks that the source exists and that the configured Azure credentials can access the specified + Fabric lakehouse. Returns a ``usable_version`` UUID if the source is accessible. + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + Args: + external_id (str): External ID of the source to verify. + + Returns: + ExternalDataSourceUsability: Contains ``usable_version`` (a UUID) if the source is accessible, + or ``None`` if the credentials are invalid or the source cannot be reached. + + Examples: + + Verify a source before running a transformation: + + >>> from cognite.client import CogniteClient + >>> client = CogniteClient() + >>> result = client.transformations.external_data_sources.verify_usability( + ... "fabric-lakehouse-prod" + ... ) + >>> assert result.usable_version is not None, ( + ... "Source not configured or credentials invalid" + ... ) + """ + return run_sync( + self.__async_client.transformations.external_data_sources.verify_usability(external_id=external_id) + ) diff --git a/cognite/client/data_classes/__init__.py b/cognite/client/data_classes/__init__.py index 8247c0d60d..8a722ff9f0 100644 --- a/cognite/client/data_classes/__init__.py +++ b/cognite/client/data_classes/__init__.py @@ -251,6 +251,18 @@ TransformationBlockedInfo, TransformationDestination, ) +from cognite.client.data_classes.transformations.external_data import ( + ExternalDataSource, + ExternalDataSourceList, + ExternalDataSourceUsability, + ExternalDataSourceWrite, + ExternalDataSourceWriteList, + OneLakeCredentialsRead, + OneLakeCredentialsWrite, + OneLakeDataSourceSettingsRead, + OneLakeDataSourceSettingsWrite, + OneLakeLocationDescription, +) from cognite.client.data_classes.transformations.jobs import ( TransformationJob, TransformationJobFilter, @@ -384,6 +396,11 @@ "EventUpdate", "EventWrite", "EventWriteList", + "ExternalDataSource", + "ExternalDataSourceList", + "ExternalDataSourceUsability", + "ExternalDataSourceWrite", + "ExternalDataSourceWriteList", "ExtractionPipeline", "ExtractionPipelineConfig", "ExtractionPipelineConfigRevision", @@ -456,6 +473,11 @@ "Limit", "LimitList", "OidcCredentials", + "OneLakeCredentialsRead", + "OneLakeCredentialsWrite", + "OneLakeDataSourceSettingsRead", + "OneLakeDataSourceSettingsWrite", + "OneLakeLocationDescription", "RawTable", "RecordId", "Relationship", diff --git a/cognite/client/data_classes/capabilities.py b/cognite/client/data_classes/capabilities.py index 35536f3a19..529a89a1aa 100644 --- a/cognite/client/data_classes/capabilities.py +++ b/cognite/client/data_classes/capabilities.py @@ -1104,6 +1104,22 @@ class Scope: DataSet = DataSetScope +@dataclass +class TransformationsExternalDataSourcesAcl(Capability): + _capability_name = "transformationsExternalDataSourcesAcl" + actions: Sequence[Action] + scope: AllScope | DataSetScope + + class Action(Capability.Action): # type: ignore [misc] + Read = "READ" + Write = "WRITE" + Use = "USE" + + class Scope: + All = AllScope + DataSet = DataSetScope + + @dataclass class TypesAcl(Capability): _capability_name = "typesAcl" diff --git a/cognite/client/data_classes/transformations/external_data.py b/cognite/client/data_classes/transformations/external_data.py new file mode 100644 index 0000000000..c70997ffd5 --- /dev/null +++ b/cognite/client/data_classes/transformations/external_data.py @@ -0,0 +1,509 @@ +from __future__ import annotations + +import warnings +from abc import ABC +from typing import Any, ClassVar + +from typing_extensions import Self + +from cognite.client.data_classes._base import ( + CogniteResource, + CogniteResourceList, + ExternalIDTransformerMixin, + WriteableCogniteResource, + WriteableCogniteResourceList, +) + +__all__ = [ + "ExternalDataSource", + "ExternalDataSourceCore", + "ExternalDataSourceList", + "ExternalDataSourceUsability", + "ExternalDataSourceWrite", + "ExternalDataSourceWriteList", + "OneLakeCredentialsRead", + "OneLakeCredentialsWrite", + "OneLakeDataSourceSettingsRead", + "OneLakeDataSourceSettingsWrite", + "OneLakeLocationDescription", +] + + +class OneLakeLocationDescription(CogniteResource): + """Location of a Fabric OneLake lakehouse. + + Args: + workspace_name (str): Fabric workspace GUID or name. + container_name (str): Fabric lakehouse GUID or name. + """ + + def __init__(self, workspace_name: str, container_name: str) -> None: + self.workspace_name = workspace_name + self.container_name = container_name + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + workspace_name=resource["workspaceName"], + container_name=resource["containerName"], + ) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + if camel_case: + return {"workspaceName": self.workspace_name, "containerName": self.container_name} + return {"workspace_name": self.workspace_name, "container_name": self.container_name} + + +class OneLakeCredentialsRead(CogniteResource): + """Read-only view of Azure credentials for Fabric OneLake (clientSecret is never returned by the API). + + Args: + client_id (str): Azure application (client) ID. + tenant_id (str): Azure tenant (directory) ID. + """ + + def __init__(self, client_id: str, tenant_id: str) -> None: + self.client_id = client_id + self.tenant_id = tenant_id + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + client_id=resource["clientId"], + tenant_id=resource["tenantId"], + ) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + if camel_case: + return {"clientId": self.client_id, "tenantId": self.tenant_id} + return {"client_id": self.client_id, "tenant_id": self.tenant_id} + + +class OneLakeCredentialsWrite(CogniteResource): + """Azure credentials for writing to Fabric OneLake. + + Args: + client_id (str): Azure application (client) ID. + tenant_id (str): Azure tenant (directory) ID. + client_secret (str | None): Azure client secret. Required for upsert; None when reconstructed + from a read model via as_write() since the API never returns the secret. + """ + + def __init__(self, client_id: str, tenant_id: str, client_secret: str | None = None) -> None: + self.client_id = client_id + self.tenant_id = tenant_id + self.client_secret = client_secret + + def __repr__(self) -> str: + secret_display = "***" if self.client_secret is not None else None + return ( + f"OneLakeCredentialsWrite(client_id={self.client_id!r}, tenant_id={self.tenant_id!r}," + f" client_secret={secret_display!r})" + ) + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + client_id=resource["clientId"], + tenant_id=resource["tenantId"], + client_secret=resource.get("clientSecret"), + ) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result: dict[str, Any] + if camel_case: + result = {"clientId": self.client_id, "tenantId": self.tenant_id} + if self.client_secret is not None: + result["clientSecret"] = self.client_secret + else: + result = {"client_id": self.client_id, "tenant_id": self.tenant_id} + if self.client_secret is not None: + result["client_secret"] = self.client_secret + return result + + +class OneLakeDataSourceSettingsRead(CogniteResource): + """Settings for a Fabric OneLake external data source (read model — no client secret). + + Args: + credentials (OneLakeCredentialsRead | None): Azure credentials (client ID and tenant ID only). + location_description (OneLakeLocationDescription | None): Fabric workspace and lakehouse identifiers. + """ + + def __init__( + self, + credentials: OneLakeCredentialsRead | None = None, + location_description: OneLakeLocationDescription | None = None, + ) -> None: + self.credentials = credentials + self.location_description = location_description + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + credentials = None + if (creds_raw := resource.get("credentials")) is not None: + credentials = OneLakeCredentialsRead._load(creds_raw) + location_description = None + if (loc_raw := resource.get("locationDescription")) is not None: + location_description = OneLakeLocationDescription._load(loc_raw) + return cls(credentials=credentials, location_description=location_description) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result: dict[str, Any] = {} + if self.credentials is not None: + result["credentials"] = self.credentials.dump(camel_case=camel_case) + if self.location_description is not None: + key = "locationDescription" if camel_case else "location_description" + result[key] = self.location_description.dump(camel_case=camel_case) + return result + + +class OneLakeDataSourceSettingsWrite(CogniteResource): + """Settings for writing a Fabric OneLake external data source (includes client secret). + + Args: + credentials (OneLakeCredentialsWrite | None): Azure credentials including client secret. + location_description (OneLakeLocationDescription | None): Fabric workspace and lakehouse identifiers. + """ + + def __init__( + self, + credentials: OneLakeCredentialsWrite | None = None, + location_description: OneLakeLocationDescription | None = None, + ) -> None: + self.credentials = credentials + self.location_description = location_description + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + credentials = None + if (creds_raw := resource.get("credentials")) is not None: + credentials = OneLakeCredentialsWrite._load(creds_raw) + location_description = None + if (loc_raw := resource.get("locationDescription")) is not None: + location_description = OneLakeLocationDescription._load(loc_raw) + return cls(credentials=credentials, location_description=location_description) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result: dict[str, Any] = {} + if self.credentials is not None: + result["credentials"] = self.credentials.dump(camel_case=camel_case) + if self.location_description is not None: + key = "locationDescription" if camel_case else "location_description" + result[key] = self.location_description.dump(camel_case=camel_case) + return result + + +class ExternalDataSourceCore(WriteableCogniteResource["ExternalDataSourceWrite"], ABC): + """Shared base for ExternalDataSource (read) and ExternalDataSourceWrite (write). + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + Args: + external_id (str): External ID of the data source. Must be unique within the project. + name (str | None): Human-readable name for the data source. + data_set_id (int | None): ID of the data set that owns this resource (for ACL scoping). + """ + + _FORMAT: ClassVar[str] = "one_lake" + + def __init__( + self, + external_id: str, + name: str | None = None, + data_set_id: int | None = None, + ) -> None: + self.external_id = external_id + self.name = name + self.data_set_id = data_set_id + + +class ExternalDataSource(ExternalDataSourceCore): + """A Fabric OneLake external data source (read model — returned by list). + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + The ``clientSecret`` field is **never** returned by the API. + + Args: + external_id (str): External ID of the data source. + name (str | None): Human-readable name. + data_set_id (int | None): Data set ID for ACL scoping. + settings (OneLakeDataSourceSettingsRead | None): Connection settings (no client secret). + format (str | None): Backend format identifier (always ``"one_lake"`` for OneLake sources). + created_time (int | None): Time the resource was created (milliseconds since epoch). + last_updated_time (int | None): Time the resource was last updated (milliseconds since epoch). + """ + + def __init__( + self, + external_id: str, + name: str | None = None, + data_set_id: int | None = None, + settings: OneLakeDataSourceSettingsRead | None = None, + format: str | None = None, + created_time: int | None = None, + last_updated_time: int | None = None, + ) -> None: + super().__init__(external_id=external_id, name=name, data_set_id=data_set_id) + self.settings = settings + self.format = format + self.created_time = created_time + self.last_updated_time = last_updated_time + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + fmt = resource.get("format") + if fmt is not None and fmt != cls._FORMAT: + warnings.warn( + f"Unknown external data source format: {fmt!r}. This version of the SDK may not fully support it.", + UserWarning, + stacklevel=2, + ) + settings = None + if (settings_raw := resource.get("settings")) is not None: + settings = OneLakeDataSourceSettingsRead._load(settings_raw) + return cls( + external_id=resource["externalId"], + name=resource.get("name"), + data_set_id=resource.get("dataSetId"), + settings=settings, + format=fmt, + created_time=resource.get("createdTime"), + last_updated_time=resource.get("lastUpdatedTime"), + ) + + def as_write(self) -> ExternalDataSourceWrite: + """Return this source as an ExternalDataSourceWrite. + + Note: The ``client_secret`` cannot be reconstructed from the read model (the API never returns it). + The returned write object will have ``client_secret=None`` on its credentials. + """ + settings_write: OneLakeDataSourceSettingsWrite | None = None + if self.settings is not None: + creds_write: OneLakeCredentialsWrite | None = None + if self.settings.credentials is not None: + creds_write = OneLakeCredentialsWrite( + client_id=self.settings.credentials.client_id, + tenant_id=self.settings.credentials.tenant_id, + client_secret=None, + ) + settings_write = OneLakeDataSourceSettingsWrite( + credentials=creds_write, + location_description=self.settings.location_description, + ) + return ExternalDataSourceWrite( + external_id=self.external_id, + name=self.name, + data_set_id=self.data_set_id, + settings=settings_write, + ) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result: dict[str, Any] = {} + if camel_case: + result["externalId"] = self.external_id + if self.name is not None: + result["name"] = self.name + if self.data_set_id is not None: + result["dataSetId"] = self.data_set_id + if self.settings is not None: + result["settings"] = self.settings.dump(camel_case=True) + if self.format is not None: + result["format"] = self.format + if self.created_time is not None: + result["createdTime"] = self.created_time + if self.last_updated_time is not None: + result["lastUpdatedTime"] = self.last_updated_time + else: + result["external_id"] = self.external_id + if self.name is not None: + result["name"] = self.name + if self.data_set_id is not None: + result["data_set_id"] = self.data_set_id + if self.settings is not None: + result["settings"] = self.settings.dump(camel_case=False) + if self.format is not None: + result["format"] = self.format + if self.created_time is not None: + result["created_time"] = self.created_time + if self.last_updated_time is not None: + result["last_updated_time"] = self.last_updated_time + return result + + +class ExternalDataSourceWrite(ExternalDataSourceCore): + """A Fabric OneLake external data source (write model — used for upsert). + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + The ``format`` field is always ``"one_lake"`` and is injected automatically on serialization. + + Args: + external_id (str): External ID of the data source. + name (str | None): Human-readable name. + data_set_id (int | None): Data set ID for ACL scoping. + settings (OneLakeDataSourceSettingsWrite | None): Connection settings including client secret. + """ + + def __init__( + self, + external_id: str, + name: str | None = None, + data_set_id: int | None = None, + settings: OneLakeDataSourceSettingsWrite | None = None, + ) -> None: + super().__init__(external_id=external_id, name=name, data_set_id=data_set_id) + self.settings = settings + + @classmethod + def onelake( + cls, + external_id: str, + client_id: str, + tenant_id: str, + client_secret: str, + workspace_name: str, + container_name: str, + name: str | None = None, + data_set_id: int | None = None, + ) -> ExternalDataSourceWrite: + """Create an ExternalDataSourceWrite for a Fabric OneLake source. + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + Args: + external_id (str): External ID for the data source. Must be unique. + client_id (str): Azure application (client) ID. + tenant_id (str): Azure tenant (directory) ID. + client_secret (str): Azure client secret. + workspace_name (str): Fabric workspace GUID or name. + container_name (str): Fabric lakehouse GUID or name. + name (str | None): Human-readable name. + data_set_id (int | None): Data set ID for ACL scoping. + + Returns: + ExternalDataSourceWrite: Ready to pass to ``client.transformations.external_data_sources.upsert()``. + + Examples: + + Register a Fabric OneLake source: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.transformations.external_data import ( + ... ExternalDataSourceWrite, + ... ) + >>> client = CogniteClient() + >>> source = ExternalDataSourceWrite.onelake( + ... external_id="fabric-lakehouse-prod", + ... name="Production lakehouse", + ... client_id="", + ... tenant_id="", + ... client_secret="", + ... workspace_name="", + ... container_name="", + ... data_set_id=123456, + ... ) + >>> client.transformations.external_data_sources.upsert(source) + """ + return cls( + external_id=external_id, + name=name, + data_set_id=data_set_id, + settings=OneLakeDataSourceSettingsWrite( + credentials=OneLakeCredentialsWrite( + client_id=client_id, + tenant_id=tenant_id, + client_secret=client_secret, + ), + location_description=OneLakeLocationDescription( + workspace_name=workspace_name, + container_name=container_name, + ), + ), + ) + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + settings = None + if (settings_raw := resource.get("settings")) is not None: + settings = OneLakeDataSourceSettingsWrite._load(settings_raw) + return cls( + external_id=resource["externalId"], + name=resource.get("name"), + data_set_id=resource.get("dataSetId"), + settings=settings, + ) + + def as_write(self) -> ExternalDataSourceWrite: + return self + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result: dict[str, Any] = {} + if camel_case: + result["externalId"] = self.external_id + result["format"] = self._FORMAT + if self.name is not None: + result["name"] = self.name + if self.data_set_id is not None: + result["dataSetId"] = self.data_set_id + if self.settings is not None: + result["settings"] = self.settings.dump(camel_case=True) + else: + result["external_id"] = self.external_id + result["format"] = self._FORMAT + if self.name is not None: + result["name"] = self.name + if self.data_set_id is not None: + result["data_set_id"] = self.data_set_id + if self.settings is not None: + result["settings"] = self.settings.dump(camel_case=False) + return result + + +class ExternalDataSourceList( + WriteableCogniteResourceList[ExternalDataSourceWrite, ExternalDataSource], ExternalIDTransformerMixin +): + """A list of ExternalDataSource (read model) objects.""" + + _RESOURCE = ExternalDataSource + + def as_write(self) -> ExternalDataSourceWriteList: + """Return all sources in their write format (client_secret will be None on each).""" + return ExternalDataSourceWriteList([item.as_write() for item in self.data]) + + +class ExternalDataSourceWriteList(CogniteResourceList[ExternalDataSourceWrite], ExternalIDTransformerMixin): + """A list of ExternalDataSourceWrite objects.""" + + _RESOURCE = ExternalDataSourceWrite + + +class ExternalDataSourceUsability(CogniteResource): + """Result of verifying a Fabric OneLake external data source's usability. + + Args: + external_id (str | None): External ID of the verified data source. + usable_version (str | None): UUID indicating the data source is accessible and credentials are valid. + ``None`` if the source cannot be accessed. + """ + + def __init__(self, external_id: str | None = None, usable_version: str | None = None) -> None: + self.external_id = external_id + self.usable_version = usable_version + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + external_id=resource.get("externalId"), + usable_version=resource.get("usableVersion"), + ) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + if camel_case: + return {"externalId": self.external_id, "usableVersion": self.usable_version} + return {"external_id": self.external_id, "usable_version": self.usable_version} diff --git a/cognite/client/testing.py b/cognite/client/testing.py index bb66f744fe..7ad558f0e1 100644 --- a/cognite/client/testing.py +++ b/cognite/client/testing.py @@ -78,6 +78,7 @@ from cognite.client._api.three_d.revisions import ThreeDRevisionsAPI from cognite.client._api.time_series import TimeSeriesAPI from cognite.client._api.transformations import TransformationsAPI +from cognite.client._api.transformations.external_data import TransformationExternalDataAPI from cognite.client._api.transformations.jobs import TransformationJobsAPI from cognite.client._api.transformations.notifications import TransformationNotificationsAPI from cognite.client._api.transformations.schedules import TransformationSchedulesAPI @@ -163,6 +164,7 @@ from cognite.client._sync_api.three_d.revisions import Sync3DRevisionsAPI from cognite.client._sync_api.time_series import SyncTimeSeriesAPI from cognite.client._sync_api.transformations import SyncTransformationsAPI +from cognite.client._sync_api.transformations.external_data import SyncTransformationExternalDataAPI from cognite.client._sync_api.transformations.jobs import SyncTransformationJobsAPI from cognite.client._sync_api.transformations.notifications import SyncTransformationNotificationsAPI from cognite.client._sync_api.transformations.schedules import SyncTransformationSchedulesAPI @@ -366,6 +368,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: tr_notifications = create_autospec(TransformationNotificationsAPI, instance=True, spec_set=True) tr_schedules = create_autospec(TransformationSchedulesAPI, instance=True, spec_set=True) tr_schema = create_autospec(TransformationSchemaAPI, instance=True, spec_set=True) + tr_external_data = create_autospec(TransformationExternalDataAPI, instance=True, spec_set=True) self.transformations = create_autospec( TransformationsAPI, instance=True, @@ -373,6 +376,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: notifications=tr_notifications, schedules=tr_schedules, schema=tr_schema, + external_data_sources=tr_external_data, ) flip_spec_set_on(self.transformations) @@ -571,6 +575,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: tr_notifications = create_autospec(SyncTransformationNotificationsAPI, instance=True, spec_set=True) tr_schedules = create_autospec(SyncTransformationSchedulesAPI, instance=True, spec_set=True) tr_schema = create_autospec(SyncTransformationSchemaAPI, instance=True, spec_set=True) + tr_external_data_sync = create_autospec(SyncTransformationExternalDataAPI, instance=True, spec_set=True) self.transformations = create_autospec( SyncTransformationsAPI, instance=True, @@ -578,6 +583,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: notifications=tr_notifications, schedules=tr_schedules, schema=tr_schema, + external_data_sources=tr_external_data_sync, ) flip_spec_set_on(self.transformations) diff --git a/docs/source/transformations.rst b/docs/source/transformations.rst index 8d2d3a5ec6..48b601fcbf 100644 --- a/docs/source/transformations.rst +++ b/docs/source/transformations.rst @@ -48,6 +48,14 @@ Transformation Schema AsyncCogniteClient.transformations.schema +External Data Sources +--------------------- + +.. toctree:: + :maxdepth: 2 + + transformations_external_data + Data classes ------------ .. automodule:: cognite.client.data_classes.transformations diff --git a/docs/source/transformations_external_data.rst b/docs/source/transformations_external_data.rst new file mode 100644 index 0000000000..7ab0b24851 --- /dev/null +++ b/docs/source/transformations_external_data.rst @@ -0,0 +1,142 @@ +External Data Sources +====================== + +.. currentmodule:: cognite.client + +.. warning:: + + **OneLake is read-only from a transform perspective.** Transforms can read data from Fabric OneLake tables + via ``ext_onelake()`` SQL, but writing back to OneLake is **not supported**. External data sources are + credentials and location information that transforms use to access OneLake tables — they are not + destinations for transform output. + +Introduction +------------ + +External data sources allow transformations to read data from Fabric OneLake by registering OneLake +workspace and lakehouse credentials. Once registered, a transform can access tables in OneLake via +the ``ext_onelake('source-id', 'table_name')`` SQL function. + +Each external data source is identified by a unique ``external_id`` and stores Azure service principal +credentials (client ID, tenant ID, and client secret) along with the target Fabric workspace and +lakehouse identifiers. + +Quickstart +---------- + +Register a Fabric OneLake Source +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Use the factory method ``ExternalDataSourceWrite.onelake()`` to create a source, then call ``upsert()`` +to register it in your project: + +.. code-block:: python + + from cognite.client import CogniteClient + from cognite.client.data_classes.transformations.external_data import ExternalDataSourceWrite + + client = CogniteClient() + + # Create the external data source + source = ExternalDataSourceWrite.onelake( + external_id="fabric-lakehouse-prod", + name="Production lakehouse", + client_id="", + tenant_id="", + client_secret="", + workspace_name="", + container_name="", + data_set_id=123456, + ) + + # Register the source + registered_source = client.transformations.external_data_sources.upsert(source) + print(f"Registered source: {registered_source.external_id}") + +Verify Source Usability +^^^^^^^^^^^^^^^^^^^^^^^ + +Before running a transform, verify that the source is accessible and credentials are valid: + +.. code-block:: python + + result = client.transformations.external_data_sources.verify_usability("fabric-lakehouse-prod") + + if result.usable_version is not None: + print("Source is accessible") + else: + print("Source cannot be accessed — check credentials") + +Create and Run a Transform Using OneLake Data +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Once the external data source is registered, use it in a transform SQL via the ``ext_onelake()`` function. +The transform can read from OneLake tables and write results to CDF: + +.. code-block:: python + + from cognite.client.data_classes.transformations import ( + Transformation, + TransformationDestination, + ) + + transform = Transformation( + external_id="onelake-to-cdf-assets", + name="OneLake to CDF Assets", + destination=TransformationDestination.assets(), + query=""" + SELECT + name, + description + FROM ext_onelake('fabric-lakehouse-prod', 'my_table') + WHERE active = true + """, + ) + + created = client.transformations.create(transform) + print(f"Created transform: {created.external_id}") + + # Run the transform + job = client.transformations.jobs.run(created.id) + print(f"Job {job.id} started") + +List Registered Sources +^^^^^^^^^^^^^^^^^^^^^^^ + +Retrieve all registered external data sources: + +.. code-block:: python + + sources = client.transformations.external_data_sources.list() + for source in sources: + print(f"Source: {source.external_id} ({source.name})") + +Delete a Source +^^^^^^^^^^^^^^^ + +Remove a source when it is no longer needed: + +.. code-block:: python + + client.transformations.external_data_sources.delete("fabric-lakehouse-prod") + print("Source deleted") + +API Reference +------------- + +TransformationExternalDataAPI +----------------------------- + +.. autosummary:: + :methods: + :toctree: generated/ + :template: custom-automethods-template.rst + + AsyncCogniteClient.transformations.external_data_sources + +Data Classes +------------ + +.. automodule:: cognite.client.data_classes.transformations.external_data + :members: + :show-inheritance: diff --git a/tests/tests_integration/test_api/test_transformations/test_external_data.py b/tests/tests_integration/test_api/test_transformations/test_external_data.py new file mode 100644 index 0000000000..20d60925d2 --- /dev/null +++ b/tests/tests_integration/test_api/test_transformations/test_external_data.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from cognite.client import CogniteClient +from cognite.client.data_classes.transformations.external_data import ( + ExternalDataSource, + ExternalDataSourceList, + ExternalDataSourceUsability, + ExternalDataSourceWrite, +) + +_JETFIRE_ENV = Path(__file__).parents[5] / "jetfire-backend" / ".env" +_SKIP_REASON = f"Fabric integration env not available ({_JETFIRE_ENV})" + + +def _load_jetfire_env() -> None: + """Parse key=value pairs from the jetfire-backend .env file into os.environ.""" + for line in _JETFIRE_ENV.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + os.environ.setdefault(key.strip(), value.strip()) + + +@pytest.fixture(scope="module", autouse=True) +def load_fabric_env() -> None: + if _JETFIRE_ENV.exists(): + _load_jetfire_env() + + +_FABRIC_ENV_VARS = ( + "FABRIC_CLIENT_ID", + "FABRIC_TENANT_ID", + "FABRIC_CLIENT_SECRET", + "FABRIC_WORKSPACE", + "FABRIC_LAKEHOUSE", +) + + +def _fabric_ci_available() -> bool: + if not _JETFIRE_ENV.exists(): + return False + _load_jetfire_env() + return all(os.environ.get(key) for key in _FABRIC_ENV_VARS) + + +@pytest.mark.skipif(not _fabric_ci_available(), reason=_SKIP_REASON) +class TestExternalDataSourcesIntegration: + """End-to-end lifecycle test: upsert → list → verify_usability → delete.""" + + _EXTERNAL_ID = "sdk-integration-test-fabric-onelake" + + def test_lifecycle(self, cognite_client: CogniteClient) -> None: + client_id = os.environ["FABRIC_CLIENT_ID"] + tenant_id = os.environ["FABRIC_TENANT_ID"] + client_secret = os.environ["FABRIC_CLIENT_SECRET"] + workspace_name = os.environ["FABRIC_WORKSPACE"] + container_name = os.environ["FABRIC_LAKEHOUSE"] + + source = ExternalDataSourceWrite.onelake( + external_id=self._EXTERNAL_ID, + client_id=client_id, + tenant_id=tenant_id, + client_secret=client_secret, + workspace_name=workspace_name, + container_name=container_name, + ) + + try: + # Upsert + upserted = cognite_client.transformations.external_data_sources.upsert(source) + assert isinstance(upserted, ExternalDataSource) + assert upserted.external_id == self._EXTERNAL_ID + assert upserted.format == "one_lake" + + # List — verify the upserted source is present + all_sources = cognite_client.transformations.external_data_sources.list() + assert isinstance(all_sources, ExternalDataSourceList) + external_ids = {s.external_id for s in all_sources} + assert self._EXTERNAL_ID in external_ids + + # Verify usability — expects valid credentials and reachable workspace + usability = cognite_client.transformations.external_data_sources.verify_usability(self._EXTERNAL_ID) + assert isinstance(usability, ExternalDataSourceUsability) + # usable_version is a UUID when the source is accessible; None when credentials are invalid + assert usability.usable_version is not None, ( + "verify_usability returned None — credentials may be invalid or workspace unreachable" + ) + + finally: + # Always delete to avoid leaving stale test resources + cognite_client.transformations.external_data_sources.delete(self._EXTERNAL_ID) diff --git a/tests/tests_unit/test_api/test_transformation_external_data.py b/tests/tests_unit/test_api/test_transformation_external_data.py new file mode 100644 index 0000000000..8ab668e99d --- /dev/null +++ b/tests/tests_unit/test_api/test_transformation_external_data.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +from collections.abc import Iterator + +import pytest +from pytest_httpx import HTTPXMock + +from cognite.client import AsyncCogniteClient, CogniteClient +from cognite.client.data_classes.transformations.external_data import ( + ExternalDataSource, + ExternalDataSourceList, + ExternalDataSourceUsability, + ExternalDataSourceWrite, +) +from tests.utils import get_url, jsgz_load + + +@pytest.fixture +def external_data_url(async_client: AsyncCogniteClient) -> str: + return get_url( + async_client.transformations.external_data_sources, + async_client.transformations.external_data_sources._RESOURCE_PATH, + ) + + +@pytest.fixture +def source_response_body() -> dict: + return { + "items": [ + { + "externalId": "x", + "format": "one_lake", + "settings": { + "credentials": {"clientId": "cid", "tenantId": "tid"}, + "locationDescription": {"workspaceName": "ws", "containerName": "cn"}, + }, + } + ] + } + + +@pytest.fixture +def mock_list_response( + httpx_mock: HTTPXMock, external_data_url: str, source_response_body: dict +) -> Iterator[HTTPXMock]: + httpx_mock.add_response( + method="GET", + url=external_data_url + "?limit=1000", + status_code=200, + json=source_response_body, + ) + yield httpx_mock + + +@pytest.fixture +def mock_upsert_response( + httpx_mock: HTTPXMock, external_data_url: str, source_response_body: dict +) -> Iterator[HTTPXMock]: + httpx_mock.add_response(method="POST", url=external_data_url, status_code=201, json=source_response_body) + yield httpx_mock + + +@pytest.fixture +def mock_delete_response(httpx_mock: HTTPXMock, external_data_url: str) -> Iterator[HTTPXMock]: + httpx_mock.add_response(method="POST", url=external_data_url + "/delete", status_code=200, json={}) + yield httpx_mock + + +@pytest.fixture +def mock_verify_usability_response(httpx_mock: HTTPXMock, external_data_url: str) -> Iterator[HTTPXMock]: + httpx_mock.add_response( + method="POST", + url=external_data_url + "/usability", + status_code=200, + json={"externalId": "x", "usableVersion": "abc-uuid"}, + ) + yield httpx_mock + + +class TestTransformationExternalDataAPI: + @pytest.mark.usefixtures("mock_list_response") + def test_list(self, cognite_client: CogniteClient) -> None: + result = cognite_client.transformations.external_data_sources.list() + + assert isinstance(result, ExternalDataSourceList) + assert len(result) == 1 + assert result[0].external_id == "x" + assert result[0].format == "one_lake" + + def test_upsert_single(self, cognite_client: CogniteClient, mock_upsert_response: HTTPXMock) -> None: + source = ExternalDataSourceWrite.onelake( + external_id="x", + client_id="cid", + tenant_id="tid", + client_secret="sec", + workspace_name="ws", + container_name="cn", + ) + result = cognite_client.transformations.external_data_sources.upsert(source) + + assert isinstance(result, ExternalDataSource) + assert result.external_id == "x" + # Verify the POST body contained "format": "one_lake" + request_body = jsgz_load(mock_upsert_response.get_requests()[-1].content) + assert request_body["items"][0]["format"] == "one_lake" + + def test_delete( + self, + cognite_client: CogniteClient, + async_client: AsyncCogniteClient, + mock_delete_response: HTTPXMock, + ) -> None: + cognite_client.transformations.external_data_sources.delete("x") + + last_request = mock_delete_response.get_requests()[-1] + url = str(last_request.url) + assert url.endswith(async_client.transformations.external_data_sources._RESOURCE_PATH + "/delete") + request_body = jsgz_load(last_request.content) + assert request_body["items"] == [{"externalId": "x"}] + + def test_verify_usability(self, cognite_client: CogniteClient, mock_verify_usability_response: HTTPXMock) -> None: + result = cognite_client.transformations.external_data_sources.verify_usability("x") + + assert isinstance(result, ExternalDataSourceUsability) + assert result.usable_version == "abc-uuid" + assert result.external_id == "x" diff --git a/tests/tests_unit/test_data_classes/test_transformations/__init__.py b/tests/tests_unit/test_data_classes/test_transformations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py b/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py new file mode 100644 index 0000000000..38f66cb3cf --- /dev/null +++ b/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import pytest + +from cognite.client.data_classes.transformations.external_data import ( + ExternalDataSource, + ExternalDataSourceUsability, + ExternalDataSourceWrite, + OneLakeCredentialsWrite, +) + + +def test_onelake_factory_produces_valid_structure() -> None: + source = ExternalDataSourceWrite.onelake( + external_id="x", + client_id="cid", + tenant_id="tid", + client_secret="sec", + workspace_name="ws", + container_name="cn", + ) + dumped = source.dump(camel_case=True) + + assert dumped["format"] == "one_lake" + assert dumped["externalId"] == "x" + assert dumped["settings"]["credentials"]["clientId"] == "cid" + assert dumped["settings"]["credentials"]["clientSecret"] == "sec" + assert dumped["settings"]["locationDescription"]["workspaceName"] == "ws" + + +def test_write_dump_always_includes_format() -> None: + source = ExternalDataSourceWrite(external_id="x") + dumped = source.dump(camel_case=True) + + assert "format" in dumped + assert dumped["format"] == "one_lake" + + +def test_read_load_parses_settings() -> None: + raw = { + "externalId": "x", + "format": "one_lake", + "settings": { + "credentials": {"clientId": "cid", "tenantId": "tid"}, + "locationDescription": {"workspaceName": "ws", "containerName": "cn"}, + }, + } + source = ExternalDataSource._load(raw) + + assert source.external_id == "x" + assert source.format == "one_lake" + assert source.settings is not None + assert source.settings.credentials is not None + assert source.settings.credentials.client_id == "cid" + assert source.settings.credentials.tenant_id == "tid" + assert source.settings.location_description is not None + assert source.settings.location_description.workspace_name == "ws" + assert source.settings.location_description.container_name == "cn" + # Read model never carries client_secret + assert not hasattr(source.settings.credentials, "client_secret") + + +def test_read_load_unknown_format_warns_not_raises() -> None: + raw = {"externalId": "x", "format": "delta_sharing", "settings": {}} + + with pytest.warns(UserWarning, match="Unknown external data source format"): + source = ExternalDataSource._load(raw) + + # Must not raise — the object is returned despite the unknown format + assert source.external_id == "x" + assert source.format == "delta_sharing" + + +def test_as_write_returns_write_with_none_secret() -> None: + raw = { + "externalId": "x", + "format": "one_lake", + "settings": { + "credentials": {"clientId": "cid", "tenantId": "tid"}, + "locationDescription": {"workspaceName": "ws", "containerName": "cn"}, + }, + } + read_source = ExternalDataSource._load(raw) + write_source = read_source.as_write() + + assert isinstance(write_source, ExternalDataSourceWrite) + assert write_source.settings is not None + assert write_source.settings.credentials is not None + assert write_source.settings.credentials.client_secret is None + + +def test_credentials_write_repr_masks_secret() -> None: + creds = OneLakeCredentialsWrite("cid", "tid", "actual-secret") + result = repr(creds) + + assert "actual-secret" not in result + assert "***" in result + + +def test_usability_load_with_version() -> None: + usability = ExternalDataSourceUsability._load({"externalId": "x", "usableVersion": "some-uuid"}) + + assert usability.external_id == "x" + assert usability.usable_version == "some-uuid" + + +def test_usability_load_with_null_version() -> None: + usability = ExternalDataSourceUsability._load({"externalId": "x", "usableVersion": None}) + + assert usability.external_id == "x" + assert usability.usable_version is None