From 8331b65139b97c615a7859f26107f485e85efb9c Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Thu, 9 Jul 2026 09:33:42 +0530 Subject: [PATCH 1/5] feat(transformations): add Fabric OneLake external data sources API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds client.transformations.external_data_sources as a new sub-resource under TransformationsAPI, enabling users to register, list, verify, and delete Fabric OneLake external data sources for use in transforms. OneLake sources are read-only — transforms read from OneLake tables via ext_onelake() SQL but writing back to OneLake is not supported. New public surface: - ExternalDataSourceWrite.onelake() factory for creating sources - client.transformations.external_data_sources.upsert() - client.transformations.external_data_sources.list() - client.transformations.external_data_sources.delete() - client.transformations.external_data_sources.verify_usability() - TransformationsExternalDataSourcesAcl capability class (READ/WRITE) Implementation follows the existing schedules/notifications sub-resource pattern: three-layer Core/Read/Write data class hierarchy, WriteList companion, sync wrapper generated from the async API, and autospec mock support in CogniteClientMock. Integration tests are local-only (require work/jetfire-backend/.env with Fabric credentials) and skip automatically in CI. --- .../client/_api/transformations/__init__.py | 2 + .../_api/transformations/external_data.py | 155 ++++++ .../_sync_api/transformations/__init__.py | 2 + .../transformations/external_data.py | 137 +++++ cognite/client/data_classes/__init__.py | 22 + cognite/client/data_classes/capabilities.py | 15 + .../transformations/external_data.py | 504 ++++++++++++++++++ cognite/client/testing.py | 6 + docs/source/transformations.rst | 8 + docs/source/transformations_external_data.rst | 142 +++++ .../test_external_data.py | 81 +++ .../test_transformation_external_data.py | 125 +++++ .../test_transformations/__init__.py | 0 .../test_external_data.py | 111 ++++ 14 files changed, 1310 insertions(+) create mode 100644 cognite/client/_api/transformations/external_data.py create mode 100644 cognite/client/_sync_api/transformations/external_data.py create mode 100644 cognite/client/data_classes/transformations/external_data.py create mode 100644 docs/source/transformations_external_data.rst create mode 100644 tests/tests_integration/test_api/test_transformations/test_external_data.py create mode 100644 tests/tests_unit/test_api/test_transformation_external_data.py create mode 100644 tests/tests_unit/test_data_classes/test_transformations/__init__.py create mode 100644 tests/tests_unit/test_data_classes/test_transformations/test_external_data.py diff --git a/cognite/client/_api/transformations/__init__.py b/cognite/client/_api/transformations/__init__.py index 5e565beb6b..9da907e274 100644 --- a/cognite/client/_api/transformations/__init__.py +++ b/cognite/client/_api/transformations/__init__.py @@ -3,6 +3,7 @@ from collections.abc import AsyncIterator, Sequence from typing import TYPE_CHECKING, Any, Literal, overload +from cognite.client._api.transformations.external_data import TransformationExternalDataAPI from cognite.client._api.transformations.jobs import TransformationJobsAPI from cognite.client._api.transformations.notifications import TransformationNotificationsAPI from cognite.client._api.transformations.schedules import TransformationSchedulesAPI @@ -36,6 +37,7 @@ def __init__(self, config: ClientConfig, api_version: str | None, cognite_client self.schedules = TransformationSchedulesAPI(config, api_version, cognite_client) self.schema = TransformationSchemaAPI(config, api_version, cognite_client) self.notifications = TransformationNotificationsAPI(config, api_version, cognite_client) + self.external_data_sources = TransformationExternalDataAPI(config, api_version, cognite_client) @overload def __call__( diff --git a/cognite/client/_api/transformations/external_data.py b/cognite/client/_api/transformations/external_data.py new file mode 100644 index 0000000000..8c921ca164 --- /dev/null +++ b/cognite/client/_api/transformations/external_data.py @@ -0,0 +1,155 @@ +from __future__ import annotations + +from collections.abc import Sequence + +from cognite.client._api_client import APIClient +from cognite.client.data_classes.transformations.external_data import ( + ExternalDataSource, + ExternalDataSourceList, + ExternalDataSourceUsability, + ExternalDataSourceWrite, + ExternalDataSourceWriteList, +) +from cognite.client.utils._identifier import IdentifierSequence +from cognite.client.utils.useful_types import SequenceNotStr + + +class TransformationExternalDataAPI(APIClient): + """`Manage Fabric OneLake external data sources for transformations `_. + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + """ + + _RESOURCE_PATH = "/transformations/external_data" + _CREATE_LIMIT = 1000 + _DELETE_LIMIT = 1000 + + async def list(self, limit: int | None = None) -> ExternalDataSourceList: + """`List Fabric OneLake external data sources `_. + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + Args: + limit (int | None): Maximum number of results to return. Use ``None`` or ``-1`` to return all. + Defaults to returning all. + + Returns: + ExternalDataSourceList: All registered OneLake external data sources. + + Examples: + + List all registered external data sources:: + + >>> from cognite.client import CogniteClient + >>> client = CogniteClient() + >>> sources = client.transformations.external_data_sources.list() + """ + return await self._list( + method="GET", + list_cls=ExternalDataSourceList, + resource_cls=ExternalDataSource, + limit=limit, + ) + + async def upsert( + self, + source: ExternalDataSourceWrite | Sequence[ExternalDataSourceWrite], + ) -> ExternalDataSource | ExternalDataSourceList: + """`Create or update (upsert) Fabric OneLake external data sources `_. + + An upsert creates the source if it doesn't exist, or overwrites it entirely if it does. + Uniqueness is determined by ``externalId``. + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + Args: + source (ExternalDataSourceWrite | Sequence[ExternalDataSourceWrite]): Single source or list of sources to upsert. + + Returns: + ExternalDataSource | ExternalDataSourceList: The upserted source(s). + + Examples: + + Register a Fabric OneLake source: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.transformations.external_data import ExternalDataSourceWrite + >>> client = CogniteClient() + >>> source = ExternalDataSourceWrite.onelake( + ... external_id="fabric-lakehouse-prod", + ... name="Production lakehouse", + ... client_id="", + ... tenant_id="", + ... client_secret="", + ... workspace_name="", + ... container_name="", + ... data_set_id=123456, + ... ) + >>> res = client.transformations.external_data_sources.upsert(source) + """ + return await self._create_multiple( + items=source, + list_cls=ExternalDataSourceList, + resource_cls=ExternalDataSource, + input_resource_cls=ExternalDataSourceWrite, + ) + + async def delete(self, external_id: str | SequenceNotStr[str]) -> None: + """`Delete Fabric OneLake external data sources `_. + + Args: + external_id (str | SequenceNotStr[str]): External ID or list of external IDs to delete. + + Examples: + + Delete a source by external ID:: + + >>> from cognite.client import CogniteClient + >>> client = CogniteClient() + >>> client.transformations.external_data_sources.delete("fabric-lakehouse-prod") + + Delete multiple sources:: + + >>> client.transformations.external_data_sources.delete( + ... ["fabric-lakehouse-prod", "fabric-lakehouse-staging"] + ... ) + """ + await self._delete_multiple( + identifiers=IdentifierSequence.load(external_ids=external_id), + wrap_ids=True, + ) + + async def verify_usability(self, external_id: str) -> ExternalDataSourceUsability: + """`Verify that a Fabric OneLake external data source is usable `_. + + Checks that the source exists and that the configured Azure credentials can access the specified + Fabric lakehouse. Returns a ``usable_version`` UUID if the source is accessible. + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + Args: + external_id (str): External ID of the source to verify. + + Returns: + ExternalDataSourceUsability: Contains ``usable_version`` (a UUID) if the source is accessible, + or ``None`` if the credentials are invalid or the source cannot be reached. + + Examples: + + Verify a source before running a transformation:: + + >>> from cognite.client import CogniteClient + >>> client = CogniteClient() + >>> result = client.transformations.external_data_sources.verify_usability("fabric-lakehouse-prod") + >>> assert result.usable_version is not None, "Source not configured or credentials invalid" + """ + res = await self._post( + url_path=self._RESOURCE_PATH + "/usability", + json={"externalId": external_id}, + semaphore=self._get_semaphore("write"), + ) + return ExternalDataSourceUsability._load(res.json()) diff --git a/cognite/client/_sync_api/transformations/__init__.py b/cognite/client/_sync_api/transformations/__init__.py index 9eb4f69eb2..660099e4f6 100644 --- a/cognite/client/_sync_api/transformations/__init__.py +++ b/cognite/client/_sync_api/transformations/__init__.py @@ -12,6 +12,7 @@ from cognite.client import AsyncCogniteClient from cognite.client._constants import DEFAULT_LIMIT_READ +from cognite.client._sync_api.transformations.external_data import SyncTransformationExternalDataAPI from cognite.client._sync_api.transformations.jobs import SyncTransformationJobsAPI from cognite.client._sync_api.transformations.notifications import SyncTransformationNotificationsAPI from cognite.client._sync_api.transformations.schedules import SyncTransformationSchedulesAPI @@ -41,6 +42,7 @@ def __init__(self, async_client: AsyncCogniteClient) -> None: self.schedules = SyncTransformationSchedulesAPI(async_client) self.schema = SyncTransformationSchemaAPI(async_client) self.notifications = SyncTransformationNotificationsAPI(async_client) + self.external_data_sources = SyncTransformationExternalDataAPI(async_client) @overload def __call__( diff --git a/cognite/client/_sync_api/transformations/external_data.py b/cognite/client/_sync_api/transformations/external_data.py new file mode 100644 index 0000000000..d10cf6fc65 --- /dev/null +++ b/cognite/client/_sync_api/transformations/external_data.py @@ -0,0 +1,137 @@ +""" +=============================================================================== +This file mirrors cognite/client/_api/transformations/external_data.py. +If the async API changes, update this file manually to match. +=============================================================================== +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from cognite.client import AsyncCogniteClient +from cognite.client._sync_api_client import SyncAPIClient +from cognite.client.data_classes.transformations.external_data import ( + ExternalDataSource, + ExternalDataSourceList, + ExternalDataSourceUsability, + ExternalDataSourceWrite, +) +from cognite.client.utils._async_helpers import run_sync +from cognite.client.utils.useful_types import SequenceNotStr + + +class SyncTransformationExternalDataAPI(SyncAPIClient): + """Sync wrapper for TransformationExternalDataAPI.""" + + def __init__(self, async_client: AsyncCogniteClient) -> None: + self.__async_client = async_client + + def list(self, limit: int | None = None) -> ExternalDataSourceList: + """List Fabric OneLake external data sources. + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + Args: + limit (int | None): Maximum number of results to return. Use ``None`` or ``-1`` to return all. + Defaults to returning all. + + Returns: + ExternalDataSourceList: All registered OneLake external data sources. + + Examples: + + List all registered external data sources:: + + >>> from cognite.client import CogniteClient + >>> client = CogniteClient() + >>> sources = client.transformations.external_data_sources.list() + """ + return run_sync(self.__async_client.transformations.external_data_sources.list(limit=limit)) + + def upsert( + self, + source: ExternalDataSourceWrite | Sequence[ExternalDataSourceWrite], + ) -> ExternalDataSource | ExternalDataSourceList: + """Create or update (upsert) Fabric OneLake external data sources. + + An upsert creates the source if it doesn't exist, or overwrites it entirely if it does. + Uniqueness is determined by ``externalId``. + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + Args: + source (ExternalDataSourceWrite | Sequence[ExternalDataSourceWrite]): Single source or list of sources to upsert. + + Returns: + ExternalDataSource | ExternalDataSourceList: The upserted source(s). + + Examples: + + Register a Fabric OneLake source: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.transformations.external_data import ExternalDataSourceWrite + >>> client = CogniteClient() + >>> source = ExternalDataSourceWrite.onelake( + ... external_id="fabric-lakehouse-prod", + ... name="Production lakehouse", + ... client_id="", + ... tenant_id="", + ... client_secret="", + ... workspace_name="", + ... container_name="", + ... data_set_id=123456, + ... ) + >>> res = client.transformations.external_data_sources.upsert(source) + """ + return run_sync(self.__async_client.transformations.external_data_sources.upsert(source=source)) + + def delete(self, external_id: str | SequenceNotStr[str]) -> None: + """Delete Fabric OneLake external data sources. + + Args: + external_id (str | SequenceNotStr[str]): External ID or list of external IDs to delete. + + Examples: + + Delete a source by external ID:: + + >>> from cognite.client import CogniteClient + >>> client = CogniteClient() + >>> client.transformations.external_data_sources.delete("fabric-lakehouse-prod") + """ + return run_sync( + self.__async_client.transformations.external_data_sources.delete(external_id=external_id) + ) + + def verify_usability(self, external_id: str) -> ExternalDataSourceUsability: + """Verify that a Fabric OneLake external data source is usable. + + Checks that the source exists and that the configured Azure credentials can access the specified + Fabric lakehouse. Returns a ``usable_version`` UUID if the source is accessible. + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + Args: + external_id (str): External ID of the source to verify. + + Returns: + ExternalDataSourceUsability: Contains ``usable_version`` (a UUID) if the source is accessible, + or ``None`` if the credentials are invalid or the source cannot be reached. + + Examples: + + Verify a source before running a transformation:: + + >>> from cognite.client import CogniteClient + >>> client = CogniteClient() + >>> result = client.transformations.external_data_sources.verify_usability("fabric-lakehouse-prod") + >>> assert result.usable_version is not None, "Source not configured or credentials invalid" + """ + return run_sync( + self.__async_client.transformations.external_data_sources.verify_usability(external_id=external_id) + ) diff --git a/cognite/client/data_classes/__init__.py b/cognite/client/data_classes/__init__.py index 8247c0d60d..b1bef372ed 100644 --- a/cognite/client/data_classes/__init__.py +++ b/cognite/client/data_classes/__init__.py @@ -259,6 +259,18 @@ TransformationJobMetricList, TransformationJobStatus, ) +from cognite.client.data_classes.transformations.external_data import ( + ExternalDataSource, + ExternalDataSourceList, + ExternalDataSourceUsability, + ExternalDataSourceWrite, + ExternalDataSourceWriteList, + OneLakeCredentialsRead, + OneLakeCredentialsWrite, + OneLakeDataSourceSettingsRead, + OneLakeDataSourceSettingsWrite, + OneLakeLocationDescription, +) from cognite.client.data_classes.transformations.notifications import ( TransformationNotification, TransformationNotificationList, @@ -526,6 +538,16 @@ "TimeSeriesWriteList", "TimestampRange", "Transformation", + "ExternalDataSource", + "ExternalDataSourceList", + "ExternalDataSourceUsability", + "ExternalDataSourceWrite", + "ExternalDataSourceWriteList", + "OneLakeCredentialsRead", + "OneLakeCredentialsWrite", + "OneLakeDataSourceSettingsRead", + "OneLakeDataSourceSettingsWrite", + "OneLakeLocationDescription", "TransformationBlockedInfo", "TransformationDestination", "TransformationJob", diff --git a/cognite/client/data_classes/capabilities.py b/cognite/client/data_classes/capabilities.py index 35536f3a19..1dfd2103f8 100644 --- a/cognite/client/data_classes/capabilities.py +++ b/cognite/client/data_classes/capabilities.py @@ -1104,6 +1104,21 @@ class Scope: DataSet = DataSetScope +@dataclass +class TransformationsExternalDataSourcesAcl(Capability): + _capability_name = "transformationsExternalDataSourcesAcl" + actions: Sequence[Action] + scope: AllScope | DataSetScope + + class Action(Capability.Action): # type: ignore [misc] + Read = "READ" + Write = "WRITE" + + class Scope: + All = AllScope + DataSet = DataSetScope + + @dataclass class TypesAcl(Capability): _capability_name = "typesAcl" diff --git a/cognite/client/data_classes/transformations/external_data.py b/cognite/client/data_classes/transformations/external_data.py new file mode 100644 index 0000000000..792b3b0a1c --- /dev/null +++ b/cognite/client/data_classes/transformations/external_data.py @@ -0,0 +1,504 @@ +from __future__ import annotations + +import warnings +from abc import ABC +from typing import Any, ClassVar + +from typing_extensions import Self + +from cognite.client.data_classes._base import ( + CogniteResource, + CogniteResourceList, + WriteableCogniteResource, + WriteableCogniteResourceList, +) + +__all__ = [ + "ExternalDataSource", + "ExternalDataSourceCore", + "ExternalDataSourceList", + "ExternalDataSourceUsability", + "ExternalDataSourceWrite", + "ExternalDataSourceWriteList", + "OneLakeCredentialsRead", + "OneLakeCredentialsWrite", + "OneLakeDataSourceSettingsRead", + "OneLakeDataSourceSettingsWrite", + "OneLakeLocationDescription", +] + + +class OneLakeLocationDescription(CogniteResource): + """Location of a Fabric OneLake lakehouse. + + Args: + workspace_name (str): Fabric workspace GUID or name. + container_name (str): Fabric lakehouse GUID or name. + """ + + def __init__(self, workspace_name: str, container_name: str) -> None: + self.workspace_name = workspace_name + self.container_name = container_name + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + workspace_name=resource["workspaceName"], + container_name=resource["containerName"], + ) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + if camel_case: + return {"workspaceName": self.workspace_name, "containerName": self.container_name} + return {"workspace_name": self.workspace_name, "container_name": self.container_name} + + +class OneLakeCredentialsRead(CogniteResource): + """Read-only view of Azure credentials for Fabric OneLake (clientSecret is never returned by the API). + + Args: + client_id (str): Azure application (client) ID. + tenant_id (str): Azure tenant (directory) ID. + """ + + def __init__(self, client_id: str, tenant_id: str) -> None: + self.client_id = client_id + self.tenant_id = tenant_id + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + client_id=resource["clientId"], + tenant_id=resource["tenantId"], + ) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + if camel_case: + return {"clientId": self.client_id, "tenantId": self.tenant_id} + return {"client_id": self.client_id, "tenant_id": self.tenant_id} + + +class OneLakeCredentialsWrite(CogniteResource): + """Azure credentials for writing to Fabric OneLake. + + Args: + client_id (str): Azure application (client) ID. + tenant_id (str): Azure tenant (directory) ID. + client_secret (str | None): Azure client secret. Required for upsert; None when reconstructed + from a read model via as_write() since the API never returns the secret. + """ + + def __init__(self, client_id: str, tenant_id: str, client_secret: str | None = None) -> None: + self.client_id = client_id + self.tenant_id = tenant_id + self.client_secret = client_secret + + def __repr__(self) -> str: + secret_display = "***" if self.client_secret is not None else None + return ( + f"OneLakeCredentialsWrite(client_id={self.client_id!r}, tenant_id={self.tenant_id!r}," + f" client_secret={secret_display!r})" + ) + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + client_id=resource["clientId"], + tenant_id=resource["tenantId"], + client_secret=resource.get("clientSecret"), + ) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result: dict[str, Any] + if camel_case: + result = {"clientId": self.client_id, "tenantId": self.tenant_id} + if self.client_secret is not None: + result["clientSecret"] = self.client_secret + else: + result = {"client_id": self.client_id, "tenant_id": self.tenant_id} + if self.client_secret is not None: + result["client_secret"] = self.client_secret + return result + + +class OneLakeDataSourceSettingsRead(CogniteResource): + """Settings for a Fabric OneLake external data source (read model — no client secret). + + Args: + credentials (OneLakeCredentialsRead | None): Azure credentials (client ID and tenant ID only). + location_description (OneLakeLocationDescription | None): Fabric workspace and lakehouse identifiers. + """ + + def __init__( + self, + credentials: OneLakeCredentialsRead | None = None, + location_description: OneLakeLocationDescription | None = None, + ) -> None: + self.credentials = credentials + self.location_description = location_description + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + credentials = None + if (creds_raw := resource.get("credentials")) is not None: + credentials = OneLakeCredentialsRead._load(creds_raw) + location_description = None + if (loc_raw := resource.get("locationDescription")) is not None: + location_description = OneLakeLocationDescription._load(loc_raw) + return cls(credentials=credentials, location_description=location_description) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result: dict[str, Any] = {} + if self.credentials is not None: + result["credentials" if not camel_case else "credentials"] = self.credentials.dump(camel_case=camel_case) + if self.location_description is not None: + key = "locationDescription" if camel_case else "location_description" + result[key] = self.location_description.dump(camel_case=camel_case) + return result + + +class OneLakeDataSourceSettingsWrite(CogniteResource): + """Settings for writing a Fabric OneLake external data source (includes client secret). + + Args: + credentials (OneLakeCredentialsWrite | None): Azure credentials including client secret. + location_description (OneLakeLocationDescription | None): Fabric workspace and lakehouse identifiers. + """ + + def __init__( + self, + credentials: OneLakeCredentialsWrite | None = None, + location_description: OneLakeLocationDescription | None = None, + ) -> None: + self.credentials = credentials + self.location_description = location_description + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + credentials = None + if (creds_raw := resource.get("credentials")) is not None: + credentials = OneLakeCredentialsWrite._load(creds_raw) + location_description = None + if (loc_raw := resource.get("locationDescription")) is not None: + location_description = OneLakeLocationDescription._load(loc_raw) + return cls(credentials=credentials, location_description=location_description) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result: dict[str, Any] = {} + if self.credentials is not None: + result["credentials"] = self.credentials.dump(camel_case=camel_case) + if self.location_description is not None: + key = "locationDescription" if camel_case else "location_description" + result[key] = self.location_description.dump(camel_case=camel_case) + return result + + +class ExternalDataSourceCore(WriteableCogniteResource["ExternalDataSourceWrite"], ABC): + """Shared base for ExternalDataSource (read) and ExternalDataSourceWrite (write). + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + Args: + external_id (str): External ID of the data source. Must be unique within the project. + name (str | None): Human-readable name for the data source. + data_set_id (int | None): ID of the data set that owns this resource (for ACL scoping). + """ + + _FORMAT: ClassVar[str] = "one_lake" + + def __init__( + self, + external_id: str, + name: str | None = None, + data_set_id: int | None = None, + ) -> None: + self.external_id = external_id + self.name = name + self.data_set_id = data_set_id + + +class ExternalDataSource(ExternalDataSourceCore): + """A Fabric OneLake external data source (read model — returned by list). + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + The ``clientSecret`` field is **never** returned by the API. + + Args: + external_id (str): External ID of the data source. + name (str | None): Human-readable name. + data_set_id (int | None): Data set ID for ACL scoping. + settings (OneLakeDataSourceSettingsRead | None): Connection settings (no client secret). + format (str | None): Backend format identifier (always ``"one_lake"`` for OneLake sources). + created_time (int | None): Time the resource was created (milliseconds since epoch). + last_updated_time (int | None): Time the resource was last updated (milliseconds since epoch). + """ + + def __init__( + self, + external_id: str, + name: str | None = None, + data_set_id: int | None = None, + settings: OneLakeDataSourceSettingsRead | None = None, + format: str | None = None, + created_time: int | None = None, + last_updated_time: int | None = None, + ) -> None: + super().__init__(external_id=external_id, name=name, data_set_id=data_set_id) + self.settings = settings + self.format = format + self.created_time = created_time + self.last_updated_time = last_updated_time + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + fmt = resource.get("format") + if fmt is not None and fmt != cls._FORMAT: + warnings.warn( + f"Unknown external data source format: {fmt!r}. This version of the SDK may not fully support it.", + UserWarning, + stacklevel=2, + ) + settings = None + if (settings_raw := resource.get("settings")) is not None: + settings = OneLakeDataSourceSettingsRead._load(settings_raw) + return cls( + external_id=resource["externalId"], + name=resource.get("name"), + data_set_id=resource.get("dataSetId"), + settings=settings, + format=fmt, + created_time=resource.get("createdTime"), + last_updated_time=resource.get("lastUpdatedTime"), + ) + + def as_write(self) -> ExternalDataSourceWrite: + """Return this source as an ExternalDataSourceWrite. + + Note: The ``client_secret`` cannot be reconstructed from the read model (the API never returns it). + The returned write object will have ``client_secret=None`` on its credentials. + """ + settings_write: OneLakeDataSourceSettingsWrite | None = None + if self.settings is not None: + creds_write: OneLakeCredentialsWrite | None = None + if self.settings.credentials is not None: + creds_write = OneLakeCredentialsWrite( + client_id=self.settings.credentials.client_id, + tenant_id=self.settings.credentials.tenant_id, + client_secret=None, + ) + settings_write = OneLakeDataSourceSettingsWrite( + credentials=creds_write, + location_description=self.settings.location_description, + ) + return ExternalDataSourceWrite( + external_id=self.external_id, + name=self.name, + data_set_id=self.data_set_id, + settings=settings_write, + ) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result: dict[str, Any] = {} + if camel_case: + result["externalId"] = self.external_id + if self.name is not None: + result["name"] = self.name + if self.data_set_id is not None: + result["dataSetId"] = self.data_set_id + if self.settings is not None: + result["settings"] = self.settings.dump(camel_case=True) + if self.format is not None: + result["format"] = self.format + if self.created_time is not None: + result["createdTime"] = self.created_time + if self.last_updated_time is not None: + result["lastUpdatedTime"] = self.last_updated_time + else: + result["external_id"] = self.external_id + if self.name is not None: + result["name"] = self.name + if self.data_set_id is not None: + result["data_set_id"] = self.data_set_id + if self.settings is not None: + result["settings"] = self.settings.dump(camel_case=False) + if self.format is not None: + result["format"] = self.format + if self.created_time is not None: + result["created_time"] = self.created_time + if self.last_updated_time is not None: + result["last_updated_time"] = self.last_updated_time + return result + + +class ExternalDataSourceWrite(ExternalDataSourceCore): + """A Fabric OneLake external data source (write model — used for upsert). + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + The ``format`` field is always ``"one_lake"`` and is injected automatically on serialization. + + Args: + external_id (str): External ID of the data source. + name (str | None): Human-readable name. + data_set_id (int | None): Data set ID for ACL scoping. + settings (OneLakeDataSourceSettingsWrite | None): Connection settings including client secret. + """ + + def __init__( + self, + external_id: str, + name: str | None = None, + data_set_id: int | None = None, + settings: OneLakeDataSourceSettingsWrite | None = None, + ) -> None: + super().__init__(external_id=external_id, name=name, data_set_id=data_set_id) + self.settings = settings + + @classmethod + def onelake( + cls, + external_id: str, + client_id: str, + tenant_id: str, + client_secret: str, + workspace_name: str, + container_name: str, + name: str | None = None, + data_set_id: int | None = None, + ) -> ExternalDataSourceWrite: + """Create an ExternalDataSourceWrite for a Fabric OneLake source. + + OneLake external data sources are **read-only** from a transform perspective — transforms can + read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. + + Args: + external_id (str): External ID for the data source. Must be unique. + client_id (str): Azure application (client) ID. + tenant_id (str): Azure tenant (directory) ID. + client_secret (str): Azure client secret. + workspace_name (str): Fabric workspace GUID or name. + container_name (str): Fabric lakehouse GUID or name. + name (str | None): Human-readable name. + data_set_id (int | None): Data set ID for ACL scoping. + + Returns: + ExternalDataSourceWrite: Ready to pass to ``client.transformations.external_data_sources.upsert()``. + + Examples: + + Register a Fabric OneLake source: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.transformations.external_data import ExternalDataSourceWrite + >>> client = CogniteClient() + >>> source = ExternalDataSourceWrite.onelake( + ... external_id="fabric-lakehouse-prod", + ... name="Production lakehouse", + ... client_id="", + ... tenant_id="", + ... client_secret="", + ... workspace_name="", + ... container_name="", + ... data_set_id=123456, + ... ) + >>> client.transformations.external_data_sources.upsert(source) + """ + return cls( + external_id=external_id, + name=name, + data_set_id=data_set_id, + settings=OneLakeDataSourceSettingsWrite( + credentials=OneLakeCredentialsWrite( + client_id=client_id, + tenant_id=tenant_id, + client_secret=client_secret, + ), + location_description=OneLakeLocationDescription( + workspace_name=workspace_name, + container_name=container_name, + ), + ), + ) + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + settings = None + if (settings_raw := resource.get("settings")) is not None: + settings = OneLakeDataSourceSettingsWrite._load(settings_raw) + return cls( + external_id=resource["externalId"], + name=resource.get("name"), + data_set_id=resource.get("dataSetId"), + settings=settings, + ) + + def as_write(self) -> ExternalDataSourceWrite: + return self + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result: dict[str, Any] = {} + if camel_case: + result["externalId"] = self.external_id + result["format"] = self._FORMAT + if self.name is not None: + result["name"] = self.name + if self.data_set_id is not None: + result["dataSetId"] = self.data_set_id + if self.settings is not None: + result["settings"] = self.settings.dump(camel_case=True) + else: + result["external_id"] = self.external_id + result["format"] = self._FORMAT + if self.name is not None: + result["name"] = self.name + if self.data_set_id is not None: + result["data_set_id"] = self.data_set_id + if self.settings is not None: + result["settings"] = self.settings.dump(camel_case=False) + return result + + +class ExternalDataSourceList(WriteableCogniteResourceList[ExternalDataSourceWrite, ExternalDataSource]): + """A list of ExternalDataSource (read model) objects.""" + + _RESOURCE = ExternalDataSource + + def as_write(self) -> ExternalDataSourceWriteList: + """Return all sources in their write format (client_secret will be None on each).""" + return ExternalDataSourceWriteList([item.as_write() for item in self.data]) + + +class ExternalDataSourceWriteList(CogniteResourceList[ExternalDataSourceWrite]): + """A list of ExternalDataSourceWrite objects.""" + + _RESOURCE = ExternalDataSourceWrite + + +class ExternalDataSourceUsability(CogniteResource): + """Result of verifying a Fabric OneLake external data source's usability. + + Args: + external_id (str | None): External ID of the verified data source. + usable_version (str | None): UUID indicating the data source is accessible and credentials are valid. + ``None`` if the source cannot be accessed. + """ + + def __init__(self, external_id: str | None = None, usable_version: str | None = None) -> None: + self.external_id = external_id + self.usable_version = usable_version + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + external_id=resource.get("externalId"), + usable_version=resource.get("usableVersion"), + ) + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + if camel_case: + return {"externalId": self.external_id, "usableVersion": self.usable_version} + return {"external_id": self.external_id, "usable_version": self.usable_version} diff --git a/cognite/client/testing.py b/cognite/client/testing.py index bb66f744fe..7ad558f0e1 100644 --- a/cognite/client/testing.py +++ b/cognite/client/testing.py @@ -78,6 +78,7 @@ from cognite.client._api.three_d.revisions import ThreeDRevisionsAPI from cognite.client._api.time_series import TimeSeriesAPI from cognite.client._api.transformations import TransformationsAPI +from cognite.client._api.transformations.external_data import TransformationExternalDataAPI from cognite.client._api.transformations.jobs import TransformationJobsAPI from cognite.client._api.transformations.notifications import TransformationNotificationsAPI from cognite.client._api.transformations.schedules import TransformationSchedulesAPI @@ -163,6 +164,7 @@ from cognite.client._sync_api.three_d.revisions import Sync3DRevisionsAPI from cognite.client._sync_api.time_series import SyncTimeSeriesAPI from cognite.client._sync_api.transformations import SyncTransformationsAPI +from cognite.client._sync_api.transformations.external_data import SyncTransformationExternalDataAPI from cognite.client._sync_api.transformations.jobs import SyncTransformationJobsAPI from cognite.client._sync_api.transformations.notifications import SyncTransformationNotificationsAPI from cognite.client._sync_api.transformations.schedules import SyncTransformationSchedulesAPI @@ -366,6 +368,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: tr_notifications = create_autospec(TransformationNotificationsAPI, instance=True, spec_set=True) tr_schedules = create_autospec(TransformationSchedulesAPI, instance=True, spec_set=True) tr_schema = create_autospec(TransformationSchemaAPI, instance=True, spec_set=True) + tr_external_data = create_autospec(TransformationExternalDataAPI, instance=True, spec_set=True) self.transformations = create_autospec( TransformationsAPI, instance=True, @@ -373,6 +376,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: notifications=tr_notifications, schedules=tr_schedules, schema=tr_schema, + external_data_sources=tr_external_data, ) flip_spec_set_on(self.transformations) @@ -571,6 +575,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: tr_notifications = create_autospec(SyncTransformationNotificationsAPI, instance=True, spec_set=True) tr_schedules = create_autospec(SyncTransformationSchedulesAPI, instance=True, spec_set=True) tr_schema = create_autospec(SyncTransformationSchemaAPI, instance=True, spec_set=True) + tr_external_data_sync = create_autospec(SyncTransformationExternalDataAPI, instance=True, spec_set=True) self.transformations = create_autospec( SyncTransformationsAPI, instance=True, @@ -578,6 +583,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: notifications=tr_notifications, schedules=tr_schedules, schema=tr_schema, + external_data_sources=tr_external_data_sync, ) flip_spec_set_on(self.transformations) diff --git a/docs/source/transformations.rst b/docs/source/transformations.rst index 8d2d3a5ec6..48b601fcbf 100644 --- a/docs/source/transformations.rst +++ b/docs/source/transformations.rst @@ -48,6 +48,14 @@ Transformation Schema AsyncCogniteClient.transformations.schema +External Data Sources +--------------------- + +.. toctree:: + :maxdepth: 2 + + transformations_external_data + Data classes ------------ .. automodule:: cognite.client.data_classes.transformations diff --git a/docs/source/transformations_external_data.rst b/docs/source/transformations_external_data.rst new file mode 100644 index 0000000000..7ab0b24851 --- /dev/null +++ b/docs/source/transformations_external_data.rst @@ -0,0 +1,142 @@ +External Data Sources +====================== + +.. currentmodule:: cognite.client + +.. warning:: + + **OneLake is read-only from a transform perspective.** Transforms can read data from Fabric OneLake tables + via ``ext_onelake()`` SQL, but writing back to OneLake is **not supported**. External data sources are + credentials and location information that transforms use to access OneLake tables — they are not + destinations for transform output. + +Introduction +------------ + +External data sources allow transformations to read data from Fabric OneLake by registering OneLake +workspace and lakehouse credentials. Once registered, a transform can access tables in OneLake via +the ``ext_onelake('source-id', 'table_name')`` SQL function. + +Each external data source is identified by a unique ``external_id`` and stores Azure service principal +credentials (client ID, tenant ID, and client secret) along with the target Fabric workspace and +lakehouse identifiers. + +Quickstart +---------- + +Register a Fabric OneLake Source +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Use the factory method ``ExternalDataSourceWrite.onelake()`` to create a source, then call ``upsert()`` +to register it in your project: + +.. code-block:: python + + from cognite.client import CogniteClient + from cognite.client.data_classes.transformations.external_data import ExternalDataSourceWrite + + client = CogniteClient() + + # Create the external data source + source = ExternalDataSourceWrite.onelake( + external_id="fabric-lakehouse-prod", + name="Production lakehouse", + client_id="", + tenant_id="", + client_secret="", + workspace_name="", + container_name="", + data_set_id=123456, + ) + + # Register the source + registered_source = client.transformations.external_data_sources.upsert(source) + print(f"Registered source: {registered_source.external_id}") + +Verify Source Usability +^^^^^^^^^^^^^^^^^^^^^^^ + +Before running a transform, verify that the source is accessible and credentials are valid: + +.. code-block:: python + + result = client.transformations.external_data_sources.verify_usability("fabric-lakehouse-prod") + + if result.usable_version is not None: + print("Source is accessible") + else: + print("Source cannot be accessed — check credentials") + +Create and Run a Transform Using OneLake Data +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Once the external data source is registered, use it in a transform SQL via the ``ext_onelake()`` function. +The transform can read from OneLake tables and write results to CDF: + +.. code-block:: python + + from cognite.client.data_classes.transformations import ( + Transformation, + TransformationDestination, + ) + + transform = Transformation( + external_id="onelake-to-cdf-assets", + name="OneLake to CDF Assets", + destination=TransformationDestination.assets(), + query=""" + SELECT + name, + description + FROM ext_onelake('fabric-lakehouse-prod', 'my_table') + WHERE active = true + """, + ) + + created = client.transformations.create(transform) + print(f"Created transform: {created.external_id}") + + # Run the transform + job = client.transformations.jobs.run(created.id) + print(f"Job {job.id} started") + +List Registered Sources +^^^^^^^^^^^^^^^^^^^^^^^ + +Retrieve all registered external data sources: + +.. code-block:: python + + sources = client.transformations.external_data_sources.list() + for source in sources: + print(f"Source: {source.external_id} ({source.name})") + +Delete a Source +^^^^^^^^^^^^^^^ + +Remove a source when it is no longer needed: + +.. code-block:: python + + client.transformations.external_data_sources.delete("fabric-lakehouse-prod") + print("Source deleted") + +API Reference +------------- + +TransformationExternalDataAPI +----------------------------- + +.. autosummary:: + :methods: + :toctree: generated/ + :template: custom-automethods-template.rst + + AsyncCogniteClient.transformations.external_data_sources + +Data Classes +------------ + +.. automodule:: cognite.client.data_classes.transformations.external_data + :members: + :show-inheritance: diff --git a/tests/tests_integration/test_api/test_transformations/test_external_data.py b/tests/tests_integration/test_api/test_transformations/test_external_data.py new file mode 100644 index 0000000000..4fdabcd640 --- /dev/null +++ b/tests/tests_integration/test_api/test_transformations/test_external_data.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from cognite.client import CogniteClient +from cognite.client.data_classes.transformations.external_data import ( + ExternalDataSource, + ExternalDataSourceList, + ExternalDataSourceUsability, + ExternalDataSourceWrite, +) + +_JETFIRE_ENV = Path(__file__).parents[5] / "jetfire-backend" / ".env" +_SKIP_REASON = f"Fabric integration env not available ({_JETFIRE_ENV})" + + +def _load_jetfire_env() -> None: + """Parse key=value pairs from the jetfire-backend .env file into os.environ.""" + for line in _JETFIRE_ENV.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + os.environ.setdefault(key.strip(), value.strip()) + + +@pytest.fixture(scope="module", autouse=True) +def load_fabric_env() -> None: + if _JETFIRE_ENV.exists(): + _load_jetfire_env() + + +@pytest.mark.skipif(not _JETFIRE_ENV.exists(), reason=_SKIP_REASON) +class TestExternalDataSourcesIntegration: + """End-to-end lifecycle test: upsert → list → verify_usability → delete.""" + + _EXTERNAL_ID = "sdk-integration-test-fabric-onelake" + + def test_lifecycle(self, cognite_client: CogniteClient) -> None: + client_id = os.environ["FABRIC_CLIENT_ID"] + tenant_id = os.environ["FABRIC_TENANT_ID"] + client_secret = os.environ["FABRIC_CLIENT_SECRET"] + workspace_name = os.environ["FABRIC_WORKSPACE_NAME"] + container_name = os.environ["FABRIC_CONTAINER_NAME"] + + source = ExternalDataSourceWrite.onelake( + external_id=self._EXTERNAL_ID, + client_id=client_id, + tenant_id=tenant_id, + client_secret=client_secret, + workspace_name=workspace_name, + container_name=container_name, + ) + + try: + # Upsert + upserted = cognite_client.transformations.external_data_sources.upsert(source) + assert isinstance(upserted, ExternalDataSource) + assert upserted.external_id == self._EXTERNAL_ID + assert upserted.format == "one_lake" + + # List — verify the upserted source is present + all_sources = cognite_client.transformations.external_data_sources.list() + assert isinstance(all_sources, ExternalDataSourceList) + external_ids = {s.external_id for s in all_sources} + assert self._EXTERNAL_ID in external_ids + + # Verify usability — expects valid credentials and reachable workspace + usability = cognite_client.transformations.external_data_sources.verify_usability(self._EXTERNAL_ID) + assert isinstance(usability, ExternalDataSourceUsability) + # usable_version is a UUID when the source is accessible; None when credentials are invalid + assert usability.usable_version is not None, ( + "verify_usability returned None — credentials may be invalid or workspace unreachable" + ) + + finally: + # Always delete to avoid leaving stale test resources + cognite_client.transformations.external_data_sources.delete(self._EXTERNAL_ID) diff --git a/tests/tests_unit/test_api/test_transformation_external_data.py b/tests/tests_unit/test_api/test_transformation_external_data.py new file mode 100644 index 0000000000..7204e663c8 --- /dev/null +++ b/tests/tests_unit/test_api/test_transformation_external_data.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +from collections.abc import Iterator + +import pytest +from pytest_httpx import HTTPXMock + +from cognite.client import AsyncCogniteClient, CogniteClient +from cognite.client.data_classes.transformations.external_data import ( + ExternalDataSource, + ExternalDataSourceList, + ExternalDataSourceUsability, + ExternalDataSourceWrite, +) +from tests.utils import get_url, jsgz_load + + +@pytest.fixture +def external_data_url(async_client: AsyncCogniteClient) -> str: + return get_url( + async_client.transformations.external_data_sources, + async_client.transformations.external_data_sources._RESOURCE_PATH, + ) + + +@pytest.fixture +def source_response_body() -> dict: + return { + "items": [ + { + "externalId": "x", + "format": "one_lake", + "settings": { + "credentials": {"clientId": "cid", "tenantId": "tid"}, + "locationDescription": {"workspaceName": "ws", "containerName": "cn"}, + }, + } + ] + } + + +@pytest.fixture +def mock_list_response( + httpx_mock: HTTPXMock, external_data_url: str, source_response_body: dict +) -> Iterator[HTTPXMock]: + httpx_mock.add_response(method="GET", url=external_data_url, status_code=200, json=source_response_body) + yield httpx_mock + + +@pytest.fixture +def mock_upsert_response( + httpx_mock: HTTPXMock, external_data_url: str, source_response_body: dict +) -> Iterator[HTTPXMock]: + httpx_mock.add_response(method="POST", url=external_data_url, status_code=200, json=source_response_body) + yield httpx_mock + + +@pytest.fixture +def mock_delete_response(httpx_mock: HTTPXMock, external_data_url: str) -> Iterator[HTTPXMock]: + httpx_mock.add_response(method="POST", url=external_data_url + "/delete", status_code=200, json={}) + yield httpx_mock + + +@pytest.fixture +def mock_verify_usability_response(httpx_mock: HTTPXMock, external_data_url: str) -> Iterator[HTTPXMock]: + httpx_mock.add_response( + method="POST", + url=external_data_url + "/usability", + status_code=200, + json={"externalId": "x", "usableVersion": "abc-uuid"}, + ) + yield httpx_mock + + +class TestTransformationExternalDataAPI: + @pytest.mark.usefixtures("mock_list_response") + def test_list(self, cognite_client: CogniteClient) -> None: + result = cognite_client.transformations.external_data_sources.list() + + assert isinstance(result, ExternalDataSourceList) + assert len(result) == 1 + assert result[0].external_id == "x" + assert result[0].format == "one_lake" + + def test_upsert_single( + self, cognite_client: CogniteClient, mock_upsert_response: HTTPXMock + ) -> None: + source = ExternalDataSourceWrite.onelake( + external_id="x", + client_id="cid", + tenant_id="tid", + client_secret="sec", + workspace_name="ws", + container_name="cn", + ) + result = cognite_client.transformations.external_data_sources.upsert(source) + + assert isinstance(result, ExternalDataSource) + assert result.external_id == "x" + # Verify the POST body contained "format": "one_lake" + request_body = jsgz_load(mock_upsert_response.get_requests()[-1].content) + assert request_body["items"][0]["format"] == "one_lake" + + def test_delete( + self, + cognite_client: CogniteClient, + async_client: AsyncCogniteClient, + mock_delete_response: HTTPXMock, + ) -> None: + cognite_client.transformations.external_data_sources.delete("x") + + last_request = mock_delete_response.get_requests()[-1] + url = str(last_request.url) + assert url.endswith(async_client.transformations.external_data_sources._RESOURCE_PATH + "/delete") + request_body = jsgz_load(last_request.content) + assert request_body["items"] == [{"externalId": "x"}] + + def test_verify_usability( + self, cognite_client: CogniteClient, mock_verify_usability_response: HTTPXMock + ) -> None: + result = cognite_client.transformations.external_data_sources.verify_usability("x") + + assert isinstance(result, ExternalDataSourceUsability) + assert result.usable_version == "abc-uuid" + assert result.external_id == "x" diff --git a/tests/tests_unit/test_data_classes/test_transformations/__init__.py b/tests/tests_unit/test_data_classes/test_transformations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py b/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py new file mode 100644 index 0000000000..38f66cb3cf --- /dev/null +++ b/tests/tests_unit/test_data_classes/test_transformations/test_external_data.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import pytest + +from cognite.client.data_classes.transformations.external_data import ( + ExternalDataSource, + ExternalDataSourceUsability, + ExternalDataSourceWrite, + OneLakeCredentialsWrite, +) + + +def test_onelake_factory_produces_valid_structure() -> None: + source = ExternalDataSourceWrite.onelake( + external_id="x", + client_id="cid", + tenant_id="tid", + client_secret="sec", + workspace_name="ws", + container_name="cn", + ) + dumped = source.dump(camel_case=True) + + assert dumped["format"] == "one_lake" + assert dumped["externalId"] == "x" + assert dumped["settings"]["credentials"]["clientId"] == "cid" + assert dumped["settings"]["credentials"]["clientSecret"] == "sec" + assert dumped["settings"]["locationDescription"]["workspaceName"] == "ws" + + +def test_write_dump_always_includes_format() -> None: + source = ExternalDataSourceWrite(external_id="x") + dumped = source.dump(camel_case=True) + + assert "format" in dumped + assert dumped["format"] == "one_lake" + + +def test_read_load_parses_settings() -> None: + raw = { + "externalId": "x", + "format": "one_lake", + "settings": { + "credentials": {"clientId": "cid", "tenantId": "tid"}, + "locationDescription": {"workspaceName": "ws", "containerName": "cn"}, + }, + } + source = ExternalDataSource._load(raw) + + assert source.external_id == "x" + assert source.format == "one_lake" + assert source.settings is not None + assert source.settings.credentials is not None + assert source.settings.credentials.client_id == "cid" + assert source.settings.credentials.tenant_id == "tid" + assert source.settings.location_description is not None + assert source.settings.location_description.workspace_name == "ws" + assert source.settings.location_description.container_name == "cn" + # Read model never carries client_secret + assert not hasattr(source.settings.credentials, "client_secret") + + +def test_read_load_unknown_format_warns_not_raises() -> None: + raw = {"externalId": "x", "format": "delta_sharing", "settings": {}} + + with pytest.warns(UserWarning, match="Unknown external data source format"): + source = ExternalDataSource._load(raw) + + # Must not raise — the object is returned despite the unknown format + assert source.external_id == "x" + assert source.format == "delta_sharing" + + +def test_as_write_returns_write_with_none_secret() -> None: + raw = { + "externalId": "x", + "format": "one_lake", + "settings": { + "credentials": {"clientId": "cid", "tenantId": "tid"}, + "locationDescription": {"workspaceName": "ws", "containerName": "cn"}, + }, + } + read_source = ExternalDataSource._load(raw) + write_source = read_source.as_write() + + assert isinstance(write_source, ExternalDataSourceWrite) + assert write_source.settings is not None + assert write_source.settings.credentials is not None + assert write_source.settings.credentials.client_secret is None + + +def test_credentials_write_repr_masks_secret() -> None: + creds = OneLakeCredentialsWrite("cid", "tid", "actual-secret") + result = repr(creds) + + assert "actual-secret" not in result + assert "***" in result + + +def test_usability_load_with_version() -> None: + usability = ExternalDataSourceUsability._load({"externalId": "x", "usableVersion": "some-uuid"}) + + assert usability.external_id == "x" + assert usability.usable_version == "some-uuid" + + +def test_usability_load_with_null_version() -> None: + usability = ExternalDataSourceUsability._load({"externalId": "x", "usableVersion": None}) + + assert usability.external_id == "x" + assert usability.usable_version is None From b779e8c545c29441b03dd8fccbcaf4e95d5fd34d Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Thu, 9 Jul 2026 09:42:43 +0530 Subject: [PATCH 2/5] feat(transformations): add USE action to TransformationsExternalDataSourcesAcl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /usability endpoint requires USE ACL on the backend — grants Spark driver access to verify credentials without granting full WRITE. --- cognite/client/data_classes/capabilities.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cognite/client/data_classes/capabilities.py b/cognite/client/data_classes/capabilities.py index 1dfd2103f8..529a89a1aa 100644 --- a/cognite/client/data_classes/capabilities.py +++ b/cognite/client/data_classes/capabilities.py @@ -1113,6 +1113,7 @@ class TransformationsExternalDataSourcesAcl(Capability): class Action(Capability.Action): # type: ignore [misc] Read = "READ" Write = "WRITE" + Use = "USE" class Scope: All = AllScope From 008a7368fedc38c19c36fed7bf6cceb59b11af0a Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Thu, 9 Jul 2026 10:07:38 +0530 Subject: [PATCH 3/5] fix(transformations): repair external data sources CI failures Align sync codegen hashes, docs stubs, list mixins, and HTTP mocks with jetfire external_data API behavior; fix Fabric integration env var names. Co-authored-by: Cursor --- .../_api/transformations/external_data.py | 9 +++-- cognite/client/_cognite_client.py | 2 ++ .../_sync_api/transformations/__init__.py | 2 +- .../transformations/external_data.py | 33 ++++++++++++------- .../transformations/external_data.py | 9 +++-- docs/source/transformations.rst | 3 ++ .../test_external_data.py | 22 +++++++++++-- .../test_transformation_external_data.py | 9 +++-- 8 files changed, 63 insertions(+), 26 deletions(-) diff --git a/cognite/client/_api/transformations/external_data.py b/cognite/client/_api/transformations/external_data.py index 8c921ca164..d8786f0b87 100644 --- a/cognite/client/_api/transformations/external_data.py +++ b/cognite/client/_api/transformations/external_data.py @@ -8,7 +8,6 @@ ExternalDataSourceList, ExternalDataSourceUsability, ExternalDataSourceWrite, - ExternalDataSourceWriteList, ) from cognite.client.utils._identifier import IdentifierSequence from cognite.client.utils.useful_types import SequenceNotStr @@ -40,7 +39,7 @@ async def list(self, limit: int | None = None) -> ExternalDataSourceList: Examples: - List all registered external data sources:: + List all registered external data sources: >>> from cognite.client import CogniteClient >>> client = CogniteClient() @@ -105,13 +104,13 @@ async def delete(self, external_id: str | SequenceNotStr[str]) -> None: Examples: - Delete a source by external ID:: + Delete a source by external ID: >>> from cognite.client import CogniteClient >>> client = CogniteClient() >>> client.transformations.external_data_sources.delete("fabric-lakehouse-prod") - Delete multiple sources:: + Delete multiple sources: >>> client.transformations.external_data_sources.delete( ... ["fabric-lakehouse-prod", "fabric-lakehouse-staging"] @@ -140,7 +139,7 @@ async def verify_usability(self, external_id: str) -> ExternalDataSourceUsabilit Examples: - Verify a source before running a transformation:: + Verify a source before running a transformation: >>> from cognite.client import CogniteClient >>> client = CogniteClient() diff --git a/cognite/client/_cognite_client.py b/cognite/client/_cognite_client.py index 8720fa9507..11f1b1fe05 100644 --- a/cognite/client/_cognite_client.py +++ b/cognite/client/_cognite_client.py @@ -97,6 +97,7 @@ ThreeDRevisionsAPI, ) from cognite.client._api.transformations import ( # type: ignore[attr-defined] + TransformationExternalDataAPI, TransformationJobsAPI, TransformationNotificationsAPI, TransformationSchedulesAPI, @@ -425,6 +426,7 @@ def _make_accessors_for_building_docs() -> None: AsyncCogniteClient.transformations.notifications = TransformationNotificationsAPI # type: ignore AsyncCogniteClient.transformations.jobs = TransformationJobsAPI # type: ignore AsyncCogniteClient.transformations.schema = TransformationSchemaAPI # type: ignore + AsyncCogniteClient.transformations.external_data_sources = TransformationExternalDataAPI # type: ignore AsyncCogniteClient.diagrams = DiagramsAPI # type: ignore AsyncCogniteClient.annotations = AnnotationsAPI # type: ignore AsyncCogniteClient.functions = FunctionsAPI # type: ignore diff --git a/cognite/client/_sync_api/transformations/__init__.py b/cognite/client/_sync_api/transformations/__init__.py index 660099e4f6..0ab16f22a9 100644 --- a/cognite/client/_sync_api/transformations/__init__.py +++ b/cognite/client/_sync_api/transformations/__init__.py @@ -1,6 +1,6 @@ """ =============================================================================== -8bca411648596701cde79f2c9e6f2a88 +e545d766fa2f6a8b3d5946accd494c89 This file is auto-generated from the Async API modules, - do not edit manually! =============================================================================== """ diff --git a/cognite/client/_sync_api/transformations/external_data.py b/cognite/client/_sync_api/transformations/external_data.py index d10cf6fc65..1c8d2049b8 100644 --- a/cognite/client/_sync_api/transformations/external_data.py +++ b/cognite/client/_sync_api/transformations/external_data.py @@ -1,7 +1,7 @@ """ =============================================================================== -This file mirrors cognite/client/_api/transformations/external_data.py. -If the async API changes, update this file manually to match. +92307f7e19fa2b08989c01a0bba703a3 +This file is auto-generated from the Async API modules, - do not edit manually! =============================================================================== """ @@ -22,13 +22,14 @@ class SyncTransformationExternalDataAPI(SyncAPIClient): - """Sync wrapper for TransformationExternalDataAPI.""" + """Auto-generated, do not modify manually.""" def __init__(self, async_client: AsyncCogniteClient) -> None: self.__async_client = async_client def list(self, limit: int | None = None) -> ExternalDataSourceList: - """List Fabric OneLake external data sources. + """ + `List Fabric OneLake external data sources `_. OneLake external data sources are **read-only** from a transform perspective — transforms can read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. @@ -42,7 +43,7 @@ def list(self, limit: int | None = None) -> ExternalDataSourceList: Examples: - List all registered external data sources:: + List all registered external data sources: >>> from cognite.client import CogniteClient >>> client = CogniteClient() @@ -51,10 +52,10 @@ def list(self, limit: int | None = None) -> ExternalDataSourceList: return run_sync(self.__async_client.transformations.external_data_sources.list(limit=limit)) def upsert( - self, - source: ExternalDataSourceWrite | Sequence[ExternalDataSourceWrite], + self, source: ExternalDataSourceWrite | Sequence[ExternalDataSourceWrite] ) -> ExternalDataSource | ExternalDataSourceList: - """Create or update (upsert) Fabric OneLake external data sources. + """ + `Create or update (upsert) Fabric OneLake external data sources `_. An upsert creates the source if it doesn't exist, or overwrites it entirely if it does. Uniqueness is determined by ``externalId``. @@ -90,25 +91,33 @@ def upsert( return run_sync(self.__async_client.transformations.external_data_sources.upsert(source=source)) def delete(self, external_id: str | SequenceNotStr[str]) -> None: - """Delete Fabric OneLake external data sources. + """ + `Delete Fabric OneLake external data sources `_. Args: external_id (str | SequenceNotStr[str]): External ID or list of external IDs to delete. Examples: - Delete a source by external ID:: + Delete a source by external ID: >>> from cognite.client import CogniteClient >>> client = CogniteClient() >>> client.transformations.external_data_sources.delete("fabric-lakehouse-prod") + + Delete multiple sources: + + >>> client.transformations.external_data_sources.delete( + ... ["fabric-lakehouse-prod", "fabric-lakehouse-staging"] + ... ) """ return run_sync( self.__async_client.transformations.external_data_sources.delete(external_id=external_id) ) def verify_usability(self, external_id: str) -> ExternalDataSourceUsability: - """Verify that a Fabric OneLake external data source is usable. + """ + `Verify that a Fabric OneLake external data source is usable `_. Checks that the source exists and that the configured Azure credentials can access the specified Fabric lakehouse. Returns a ``usable_version`` UUID if the source is accessible. @@ -125,7 +134,7 @@ def verify_usability(self, external_id: str) -> ExternalDataSourceUsability: Examples: - Verify a source before running a transformation:: + Verify a source before running a transformation: >>> from cognite.client import CogniteClient >>> client = CogniteClient() diff --git a/cognite/client/data_classes/transformations/external_data.py b/cognite/client/data_classes/transformations/external_data.py index 792b3b0a1c..4c537c2e1c 100644 --- a/cognite/client/data_classes/transformations/external_data.py +++ b/cognite/client/data_classes/transformations/external_data.py @@ -9,6 +9,7 @@ from cognite.client.data_classes._base import ( CogniteResource, CogniteResourceList, + ExternalIDTransformerMixin, WriteableCogniteResource, WriteableCogniteResourceList, ) @@ -150,7 +151,7 @@ def _load(cls, resource: dict[str, Any]) -> Self: def dump(self, camel_case: bool = True) -> dict[str, Any]: result: dict[str, Any] = {} if self.credentials is not None: - result["credentials" if not camel_case else "credentials"] = self.credentials.dump(camel_case=camel_case) + result["credentials"] = self.credentials.dump(camel_case=camel_case) if self.location_description is not None: key = "locationDescription" if camel_case else "location_description" result[key] = self.location_description.dump(camel_case=camel_case) @@ -462,7 +463,9 @@ def dump(self, camel_case: bool = True) -> dict[str, Any]: return result -class ExternalDataSourceList(WriteableCogniteResourceList[ExternalDataSourceWrite, ExternalDataSource]): +class ExternalDataSourceList( + WriteableCogniteResourceList[ExternalDataSourceWrite, ExternalDataSource], ExternalIDTransformerMixin +): """A list of ExternalDataSource (read model) objects.""" _RESOURCE = ExternalDataSource @@ -472,7 +475,7 @@ def as_write(self) -> ExternalDataSourceWriteList: return ExternalDataSourceWriteList([item.as_write() for item in self.data]) -class ExternalDataSourceWriteList(CogniteResourceList[ExternalDataSourceWrite]): +class ExternalDataSourceWriteList(CogniteResourceList[ExternalDataSourceWrite], ExternalIDTransformerMixin): """A list of ExternalDataSourceWrite objects.""" _RESOURCE = ExternalDataSourceWrite diff --git a/docs/source/transformations.rst b/docs/source/transformations.rst index 48b601fcbf..25bde04ec4 100644 --- a/docs/source/transformations.rst +++ b/docs/source/transformations.rst @@ -76,3 +76,6 @@ Data classes .. automodule:: cognite.client.data_classes.transformations.common :members: :show-inheritance: +.. automodule:: cognite.client.data_classes.transformations.external_data + :members: + :show-inheritance: diff --git a/tests/tests_integration/test_api/test_transformations/test_external_data.py b/tests/tests_integration/test_api/test_transformations/test_external_data.py index 4fdabcd640..20d60925d2 100644 --- a/tests/tests_integration/test_api/test_transformations/test_external_data.py +++ b/tests/tests_integration/test_api/test_transformations/test_external_data.py @@ -33,7 +33,23 @@ def load_fabric_env() -> None: _load_jetfire_env() -@pytest.mark.skipif(not _JETFIRE_ENV.exists(), reason=_SKIP_REASON) +_FABRIC_ENV_VARS = ( + "FABRIC_CLIENT_ID", + "FABRIC_TENANT_ID", + "FABRIC_CLIENT_SECRET", + "FABRIC_WORKSPACE", + "FABRIC_LAKEHOUSE", +) + + +def _fabric_ci_available() -> bool: + if not _JETFIRE_ENV.exists(): + return False + _load_jetfire_env() + return all(os.environ.get(key) for key in _FABRIC_ENV_VARS) + + +@pytest.mark.skipif(not _fabric_ci_available(), reason=_SKIP_REASON) class TestExternalDataSourcesIntegration: """End-to-end lifecycle test: upsert → list → verify_usability → delete.""" @@ -43,8 +59,8 @@ def test_lifecycle(self, cognite_client: CogniteClient) -> None: client_id = os.environ["FABRIC_CLIENT_ID"] tenant_id = os.environ["FABRIC_TENANT_ID"] client_secret = os.environ["FABRIC_CLIENT_SECRET"] - workspace_name = os.environ["FABRIC_WORKSPACE_NAME"] - container_name = os.environ["FABRIC_CONTAINER_NAME"] + workspace_name = os.environ["FABRIC_WORKSPACE"] + container_name = os.environ["FABRIC_LAKEHOUSE"] source = ExternalDataSourceWrite.onelake( external_id=self._EXTERNAL_ID, diff --git a/tests/tests_unit/test_api/test_transformation_external_data.py b/tests/tests_unit/test_api/test_transformation_external_data.py index 7204e663c8..dff5b48fc5 100644 --- a/tests/tests_unit/test_api/test_transformation_external_data.py +++ b/tests/tests_unit/test_api/test_transformation_external_data.py @@ -43,7 +43,12 @@ def source_response_body() -> dict: def mock_list_response( httpx_mock: HTTPXMock, external_data_url: str, source_response_body: dict ) -> Iterator[HTTPXMock]: - httpx_mock.add_response(method="GET", url=external_data_url, status_code=200, json=source_response_body) + httpx_mock.add_response( + method="GET", + url=external_data_url + "?limit=1000", + status_code=200, + json=source_response_body, + ) yield httpx_mock @@ -51,7 +56,7 @@ def mock_list_response( def mock_upsert_response( httpx_mock: HTTPXMock, external_data_url: str, source_response_body: dict ) -> Iterator[HTTPXMock]: - httpx_mock.add_response(method="POST", url=external_data_url, status_code=200, json=source_response_body) + httpx_mock.add_response(method="POST", url=external_data_url, status_code=201, json=source_response_body) yield httpx_mock From 0af324fee7a6c16127b703f452a794fcd8fb0110 Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Thu, 9 Jul 2026 10:22:00 +0530 Subject: [PATCH 4/5] fix(transformations): fix lint and docs linkcheck for external data Remove unpublished api-docs URLs from docstrings, apply ruff formatting, reorder data class exports, and refresh sync codegen hash. Co-authored-by: Cursor --- .../_api/transformations/external_data.py | 22 +++++++----- .../transformations/external_data.py | 26 ++++++++------ cognite/client/data_classes/__init__.py | 36 +++++++++---------- .../transformations/external_data.py | 4 ++- .../test_transformation_external_data.py | 8 ++--- 5 files changed, 52 insertions(+), 44 deletions(-) diff --git a/cognite/client/_api/transformations/external_data.py b/cognite/client/_api/transformations/external_data.py index d8786f0b87..bb4f482cbc 100644 --- a/cognite/client/_api/transformations/external_data.py +++ b/cognite/client/_api/transformations/external_data.py @@ -14,7 +14,7 @@ class TransformationExternalDataAPI(APIClient): - """`Manage Fabric OneLake external data sources for transformations `_. + """Manage Fabric OneLake external data sources for transformations. OneLake external data sources are **read-only** from a transform perspective — transforms can read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. @@ -25,7 +25,7 @@ class TransformationExternalDataAPI(APIClient): _DELETE_LIMIT = 1000 async def list(self, limit: int | None = None) -> ExternalDataSourceList: - """`List Fabric OneLake external data sources `_. + """List Fabric OneLake external data sources. OneLake external data sources are **read-only** from a transform perspective — transforms can read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. @@ -56,7 +56,7 @@ async def upsert( self, source: ExternalDataSourceWrite | Sequence[ExternalDataSourceWrite], ) -> ExternalDataSource | ExternalDataSourceList: - """`Create or update (upsert) Fabric OneLake external data sources `_. + """Create or update (upsert) Fabric OneLake external data sources. An upsert creates the source if it doesn't exist, or overwrites it entirely if it does. Uniqueness is determined by ``externalId``. @@ -75,7 +75,9 @@ async def upsert( Register a Fabric OneLake source: >>> from cognite.client import CogniteClient - >>> from cognite.client.data_classes.transformations.external_data import ExternalDataSourceWrite + >>> from cognite.client.data_classes.transformations.external_data import ( + ... ExternalDataSourceWrite, + ... ) >>> client = CogniteClient() >>> source = ExternalDataSourceWrite.onelake( ... external_id="fabric-lakehouse-prod", @@ -97,7 +99,7 @@ async def upsert( ) async def delete(self, external_id: str | SequenceNotStr[str]) -> None: - """`Delete Fabric OneLake external data sources `_. + """Delete Fabric OneLake external data sources. Args: external_id (str | SequenceNotStr[str]): External ID or list of external IDs to delete. @@ -122,7 +124,7 @@ async def delete(self, external_id: str | SequenceNotStr[str]) -> None: ) async def verify_usability(self, external_id: str) -> ExternalDataSourceUsability: - """`Verify that a Fabric OneLake external data source is usable `_. + """Verify that a Fabric OneLake external data source is usable. Checks that the source exists and that the configured Azure credentials can access the specified Fabric lakehouse. Returns a ``usable_version`` UUID if the source is accessible. @@ -143,8 +145,12 @@ async def verify_usability(self, external_id: str) -> ExternalDataSourceUsabilit >>> from cognite.client import CogniteClient >>> client = CogniteClient() - >>> result = client.transformations.external_data_sources.verify_usability("fabric-lakehouse-prod") - >>> assert result.usable_version is not None, "Source not configured or credentials invalid" + >>> result = client.transformations.external_data_sources.verify_usability( + ... "fabric-lakehouse-prod" + ... ) + >>> assert result.usable_version is not None, ( + ... "Source not configured or credentials invalid" + ... ) """ res = await self._post( url_path=self._RESOURCE_PATH + "/usability", diff --git a/cognite/client/_sync_api/transformations/external_data.py b/cognite/client/_sync_api/transformations/external_data.py index 1c8d2049b8..5ec963f7d6 100644 --- a/cognite/client/_sync_api/transformations/external_data.py +++ b/cognite/client/_sync_api/transformations/external_data.py @@ -1,6 +1,6 @@ """ =============================================================================== -92307f7e19fa2b08989c01a0bba703a3 +53ec7cc4c0a0655facc509b38fedaff9 This file is auto-generated from the Async API modules, - do not edit manually! =============================================================================== """ @@ -29,7 +29,7 @@ def __init__(self, async_client: AsyncCogniteClient) -> None: def list(self, limit: int | None = None) -> ExternalDataSourceList: """ - `List Fabric OneLake external data sources `_. + List Fabric OneLake external data sources. OneLake external data sources are **read-only** from a transform perspective — transforms can read data from OneLake tables via ``ext_onelake()`` SQL, but writing to OneLake is not supported. @@ -55,7 +55,7 @@ def upsert( self, source: ExternalDataSourceWrite | Sequence[ExternalDataSourceWrite] ) -> ExternalDataSource | ExternalDataSourceList: """ - `Create or update (upsert) Fabric OneLake external data sources `_. + Create or update (upsert) Fabric OneLake external data sources. An upsert creates the source if it doesn't exist, or overwrites it entirely if it does. Uniqueness is determined by ``externalId``. @@ -74,7 +74,9 @@ def upsert( Register a Fabric OneLake source: >>> from cognite.client import CogniteClient - >>> from cognite.client.data_classes.transformations.external_data import ExternalDataSourceWrite + >>> from cognite.client.data_classes.transformations.external_data import ( + ... ExternalDataSourceWrite, + ... ) >>> client = CogniteClient() >>> source = ExternalDataSourceWrite.onelake( ... external_id="fabric-lakehouse-prod", @@ -92,7 +94,7 @@ def upsert( def delete(self, external_id: str | SequenceNotStr[str]) -> None: """ - `Delete Fabric OneLake external data sources `_. + Delete Fabric OneLake external data sources. Args: external_id (str | SequenceNotStr[str]): External ID or list of external IDs to delete. @@ -111,13 +113,11 @@ def delete(self, external_id: str | SequenceNotStr[str]) -> None: ... ["fabric-lakehouse-prod", "fabric-lakehouse-staging"] ... ) """ - return run_sync( - self.__async_client.transformations.external_data_sources.delete(external_id=external_id) - ) + return run_sync(self.__async_client.transformations.external_data_sources.delete(external_id=external_id)) def verify_usability(self, external_id: str) -> ExternalDataSourceUsability: """ - `Verify that a Fabric OneLake external data source is usable `_. + Verify that a Fabric OneLake external data source is usable. Checks that the source exists and that the configured Azure credentials can access the specified Fabric lakehouse. Returns a ``usable_version`` UUID if the source is accessible. @@ -138,8 +138,12 @@ def verify_usability(self, external_id: str) -> ExternalDataSourceUsability: >>> from cognite.client import CogniteClient >>> client = CogniteClient() - >>> result = client.transformations.external_data_sources.verify_usability("fabric-lakehouse-prod") - >>> assert result.usable_version is not None, "Source not configured or credentials invalid" + >>> result = client.transformations.external_data_sources.verify_usability( + ... "fabric-lakehouse-prod" + ... ) + >>> assert result.usable_version is not None, ( + ... "Source not configured or credentials invalid" + ... ) """ return run_sync( self.__async_client.transformations.external_data_sources.verify_usability(external_id=external_id) diff --git a/cognite/client/data_classes/__init__.py b/cognite/client/data_classes/__init__.py index b1bef372ed..8a722ff9f0 100644 --- a/cognite/client/data_classes/__init__.py +++ b/cognite/client/data_classes/__init__.py @@ -251,14 +251,6 @@ TransformationBlockedInfo, TransformationDestination, ) -from cognite.client.data_classes.transformations.jobs import ( - TransformationJob, - TransformationJobFilter, - TransformationJobList, - TransformationJobMetric, - TransformationJobMetricList, - TransformationJobStatus, -) from cognite.client.data_classes.transformations.external_data import ( ExternalDataSource, ExternalDataSourceList, @@ -271,6 +263,14 @@ OneLakeDataSourceSettingsWrite, OneLakeLocationDescription, ) +from cognite.client.data_classes.transformations.jobs import ( + TransformationJob, + TransformationJobFilter, + TransformationJobList, + TransformationJobMetric, + TransformationJobMetricList, + TransformationJobStatus, +) from cognite.client.data_classes.transformations.notifications import ( TransformationNotification, TransformationNotificationList, @@ -396,6 +396,11 @@ "EventUpdate", "EventWrite", "EventWriteList", + "ExternalDataSource", + "ExternalDataSourceList", + "ExternalDataSourceUsability", + "ExternalDataSourceWrite", + "ExternalDataSourceWriteList", "ExtractionPipeline", "ExtractionPipelineConfig", "ExtractionPipelineConfigRevision", @@ -468,6 +473,11 @@ "Limit", "LimitList", "OidcCredentials", + "OneLakeCredentialsRead", + "OneLakeCredentialsWrite", + "OneLakeDataSourceSettingsRead", + "OneLakeDataSourceSettingsWrite", + "OneLakeLocationDescription", "RawTable", "RecordId", "Relationship", @@ -538,16 +548,6 @@ "TimeSeriesWriteList", "TimestampRange", "Transformation", - "ExternalDataSource", - "ExternalDataSourceList", - "ExternalDataSourceUsability", - "ExternalDataSourceWrite", - "ExternalDataSourceWriteList", - "OneLakeCredentialsRead", - "OneLakeCredentialsWrite", - "OneLakeDataSourceSettingsRead", - "OneLakeDataSourceSettingsWrite", - "OneLakeLocationDescription", "TransformationBlockedInfo", "TransformationDestination", "TransformationJob", diff --git a/cognite/client/data_classes/transformations/external_data.py b/cognite/client/data_classes/transformations/external_data.py index 4c537c2e1c..c70997ffd5 100644 --- a/cognite/client/data_classes/transformations/external_data.py +++ b/cognite/client/data_classes/transformations/external_data.py @@ -394,7 +394,9 @@ def onelake( Register a Fabric OneLake source: >>> from cognite.client import CogniteClient - >>> from cognite.client.data_classes.transformations.external_data import ExternalDataSourceWrite + >>> from cognite.client.data_classes.transformations.external_data import ( + ... ExternalDataSourceWrite, + ... ) >>> client = CogniteClient() >>> source = ExternalDataSourceWrite.onelake( ... external_id="fabric-lakehouse-prod", diff --git a/tests/tests_unit/test_api/test_transformation_external_data.py b/tests/tests_unit/test_api/test_transformation_external_data.py index dff5b48fc5..8ab668e99d 100644 --- a/tests/tests_unit/test_api/test_transformation_external_data.py +++ b/tests/tests_unit/test_api/test_transformation_external_data.py @@ -87,9 +87,7 @@ def test_list(self, cognite_client: CogniteClient) -> None: assert result[0].external_id == "x" assert result[0].format == "one_lake" - def test_upsert_single( - self, cognite_client: CogniteClient, mock_upsert_response: HTTPXMock - ) -> None: + def test_upsert_single(self, cognite_client: CogniteClient, mock_upsert_response: HTTPXMock) -> None: source = ExternalDataSourceWrite.onelake( external_id="x", client_id="cid", @@ -120,9 +118,7 @@ def test_delete( request_body = jsgz_load(last_request.content) assert request_body["items"] == [{"externalId": "x"}] - def test_verify_usability( - self, cognite_client: CogniteClient, mock_verify_usability_response: HTTPXMock - ) -> None: + def test_verify_usability(self, cognite_client: CogniteClient, mock_verify_usability_response: HTTPXMock) -> None: result = cognite_client.transformations.external_data_sources.verify_usability("x") assert isinstance(result, ExternalDataSourceUsability) From 8581886bafe7038b661032e5f46c798e73daf796 Mon Sep 17 00:00:00 2001 From: Khyat-Cognite Date: Thu, 9 Jul 2026 11:39:29 +0530 Subject: [PATCH 5/5] fix(docs): drop duplicate external_data automodule entry External data classes are documented on transformations_external_data.rst; keeping the automodule in transformations.rst caused 23 Sphinx duplicate object warnings that fail linkcheck. Co-authored-by: Cursor --- docs/source/transformations.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/source/transformations.rst b/docs/source/transformations.rst index 25bde04ec4..48b601fcbf 100644 --- a/docs/source/transformations.rst +++ b/docs/source/transformations.rst @@ -76,6 +76,3 @@ Data classes .. automodule:: cognite.client.data_classes.transformations.common :members: :show-inheritance: -.. automodule:: cognite.client.data_classes.transformations.external_data - :members: - :show-inheritance: