From b8ff5162d787a173c0a8739fc3be1090964063bf Mon Sep 17 00:00:00 2001 From: Edwin Pavlovsky Date: Mon, 23 Feb 2026 15:50:34 -0500 Subject: [PATCH 1/3] Enable deduplication in nucleus sdk --- CHANGELOG.md | 27 +++++++++++ nucleus/__init__.py | 3 ++ nucleus/constants.py | 1 + nucleus/dataset.py | 102 +++++++++++++++++++++++++++++++++++++++ nucleus/deduplication.py | 16 ++++++ pyproject.toml | 2 +- 6 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 nucleus/deduplication.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 29f59d10..1a4305fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,33 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.17.12](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.17.12) - 2026-02-23 + +### Added +- `Dataset.deduplicate()` method to deduplicate images using perceptual hashing. Accepts optional `reference_ids` to deduplicate specific items, or deduplicates the entire dataset when only `threshold` is provided. Required `threshold` parameter (0-64) controls similarity matching (lower = stricter, 0 = exact matches only). +- `Dataset.deduplicate_by_ids()` method for deduplication using internal `dataset_item_ids` directly, avoiding the reference ID to item ID mapping for improved efficiency. +- `DeduplicationResult` and `DeduplicationStats` dataclasses for structured deduplication results. + +Example usage: + +```python +dataset = client.get_dataset("ds_...") + +# Deduplicate entire dataset +result = dataset.deduplicate(threshold=10) + +# Deduplicate specific items by reference IDs +result = dataset.deduplicate(threshold=10, reference_ids=["ref_1", "ref_2", "ref_3"]) + +# Deduplicate by internal item IDs (more efficient if you have them) +result = dataset.deduplicate_by_ids(threshold=10, dataset_item_ids=["item_1", "item_2"]) + +# Access results +print(f"Threshold: {result.stats.threshold}") +print(f"Original: {result.stats.original_count}, Unique: {result.stats.deduplicated_count}") +print(result.unique_reference_ids) +``` + ## [0.17.11](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.17.11) - 2025-11-03 ### Added diff --git a/nucleus/__init__.py b/nucleus/__init__.py index 3f970c2b..df97ddec 100644 --- a/nucleus/__init__.py +++ b/nucleus/__init__.py @@ -4,6 +4,8 @@ "AsyncJob", "EmbeddingsExportJob", "BoxAnnotation", + "DeduplicationResult", + "DeduplicationStats", "BoxPrediction", "CameraParams", "CategoryAnnotation", @@ -128,6 +130,7 @@ from .data_transfer_object.job_status import JobInfoRequestPayload from .dataset import Dataset from .dataset_item import DatasetItem +from .deduplication import DeduplicationResult, DeduplicationStats from .deprecation_warning import deprecated from .errors import ( DatasetItemRetrievalError, diff --git a/nucleus/constants.py b/nucleus/constants.py index 0a2bbf46..ebad94f5 100644 --- a/nucleus/constants.py +++ b/nucleus/constants.py @@ -149,6 +149,7 @@ SLICE_TAGS_KEY = "slice_tags" TAXONOMY_NAME_KEY = "taxonomy_name" TASK_ID_KEY = "task_id" +THRESHOLD_KEY = "threshold" TRACK_REFERENCE_ID_KEY = "track_reference_id" TRACK_REFERENCE_IDS_KEY = "track_reference_ids" TRACKS_KEY = "tracks" diff --git a/nucleus/dataset.py b/nucleus/dataset.py index ea95f840..ff1d421e 100644 --- a/nucleus/dataset.py +++ b/nucleus/dataset.py @@ -67,6 +67,7 @@ REQUEST_ID_KEY, SCENE_IDS_KEY, SLICE_ID_KEY, + THRESHOLD_KEY, TRACK_REFERENCE_IDS_KEY, TRACKS_KEY, TRAINED_SLICE_ID_KEY, @@ -74,6 +75,7 @@ VIDEO_URL_KEY, ) from .data_transfer_object.dataset_info import DatasetInfo +from .deduplication import DeduplicationResult, DeduplicationStats from .data_transfer_object.dataset_size import DatasetSize from .data_transfer_object.scenes_list import ScenesList, ScenesListEntry from .dataset_item import ( @@ -1006,6 +1008,106 @@ def create_slice_by_ids( ) return Slice(response[SLICE_ID_KEY], self._client) + def deduplicate( + self, + threshold: int, + reference_ids: Optional[List[str]] = None, + ) -> DeduplicationResult: + """Deduplicate images or frames in this dataset. + + Parameters: + threshold: Hamming distance threshold (0-64). Lower = stricter. + 0 = exact matches only. + reference_ids: Optional list of reference IDs to deduplicate. + If not provided (or None), deduplicates the entire dataset. + Cannot be an empty list - use None for entire dataset. + + Returns: + DeduplicationResult with unique_reference_ids, unique_item_ids, and stats. + + Raises: + ValueError: If reference_ids is an empty list (use None for entire dataset). + NucleusAPIError: If threshold is not an integer between 0 and 64 inclusive. + NucleusAPIError: If any reference_id is not found in the dataset. + NucleusAPIError: If any item is missing a perceptual hash (pHash). + Contact Scale support if this occurs. + + Note: + - For scene datasets, this deduplicates the underlying scene frames, + not the scenes themselves. Frame reference IDs or dataset item IDs + should be provided for scene datasets. + - For very large datasets, this operation may take significant time. + """ + # Client-side validation + if reference_ids is not None and len(reference_ids) == 0: + raise ValueError( + "reference_ids cannot be empty. Omit reference_ids parameter to deduplicate entire dataset." + ) + + payload: Dict[str, Any] = {THRESHOLD_KEY: threshold} + if reference_ids is not None: + payload[REFERENCE_IDS_KEY] = reference_ids + + response = self._client.make_request( + payload, f"dataset/{self.id}/deduplicate" + ) + return DeduplicationResult( + unique_item_ids=response["unique_item_ids"], + unique_reference_ids=response["unique_reference_ids"], + stats=DeduplicationStats( + threshold=threshold, + original_count=response["stats"]["original_count"], + deduplicated_count=response["stats"]["deduplicated_count"], + ), + ) + + def deduplicate_by_ids( + self, + threshold: int, + dataset_item_ids: List[str], + ) -> DeduplicationResult: + """Deduplicate images or frames by internal dataset item IDs. + + Parameters: + threshold: Hamming distance threshold (0-64). Lower = stricter. + 0 = exact matches only. + dataset_item_ids: List of internal dataset item IDs to deduplicate. + Must be non-empty. To deduplicate the entire dataset, refer to + the documentation for `deduplicate()` instead. + + Returns: + DeduplicationResult with unique_item_ids, unique_reference_ids, and stats. + + Raises: + ValueError: If dataset_item_ids is empty. + NucleusAPIError: If threshold is not an integer between 0 and 64 inclusive. + NucleusAPIError: If any dataset_item_id is not found in the dataset. + NucleusAPIError: If any item is missing a perceptual hash (pHash). + Contact Scale support if this occurs. + """ + # Client-side validation + if not dataset_item_ids: + raise ValueError( + "dataset_item_ids must be non-empty. Use deduplicate() for entire dataset." + ) + + payload = { + DATASET_ITEM_IDS_KEY: dataset_item_ids, + THRESHOLD_KEY: threshold, + } + response = self._client.make_request( + payload, f"dataset/{self.id}/deduplicate" + ) + return DeduplicationResult( + unique_item_ids=response["unique_item_ids"], + unique_reference_ids=response["unique_reference_ids"], + stats=DeduplicationStats( + threshold=threshold, + original_count=response["stats"]["original_count"], + deduplicated_count=response["stats"]["deduplicated_count"], + ), + ) + def build_slice( self, name: str, diff --git a/nucleus/deduplication.py b/nucleus/deduplication.py new file mode 100644 index 00000000..f427c004 --- /dev/null +++ b/nucleus/deduplication.py @@ -0,0 +1,16 @@ +from dataclasses import dataclass +from typing import List + + +@dataclass +class DeduplicationStats: + threshold: int + original_count: int + deduplicated_count: int + + +@dataclass +class DeduplicationResult: + unique_item_ids: List[str] # Internal dataset item IDs + unique_reference_ids: List[str] # User-defined reference IDs + stats: DeduplicationStats diff --git a/pyproject.toml b/pyproject.toml index 4fe1aaa2..6622dcd6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ ignore = ["E501", "E741", "E731", "F401"] # Easy ignore for getting it running [tool.poetry] name = "scale-nucleus" -version = "0.17.11" +version = "0.17.12" description = "The official Python client library for Nucleus, the Data Platform for AI" license = "MIT" authors = ["Scale AI Nucleus Team "] From 436fbafceb86adf1936e7ac9811425a96ed0b418 Mon Sep 17 00:00:00 2001 From: Edwin Pavlovsky Date: Mon, 23 Feb 2026 15:54:22 -0500 Subject: [PATCH 2/3] Lint fixes --- nucleus/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nucleus/dataset.py b/nucleus/dataset.py index ff1d421e..030d334f 100644 --- a/nucleus/dataset.py +++ b/nucleus/dataset.py @@ -1034,7 +1034,7 @@ def deduplicate( Note: - For scene datasets, this deduplicates the underlying scene frames, - not the scenes themselves. Frame reference IDs or dataset item IDs + not the scenes themselves. Frame reference IDs or dataset item IDs should be provided for scene datasets. - For very large datasets, this operation may take significant time. """ From 0a1c8d23a46281c7f6c6c4aa55c061947439f32e Mon Sep 17 00:00:00 2001 From: Edwin Pavlovsky Date: Mon, 23 Feb 2026 16:00:04 -0500 Subject: [PATCH 3/3] Fix import order --- nucleus/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nucleus/dataset.py b/nucleus/dataset.py index 030d334f..be1c9242 100644 --- a/nucleus/dataset.py +++ b/nucleus/dataset.py @@ -75,7 +75,6 @@ VIDEO_URL_KEY, ) from .data_transfer_object.dataset_info import DatasetInfo -from .deduplication import DeduplicationResult, DeduplicationStats from .data_transfer_object.dataset_size import DatasetSize from .data_transfer_object.scenes_list import ScenesList, ScenesListEntry from .dataset_item import ( @@ -85,6 +84,7 @@ check_items_have_dimensions, ) from .dataset_item_uploader import DatasetItemUploader +from .deduplication import DeduplicationResult, DeduplicationStats from .deprecation_warning import deprecated from .errors import NotFoundError, NucleusAPIError from .job import CustomerJobTypes, jobs_status_overview