fix(python-examples): align Python dynamic benchmark logic with C++

- Match dynamic graph construction in `main.py` with `build.h` by using a
  single builder instance for interleaved add/remove stream types
- Handle dataset remainder vectors when base size is not divisible by 4
  (fixes vertex count mismatch on datasets with odd size like Audio: 26693 vs 26692)
- Remove `find_file` wildcard matching in `dataset_utils.py` and enforce exact
  file paths matching C++ `dataset.h` (fixes wrong query file loading and low recall)
- Fix SIFT1M metadata base/query filenames (`sift1m_base.fvecs` / `sift1m_query.fvecs`)
- Fix dataset folder resolution before archive download checks
- Rename dataset preset key `glove-100` to `glove` and remove `DATASET_ALIASES`
- Cast numpy scalar labels to python `int` in `add_entry()` loops

Author: Neiko2002

cpp/deglib/include/graph/dynamic_graph.h

@@ -13,12 +13,12 @@
         "name": "SIFT1M",
         "url": "https://static.visual-computing.com/paper/DEG/sift.tar.gz",
         "archive": "sift.tar.gz",
-        "folder": "sift",
+        "folder": "sift1m",
         "metric": Metric.FP32_L2,
         "dim": 128,
         "base_count": 1000000,
-        "base_file": "sift_base.fvecs",
-        "query_file": "sift_query.fvecs",
+        "base_file": "sift1m_base.fvecs",
+        "query_file": "sift1m_query.fvecs",
         "gt_file": "sift1m_groundtruth_top100_nb1000000.ivecs",
         "gt_half_file": "sift1m_groundtruth_top100_nb500000.ivecs",
     },
@@ -61,7 +61,7 @@
         "gt_file": "deep1m_groundtruth_top100_nb1000000.ivecs",
         "gt_half_file": "deep1m_groundtruth_top100_nb500000.ivecs",
     },
-    "glove-100": {
+    "glove": {
         "name": "GloVe-100",
         "url": "https://static.visual-computing.com/paper/DEG/glove-100.tar.gz",
         "archive": "glove-100.tar.gz",
@@ -76,14 +76,8 @@
     },
 }
 
-# Aliases
-DATASET_ALIASES = {
-    "glove": "glove-100",
-}
-
 def resolve_dataset_key(key: str) -> str:
-    key = key.lower()
-    return DATASET_ALIASES.get(key, key)
+    return key.lower()
 
 def get_default_cache_dir() -> Path:
     """Returns the default dataset cache directory (~/.cache/deg_datasets or DEG_CACHE_DIR)."""
@@ -178,41 +172,41 @@ def ensure_dataset(dataset_key: str, cache_dir: Path) -> Path:
 
     meta = DATASET_METADATA[key]
     archive_path = cache_dir / meta["archive"]
+    
+    # 1. Check if direct folder (e.g., D:\Data\DEG\sift1m or D:\Data\DEG\sift) exists
     extracted_folder = cache_dir / meta["folder"]
+    if not extracted_folder.is_dir():
+        # Check case-insensitive / fallback matches before triggering a download
+        subdirs = [p for p in cache_dir.iterdir() if p.is_dir() and meta["folder"].lower() in p.name.lower()]
+        if subdirs:
+            extracted_folder = subdirs[0]
 
+    # 2. If folder is still not found, check/download archive and extract
     if not extracted_folder.is_dir():
         if not archive_path.is_file():
-            download_file(meta["url"], archive_path)
+            # Also check if archive exists inside a subfolder or cache_dir
+            archive_matches = list(cache_dir.rglob(meta["archive"]))
+            if archive_matches:
+                archive_path = archive_matches[0]
+            else:
+                download_file(meta["url"], archive_path)
 
         print(f"Extracting {archive_path} into {cache_dir}...")
         with tarfile.open(archive_path, "r:gz") as tar:
             tar.extractall(path=cache_dir)
         print("Extraction complete.")
-    
-    if not extracted_folder.is_dir():
-        subdirs = [p for p in cache_dir.iterdir() if p.is_dir() and meta["folder"].lower() in p.name.lower()]
-        if subdirs:
-            extracted_folder = subdirs[0]
-        else:
-            extracted_folder = cache_dir
+
+        extracted_folder = cache_dir / meta["folder"]
+        if not extracted_folder.is_dir():
+            subdirs = [p for p in cache_dir.iterdir() if p.is_dir() and meta["folder"].lower() in p.name.lower()]
+            if subdirs:
+                extracted_folder = subdirs[0]
+            else:
+                extracted_folder = cache_dir
 
     return extracted_folder
 
-def find_file(directory: Path, expected_name: str, pattern: str) -> Path:
-    """Finds a file by exact name or matching pattern in directory tree."""
-    target = directory / expected_name
-    if target.is_file():
-        return target
-    
-    matches = list(directory.rglob(expected_name))
-    if matches:
-        return matches[0]
 
-    matches = list(directory.rglob(f"*{pattern}*"))
-    if matches:
-        return matches[0]
-    
-    raise FileNotFoundError(f"Could not find file '{expected_name}' or pattern '{pattern}' in {directory}")
 
 def compute_and_save_anns_gt(base_vecs: np.ndarray, query_vecs: np.ndarray, float_space: FloatSpace, k: int, out_path: Path):
     """Computes ANNS ground truth against the full base dataset."""
@@ -275,8 +269,10 @@ def load_dataset_for_dynamic(
 
     cleanup_legacy_gt(folder)
 
-    base_path = find_file(folder, meta["base_file"], "base")
-    query_path = find_file(folder, meta["query_file"], "query")
+    files_dir = folder / meta["folder"] if (folder / meta["folder"]).is_dir() else folder
+
+    base_path = files_dir / meta["base_file"]
+    query_path = files_dir / meta["query_file"]
 
     print(f"Loading base features from {base_path}...")
     base_vecs = repo.fvecs_read(base_path)
@@ -289,29 +285,19 @@ def load_dataset_for_dynamic(
     float_space = FloatSpace.create(dims, metric)
 
     # Full ANNS Ground Truth
-    gt_file_name = meta["gt_file"]
-    gt_path = folder / gt_file_name
+    gt_path = files_dir / meta["gt_file"]
     if not gt_path.is_file():
-        sub_matches = list(folder.rglob(gt_file_name))
-        if sub_matches:
-            gt_path = sub_matches[0]
-        else:
-            print(f"ANNS Full Ground Truth not found at {gt_path}.")
-            compute_and_save_anns_gt(base_vecs, query_vecs, float_space, 100, gt_path)
+        print(f"ANNS Full Ground Truth not found at {gt_path}.")
+        compute_and_save_anns_gt(base_vecs, query_vecs, float_space, 100, gt_path)
 
     print(f"Loading full groundtruth indices from {gt_path}...")
     gt_vecs_full = repo.ivecs_read(gt_path)
 
     # Half ANNS Ground Truth (against first base_count/2 vectors)
-    gt_half_file_name = meta["gt_half_file"]
-    gt_half_path = folder / gt_half_file_name
+    gt_half_path = files_dir / meta["gt_half_file"]
     if not gt_half_path.is_file():
-        sub_matches = list(folder.rglob(gt_half_file_name))
-        if sub_matches:
-            gt_half_path = sub_matches[0]
-        else:
-            print(f"ANNS Half Ground Truth not found at {gt_half_path}.")
-            compute_and_save_anns_gt_half(base_vecs, query_vecs, float_space, 100, gt_half_path)
+        print(f"ANNS Half Ground Truth not found at {gt_half_path}.")
+        compute_and_save_anns_gt_half(base_vecs, query_vecs, float_space, 100, gt_half_path)
 
     print(f"Loading half groundtruth indices from {gt_half_path}...")
     gt_vecs_half = repo.ivecs_read(gt_half_path)

examples/dynamic_data/dataset_utils.py

@@ -242,63 +242,27 @@ def build_dynamic_graph(
             builder.remove_entry(int(all_labels[i]))
 
     elif stream_type == DataStreamType.AddHalfRemoveAndAddOneAtATime:
-        # Interleaved add/remove.
-        # Mirrors C++ build.h:
-        #   quarter = n/4
-        #   add [0..quarter) + [half..half+quarter)  (first wave)
-        #   then for i in [0..quarter):
-        #       add [quarter+i] + [half+quarter+i]
-        #       remove [half + 2*i]
-        #       remove [half + 2*i + 1]
+        # Interleaved add/remove matching C++ build.h.
         quarter = n // 4
 
-        # First wave: indices [0..n/4) and [n/2..3n/4)
-        first_labels = np.concatenate([all_labels[:quarter], all_labels[half:half + quarter]])
-        first_vecs = np.concatenate([base_vecs[:quarter], base_vecs[half:half + quarter]])
-        builder.add_entry(first_labels, first_vecs)
-        builder.build(callback="progress")  # build first wave before interleaving
-
-        # Recreate builder on the same graph for interleaved phase
-        builder2 = deglib.builder.EvenRegularGraphBuilder(
-            graph_mut,
-            rng=deglib.Mt19937(7),
-            optimization_target=deglib.builder.OptimizationTarget.StreamingData,
-            extend_k=preset.get("extend_k", k),
-            extend_eps=preset["build_eps"],
-            improve_k=preset.get("improve_k", 0),
-            improve_eps=preset.get("improve_eps", 0.0),
-        )
-        builder2.set_batch_size(10, 10)
-        if build_threads > 1:
-            builder2.set_thread_count(build_threads)
+        # 1st loop: add base_size_quarter pairs from [0, quarter) and [half, half + quarter)
+        for i in range(quarter):
+            builder.add_entry(int(all_labels[i]), base_vecs[i : i + 1])
+            builder.add_entry(int(all_labels[half + i]), base_vecs[half + i : half + i + 1])
 
-        # Second wave: interleaved add [n/4+i, 3n/4+i] + remove [n/2+2i, n/2+2i+1]
+        # 2nd loop: add base_size_quarter pairs from [quarter, half) and remove same number
         for i in range(quarter):
-            builder2.add_entry(
-                np.array([all_labels[quarter + i]], dtype=np.uint32),
-                base_vecs[quarter + i : quarter + i + 1],
-            )
-            builder2.add_entry(
-                np.array([all_labels[half + quarter + i]], dtype=np.uint32),
-                base_vecs[half + quarter + i : half + quarter + i + 1],
-            )
-            builder2.remove_entry(int(all_labels[half + 2 * i]))
-            builder2.remove_entry(int(all_labels[half + 2 * i + 1]))
-
-        build_start = time.perf_counter()
-        builder2.build(callback="progress")
-        build_time = time.perf_counter() - build_start
-        print(f"Graph built in {build_time:.2f} seconds ({graph_mut.size()} vertices).")
-
-        if graph_path:
-            graph_path.parent.mkdir(parents=True, exist_ok=True)
-            graph_mut.save_graph(str(graph_path))
-            print(f"Graph saved to: {graph_path}")
-            return deglib.graph.load_readonly_graph(str(graph_path))
-        else:
-            return deglib.graph.ReadOnlyGraph.from_graph(graph_mut)
+            builder.add_entry(int(all_labels[quarter + i]), base_vecs[quarter + i : quarter + i + 1])
+            builder.add_entry(int(all_labels[half + quarter + i]), base_vecs[half + quarter + i : half + quarter + i + 1])
+            builder.remove_entry(int(all_labels[half + (i * 2) + 0]))
+            builder.remove_entry(int(all_labels[half + (i * 2) + 1]))
+
+        # Remainder wave: If base_size is not divisible by 4, add remaining entries to reach exactly half vertices
+        remainder = half - (quarter * 2)
+        for i in range(remainder):
+            rem_idx = quarter * 2 + i
+            builder.add_entry(int(all_labels[rem_idx]), base_vecs[rem_idx : rem_idx + 1])
 
-    # For AddHalf and AddAllRemoveHalf: run build now
     build_start = time.perf_counter()
     builder.build(callback="progress")
     build_time = time.perf_counter() - build_start
@@ -485,9 +449,9 @@ def main():
     parser.add_argument(
         "dataset",
         nargs="?",
-        default=None,
-        choices=["sift1m", "deep1m", "glove", "glove-100", "audio", "enron", "all"],
-        help="Dataset name (e.g. sift1m, deep1m, glove-100, audio, enron, all) (default: sift1m)",
+        default="sift1m",
+        choices=["sift1m", "deep1m", "glove", "audio", "enron", "all"],
+        help="Dataset name (e.g. sift1m, deep1m, glove, audio, enron, all) (default: sift1m)",
     )
     parser.add_argument(
         "--graph-dir",

examples/dynamic_data/main.py

@@ -45,7 +45,7 @@
         "anns_repeat": 1,
         "search_eps_list": [0.01, 0.02, 0.03, 0.04, 0.06, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.8, 1.0, 1.5, 2.0],
     },
-    "glove-100": {
+    "glove": {
         "metric": Metric.FP32_InnerProduct,
         "k": 30,
         "extend_k": 60,
@@ -57,15 +57,9 @@
     },
 }
 
-# Alias mapping
-DATASET_ALIASES = {
-    "glove": "glove-100",
-}
-
 def get_preset(dataset_key: str) -> Dict[str, Any]:
     """Returns preset graph build and benchmark parameters for dataset_key."""
     key = dataset_key.lower()
-    key = DATASET_ALIASES.get(key, key)
     if key not in DYNAMIC_DATASET_PRESETS:
         raise ValueError(f"No preset for dataset '{dataset_key}'. Standard presets: {list(DYNAMIC_DATASET_PRESETS.keys())}")
     return DYNAMIC_DATASET_PRESETS[key]