50% Format Annotations are Missing!

[aflah02]

Hi
I was trying to use the topic format annotations in the HF dataset but it is missing almost 50% of the annotation data.

Here's the script I'm using to measure this after downloading both the documents/ and domains_formats/ directories -

# check_missing_labels.py

import os
import re

# Paths

BASE = "/dais/fs/scratch/afkhan/Creativity_Project/Data/Corpus-200B-Documents"
DOCS_DIR = f"{BASE}/documents"
LABELS_DIR = f"{BASE}/domains_formats"

# Regex to extract shard number
SHARD_RE = re.compile(r"CC_shard_(\d+)_processed\.jsonl")

# Collect all document files
doc_files = [f for f in os.listdir(DOCS_DIR) if f.endswith(".jsonl")]

missing_files = []
missing_shard_numbers = []

for doc_file in doc_files:
    base = doc_file.replace(".jsonl", "")
    label_file = f"{base}__choice.npy"
    label_path = os.path.join(LABELS_DIR, label_file)
    if not os.path.exists(label_path):
        missing_files.append(doc_file)

        # Extract shard number
        match = SHARD_RE.match(doc_file)
        if match:
            missing_shard_numbers.append(int(match.group(1)))

# Print missing files
if missing_files:
    print("The following document files do NOT have a corresponding label file:")
    for f in missing_files:
        print(f" - {f}")
else:
    print("All document files have label files.")

# Percentage calculation
total = len(doc_files)
missing_count = len(missing_files)
percent_missing = (missing_count / total) * 100

print(f"\nTotal documents: {total}")
print(f"Missing label files: {missing_count} ({percent_missing:.2f}%)")

# Print smallest shard number missing
if missing_shard_numbers:
    smallest_shard = min(missing_shard_numbers)
    print(f"Smallest shard number missing a label file: {smallest_shard:08d}")

Here's the output -

Total documents: 9889
Missing label files: 4937 (49.92%)
Smallest shard number missing a label file: 00000005

and if you check the dataset on HF there is indeed no label file for shard 00000005. Is this expected/am I missing something here?

[aflah02]

I did not download the logit folder but at a glance it seems to have atleast some of those files inside like shard 00000005 is present here https://huggingface.co/datasets/WebOrganizer/Corpus-200B/tree/main/domains_topics__logits

[aflah02]

Update: Just checked and all the files are present in the logits folder atleast. Here's my script to check the same

# check_missing_labels.py

import os
import re
import argparse

# Paths

BASE = "/dais/fs/scratch/afkhan/Creativity_Project/Data/Corpus-200B-Documents"
DOCS_DIR = f"{BASE}/documents"
LABELS_DIRS = [f"{BASE}/domains_formats", f"{BASE}/domains_formats__logits"]

# Regex to extract shard number
SHARD_RE = re.compile(r"CC_shard_(\d+)_processed\.jsonl")

# Argument parser
parser = argparse.ArgumentParser(description="Check for missing label files")
parser.add_argument("--verbose", action="store_true", help="Print all missing files")
args = parser.parse_args()

# Collect all document files
doc_files = [f for f in os.listdir(DOCS_DIR) if f.endswith(".jsonl")]

def check_labels_dir(labels_dir, dir_name):
    missing_files = []
    missing_shard_numbers = []

    # Determine label file suffix based on directory name
    if dir_name == "domains_formats__logits":
        label_suffix = "__logits.npy"
    else:
        label_suffix = "__choice.npy"

    for doc_file in doc_files:
        base = doc_file.replace(".jsonl", "")
        label_file = f"{base}{label_suffix}"
        label_path = os.path.join(labels_dir, label_file)
        if not os.path.exists(label_path):
            missing_files.append(doc_file)

            # Extract shard number
            match = SHARD_RE.match(doc_file)
            if match:
                missing_shard_numbers.append(int(match.group(1)))

    # Print missing files
    if missing_files:
        print(f"\n{dir_name}:")
        print(f"  The following document files do NOT have a corresponding label file ({label_suffix}):")
        if args.verbose:
            for f in missing_files:
                print(f"    - {f}")
        else:
            print(f"    (omitted {len(missing_files)} files, use --verbose to show all)")
    else:
        print(f"\n{dir_name}:")
        print(f"  All document files have label files ({label_suffix}).")

    # Percentage calculation
    total = len(doc_files)
    missing_count = len(missing_files)
    percent_missing = (missing_count / total) * 100

    print(f"  Total documents: {total}")
    print(f"  Missing label files: {missing_count} ({percent_missing:.2f}%)")

    # Print smallest shard number missing
    if missing_shard_numbers:
        smallest_shard = min(missing_shard_numbers)
        print(f"  Smallest shard number missing ({label_suffix}): {smallest_shard:08d}")

    return missing_files, missing_shard_numbers

# Check all label directories
for labels_dir in LABELS_DIRS:
    dir_name = os.path.basename(labels_dir)
    check_labels_dir(labels_dir, dir_name)

Output -

(data_process_env) (rope_pct_eval_env) afkhan@dais11:/dais/fs/scratch/afkhan/Creativity_Project/Creativity_LM_Research> python quantify_missing_labels.py

domains_formats:
  The following document files do NOT have a corresponding label file (__choice.npy):
    (omitted 4937 files, use --verbose to show all)
  Total documents: 9889
  Missing label files: 4937 (49.92%)
  Smallest shard number missing (__choice.npy): 00000005

domains_formats__logits:
  All document files have label files (__logits.npy).
  Total documents: 9889
  Missing label files: 0 (0.00%)

[CodeCreator]

Thank you for the report! I've looked into this and I think it's just been some issue with HF dataset upload.

I will fix it this weekend.