create_sprint_files.py

#!/usr/bin/env python3
"""
scrape_intv_games.py - Scrapes game images from LaunchBox Games Database for Mattel Intellivision games.

Requirements:
    pip install requests beautifulsoup4 pillow
"""

import requests
from bs4 import BeautifulSoup
from PIL import Image
import io
import csv
import json
import time
import random
import argparse
import re
import hashlib
from pathlib import Path
from datetime import datetime
from typing import Dict, List, Optional, Tuple

VERSION = "v1.0"

# Embedded mapper configurations (0.cfg through 9.cfg)
MAPPER = [
    # 0.cfg
    [
        "$0000 - $1FFF = $5000   ;  8K to $5000 - $6FFF",
        "$2000 - $2FFF = $D000   ;  4K to $D000 - $DFFF",
        "$3000 - $3FFF = $F000   ;  4K to $F000 - $FFFF",
    ],
    # 1.cfg
    [
        "$0000 - $1FFF = $5000   ;  8K to $5000 - $6FFF",
        "$2000 - $4FFF = $D000   ; 12K to $D000 - $FFFF",
    ],
    # 2.cfg
    [
        "$0000 - $1FFF = $5000   ;  8K to $5000 - $6FFF",
        "$2000 - $4FFF = $9000   ; 12K to $9000 - $BFFF",
        "$5000 - $5FFF = $D000   ;  4K to $D000 - $DFFF",
    ],
    # 3.cfg
    [
        "$0000 - $1FFF = $5000   ;  8K to $5000 - $6FFF",
        "$2000 - $3FFF = $9000   ;  8K to $9000 - $AFFF",
        "$4000 - $4FFF = $D000   ;  4K to $D000 - $DFFF",
        "$5000 - $5FFF = $F000   ;  4K to $F000 - $FFFF",
    ],
    # 4.cfg
    [
        "$0000 - $1FFF = $5000   ;  8K to $5000 - $6FFF",
        "",
        "[memattr]",
        "$D000 - $D3FF = RAM 8",
    ],
    # 5.cfg
    [
        "$0000 - $2FFF = $5000   ; 12K to $5000 - $7FFF",
        "$3000 - $5FFF = $9000   ; 12K to $9000 - $BFFF",
    ],
    # 6.cfg
    [
        "$0000 - $1FFF = $6000   ;  8K to $6000 - $7FFF",
    ],
    # 7.cfg
    [
        "$0000 - $1FFF = $4800   ;  8K to $4800 - $67FF",
    ],
    # 8.cfg
    [
        "$0000 - $0FFF = $5000   ;  4K to $5000 - $6000",
        "$1000 - $1FFF = $7000   ;  4K to $7000 - $7FFF",
    ],
    # 9.cfg
    [
        "$0000 - $1FFF = $5000   ;  8K to $5000 - $6FFF",
        "$2000 - $3FFF = $9000   ;  8K to $9000 - $AFFF",
        "$4000 - $4FFF = $D000   ;  4K to $D000 - $DFFF",
        "$5000 - $5FFF = $F000   ;  4K to $F000 - $FFFF",
        "",
        "[memattr]",
        "$8800 - $8FFF = RAM 8",
    ],
]


def get_mapper(file_num: int) -> List[str]:
    """Get all lines from a specific .cfg file.
    
    Args:
        file_num: The .cfg file number (0-9)
        
    Returns:
        List of lines from the specified file, with [mapping] header prepended
        
    Raises:
        ValueError: If file_num is not in range 0-9
    """
    if file_num < 0 or file_num > 9:
        raise ValueError(f"File number must be between 0 and 9, got {file_num}")
    
    return ["[mapping]"] + MAPPER[file_num]


# Constants
BASE_URL = "https://gamesdb.launchbox-app.com"
PLATFORM_URL = f"{BASE_URL}/platforms/games/15-mattel-intellivision"
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
OUTPUT_DIR = Path("output")

# Category mapping: LaunchBox category -> output filename suffix
CATEGORY_MAPPING = {
    "box - front": ".png",
    "arcade - control": "_big_overlay.png",  # Matches "arcade - controls information", "arcade - control panel", etc.
    "screenshot - gameplay": "_snap"  # Special case: will append numbers
}

# Target dimensions for image rescaling: suffix -> (width, height)
TARGET_DIMENSIONS = {
    ".png": (186, 256),           # Box art
    "_big_overlay.png": (300, 478),  # Arcade controls overlay
    "_small.png": (148, 204),     # Small thumbnail
    "_snap": (640, 400)           # Gameplay screenshots
}


def sanitize_filename(name: str) -> str:
    """Sanitize filename by replacing invalid characters with underscores."""
    invalid_chars = '/\\:*?"<>|'
    sanitized = name
    for char in invalid_chars:
        sanitized = sanitized.replace(char, '_')
    # Keep spaces in the filename
    return sanitized


def get_roms_directory() -> Path:
    """Get roms directory from user or return default."""
    default_roms = Path("roms")
    
    print("ROM Directory Setup")
    print("-" * 50)
    print(f"Default location: {default_roms.absolute()}")
    
    user_input = input("Press Enter to use default, or enter a different path: ").strip()
    
    if user_input:
        roms_dir = Path(user_input).expanduser().absolute()
    else:
        roms_dir = default_roms.absolute()
    
    if not roms_dir.exists():
        print(f"Creating directory: {roms_dir}")
        roms_dir.mkdir(parents=True, exist_ok=True)
    
    return roms_dir


def get_rom_files(roms_dir: Path) -> List[Path]:
    """Get list of ROM files from directory, with common ROM extensions."""
    rom_extensions = {'.bin', '.int', '.rom'}
    return sorted([f for f in roms_dir.glob('*') if f.is_file() and f.suffix.lower() in rom_extensions])


def normalize_rom_name(rom_path: Path) -> str:
    """Normalize ROM filename for matching with LaunchBox games."""
    # Get filename without extension
    name = rom_path.stem
    # Remove underscores, convert to title case for matching
    name = name.replace('_', ' ')
    return name


def normalize_key(name: str) -> str:
    """Normalize strings for loose matching: lowercase and collapse non-alnum to single spaces."""
    key = re.sub(r"[^a-z0-9]+", " ", name.lower())
    return " ".join(key.split())


def generate_search_variants(name: str) -> List[str]:
    """Generate alternate title variants (e.g., handling trailing ', The')."""
    variants: List[str] = []
    seen = set()

    def add_variant(candidate: str):
        cand = candidate.strip()
        if cand.lower() not in seen:
            variants.append(cand)
            seen.add(cand.lower())

    add_variant(name)

    # Handle titles formatted with trailing articles, e.g., "Title, The"
    m = re.match(r"^(.*?),\s*(the|a|an)(.*)$", name, flags=re.IGNORECASE)
    if m:
        base = m.group(1).strip()
        article = m.group(2).strip().title()
        rest = m.group(3)
        add_variant(f"{article} {base}{rest}")

    return variants


def load_asset_cache(cache_path: Path = Path("asset_cache.json")) -> Dict[str, Dict[str, str]]:
    """Load asset cache index mapping game titles to their generated asset files.
    Returns: Dict[str, Dict[str, str]] - game_title -> {asset_type: filepath, ...}"""
    if not cache_path.exists():
        return {}
    
    try:
        with open(cache_path, 'r', encoding='utf-8') as f:
            return json.load(f)
    except Exception as e:
        print(f"WARNING: Failed to load asset cache: {e}")
        return {}


def save_asset_cache(cache: Dict[str, Dict[str, str]], cache_path: Path = Path("asset_cache.json")) -> bool:
    """Save asset cache index to file.
    Returns True if successful."""
    try:
        cache_path.parent.mkdir(parents=True, exist_ok=True)
        with open(cache_path, 'w', encoding='utf-8') as f:
            json.dump(cache, f, indent=2, ensure_ascii=False)
        return True
    except Exception as e:
        print(f"WARNING: Failed to save asset cache: {e}")
        return False


def load_memory_map(memory_map_path: Path = Path("game_maps.csv")) -> Dict[str, Tuple[str, int]]:
    """Load memory map CSV to map ROM filenames to game titles and mapper numbers.
    Returns: Dict[str, Tuple[str, int]] - filename -> (title, mapper_num)"""
    mapping: Dict[str, Tuple[str, int]] = {}
    if not memory_map_path.exists():
        print(f"WARNING: memory map not found at {memory_map_path}, using filename matching")
        return mapping

    try:
        with open(memory_map_path, newline='', encoding='utf-8') as f:
            reader = csv.DictReader(f)
            for row_num, row in enumerate(reader, start=2):  # Start at 2 (1 is header)
                title = (row.get("Game Title") or "").strip()
                filenames_field = row.get("Filename") or ""
                
                # Safely parse mapper number
                try:
                    mapper_num = int(row.get("Map #", 0))
                except (ValueError, TypeError):
                    print(f"WARNING: Invalid mapper number in row {row_num}, using default 0")
                    mapper_num = 0
                
                if not title or not filenames_field:
                    continue

                for variant in filenames_field.split(';'):
                    variant = variant.strip().strip('"')
                    if not variant:
                        continue
                    # Store by full name and stem, lowercase for case-insensitive matching
                    mapping[variant.lower()] = (title, mapper_num)
                    mapping[Path(variant).stem.lower()] = (title, mapper_num)
                    mapping[normalize_key(variant)] = (title, mapper_num)

                # Also allow lookups by the Game Title itself (case-insensitive)
                mapping[title.lower()] = (title, mapper_num)
                # And a simple underscore form in case filenames use underscores
                mapping[title.replace(' ', '_').lower()] = (title, mapper_num)
                # And a normalized key form for loose matching
                mapping[normalize_key(title)] = (title, mapper_num)

        print(f"Loaded {len(mapping)} filename mappings from memory map")
    except Exception as exc:
        print(f"WARNING: failed to load memory map: {exc}")

    return mapping


def resolve_rom_to_title(rom_path: Path, memory_map: Dict[str, Tuple[str, int]]) -> Optional[Tuple[str, int]]:
    """Return mapped game title and mapper number for a ROM if present in memory map.
    Returns: Optional[Tuple[str, int]] - (title, mapper_num) or None"""
    if not memory_map:
        return None

    full_name = rom_path.name.lower()
    stem_name = rom_path.stem.lower()
    normalized_name = stem_name.replace('_', ' ').replace('-', ' ').strip()
    loose_name = normalize_key(stem_name)

    if full_name in memory_map:
        return memory_map[full_name]
    if stem_name in memory_map:
        return memory_map[stem_name]
    if normalized_name in memory_map:
        return memory_map[normalized_name]
    if loose_name in memory_map:
        return memory_map[loose_name]
    return None


def generate_cfg_file(output_name: str, mapper_num: int, output_dir: Path) -> bool:
    """Generate .cfg file for a game using the specified mapper number.
    Returns True if successful."""
    try:
        cfg_path = output_dir / f"{output_name}.cfg"
        
        # Skip if already exists
        if cfg_path.exists():
            return True
        
        # Get mapper lines
        lines = get_mapper(mapper_num)
        
        # Write to file
        with open(cfg_path, 'w', encoding='utf-8') as f:
            f.write('\n'.join(lines) + '\n')
        
        return True
    except Exception as e:
        print(f"         WARNING: Failed to generate .cfg file: {e}", flush=True)
        return False


def random_delay():
    """Add random delay between 1-2 seconds between requests."""
    time.sleep(random.uniform(1.0, 2.0))


def compute_image_checksum(img: Image.Image) -> str:
    """Compute SHA256 checksum of image data for deduplication."""
    try:
        img_bytes = io.BytesIO()
        img.save(img_bytes, format='PNG')
        img_bytes.seek(0)
        return hashlib.sha256(img_bytes.read()).hexdigest()
    except Exception:
        return ""





def get_page(url: str) -> Optional[BeautifulSoup]:
    """Fetch and parse a page with error handling."""
    try:
        headers = {"User-Agent": USER_AGENT}
        response = requests.get(url, headers=headers, timeout=30)
        response.raise_for_status()
        return BeautifulSoup(response.content, 'html.parser')
    except Exception:
        return None


def download_image(url: str) -> Optional[Image.Image]:
    """Download image and convert to PIL Image."""
    try:
        headers = {"User-Agent": USER_AGENT}
        response = requests.get(url, headers=headers, timeout=30)
        response.raise_for_status()
        img = Image.open(io.BytesIO(response.content))
        return img.convert('RGBA') if img.mode != 'RGBA' else img
    except Exception:
        return None


def resize_image_to_dimensions(img: Image.Image, target_width: int, target_height: int) -> Image.Image:
    """
    Resize image to exact target dimensions with transparent letterboxing.
    
    Preserves aspect ratio by scaling to fit within target, then centers on transparent canvas.
    Uses high-quality LANCZOS resampling.
    """
    try:
        # Calculate aspect ratios
        img_aspect = img.width / img.height if img.height > 0 else 1
        target_aspect = target_width / target_height if target_height > 0 else 1
        
        # Determine scaling to fit within target while preserving aspect ratio
        if img_aspect > target_aspect:
            # Image is wider, scale by width
            new_width = target_width
            new_height = int(target_width / img_aspect)
        else:
            # Image is taller, scale by height
            new_height = target_height
            new_width = int(target_height * img_aspect)
        
        # Ensure dimensions don't exceed target
        new_width = min(new_width, target_width)
        new_height = min(new_height, target_height)
        
        # Resize image with high-quality resampling
        resized = img.resize((new_width, new_height), Image.Resampling.LANCZOS)
        
        # Create transparent canvas at target dimensions
        canvas = Image.new('RGBA', (target_width, target_height), (0, 0, 0, 0))
        
        # Calculate offset to center the resized image
        x_offset = (target_width - new_width) // 2
        y_offset = (target_height - new_height) // 2
        
        # Paste resized image onto transparent canvas
        canvas.paste(resized, (x_offset, y_offset), resized)
        
        return canvas
    except Exception:
        return img


def load_image_from_path(filepath: Path) -> Optional[Image.Image]:
    """Load image from file path, converting to RGBA."""
    try:
        return Image.open(filepath).convert('RGBA')
    except Exception:
        return None


def save_image(img: Image.Image, filepath: Path, suffix: str = ""):
    """Save PIL Image as PNG, optionally rescaling to target dimensions based on suffix.
    Skips saving if file already exists."""
    try:
        if filepath.exists():
            return
        filepath.parent.mkdir(parents=True, exist_ok=True)
        if suffix in TARGET_DIMENSIONS:
            target_w, target_h = TARGET_DIMENSIONS[suffix]
            img = resize_image_to_dimensions(img, target_w, target_h)
        img.save(filepath, 'PNG')
    except Exception:
        pass


def copy_cached_assets(cache_entry: Dict[str, str], unique_name: str, output_dir: Path) -> bool:
    """Copy cached assets for a game to the output directory.
    Returns True if successful."""
    try:
        any_copied = False
        for asset_type, source_filepath in cache_entry.items():
            source_path = Path(source_filepath)
            
            # Skip if source doesn't exist
            if not source_path.exists():
                continue
            
            # Determine destination based on asset type
            dest_filename = f"{unique_name}{asset_type}"
            dest_path = output_dir / dest_filename
            
            # Skip if destination already exists
            if dest_path.exists():
                continue
            
            # Copy the file
            dest_path.parent.mkdir(parents=True, exist_ok=True)
            import shutil
            shutil.copy2(source_path, dest_path)
            any_copied = True
        
        return any_copied
    except Exception as e:
        print(f"         WARNING: Failed to copy cached assets: {e}", flush=True)
        return False


def register_game_assets(game_title: str, unique_name: str, output_dir: Path, asset_cache: Dict[str, Dict[str, str]]) -> None:
    """Register all generated assets for a game in the asset cache.
    Scans output_dir for files matching the unique_name pattern."""
    try:
        if game_title not in asset_cache:
            asset_cache[game_title] = {}
        
        # Scan all files matching this game and register them
        for asset_file in sorted(output_dir.glob(f"{unique_name}*")):
            if asset_file.is_file():
                suffix = asset_file.name[len(unique_name):]
                asset_cache[game_title][suffix] = str(asset_file)
    except Exception as e:
        print(f"         WARNING: Failed to register assets: {e}", flush=True)


def assets_complete(unique_name: str, output_dir: Path) -> bool:
    """Return True if key assets already exist for this ROM name."""
    base = output_dir / f"{unique_name}"
    required = [
        base.with_suffix('.json'),
        base.with_suffix('.png'),
        output_dir / f"{unique_name}_small.png",
        output_dir / f"{unique_name}_big_overlay.png",
        output_dir / f"{unique_name}_overlay.png",
    ]
    return all(p.exists() for p in required)


def composite_overlay(big_overlay_img: Image.Image, game_name: str, output_dir: Path) -> bool:
    """
    Composite the big overlay behind the controller template and save as <game_name>_overlay.png.
    - Resize the big overlay to 175x282 (letterboxed to exact size)
    - Place the resized overlay at the top-center of the template
    - Template is the topmost layer; overlay shows through transparent areas
    """
    try:
        # Check if overlay file already exists
        output_path = output_dir / f"{game_name}_overlay.png"
        if output_path.exists():
            return True
        
        # Load controller template (top layer) - look in script directory first, then current directory
        script_dir = Path(__file__).parent
        template_path = script_dir / "controller_template.png"
        if not template_path.exists():
            template_path = Path("controller_template.png")
        
        if not template_path.exists():
            print(f"         WARNING: controller_template.png not found at {template_path.absolute()} - skipping overlay composite", flush=True)
            return False

        template = Image.open(template_path).convert('RGBA')

        # Ensure big overlay is RGBA
        if big_overlay_img.mode != 'RGBA':
            big_overlay_img = big_overlay_img.convert('RGBA')

        # Resize the big overlay to 175x282 with letterboxing to exact size
        resized_overlay = resize_image_to_dimensions(big_overlay_img, 180, 300)

        # Create a transparent canvas matching the template size
        canvas = Image.new('RGBA', template.size, (0, 0, 0, 0))

        # Position the overlay at top-center
        x_offset = (template.width - resized_overlay.width) // 2
        y_offset = 7  # Shift down from top
        canvas.paste(resized_overlay, (x_offset, y_offset), resized_overlay)

        # Paste template on top (template is top layer)
        canvas.paste(template, (0, 0), template)

        # Save result
        canvas.save(output_path, 'PNG')
        return True
    except Exception as e:
        print(f"         WARNING: Failed to create overlay composite: {e}", flush=True)
        return False


def extract_games_from_page(soup: BeautifulSoup) -> List[Dict]:
    """Extract game information from listing page."""
    games = []
    try:
        # Find all game containers (games-grid-card divs)
        game_items = soup.find_all('div', class_='games-grid-card')
        
        for item in game_items:
            try:
                # Find game link
                link = item.find('a', href=lambda x: x and '/games/details/' in x)
                if not link:
                    continue
                
                href = link.get('href', '')
                if not href or '/games/details/' not in href:
                    continue
                
                # Extract ID and slug from URL: /games/details/{id}-{slug}
                parts = href.split('/games/details/')[-1]
                if '-' not in parts:
                    continue
                
                game_id = parts.split('-')[0]
                slug = '-'.join(parts.split('-')[1:])
                
                # Extract game name from link text
                link_text = link.get_text(strip=True)
                # Name is usually the first part before platform name
                game_name = link_text.split('Mattel Intellivision')[0].strip()
                if not game_name:
                    game_name = slug.replace('-', ' ').title()
                
                # Extract release status and year from link text
                release_status = ""
                year = ""
                
                link_text_lower = link_text.lower()
                if 'homebrew' in link_text_lower:
                    release_status = 'homebrew'
                elif 'unreleased' in link_text_lower:
                    release_status = 'unreleased'
                elif 'unlicensed' in link_text_lower:
                    release_status = 'unlicensed'
                elif 'prototype' in link_text_lower:
                    release_status = 'prototype'
                elif 'released' in link_text_lower:
                    release_status = 'released'
                
                # Extract year - look for 4-digit numbers (1900-2999 range)
                year_match = re.search(r'(19|20)\d{2}', link_text)
                if year_match:
                    year = year_match.group(0)
                
                # Extract thumbnail from first img in cardImgPart
                thumbnail = None
                card_img = item.find('div', class_='cardImgPart')
                if card_img:
                    img = card_img.find('img')
                    if img and img.get('src'):
                        thumbnail = img['src']
                        if not thumbnail.startswith('http'):
                            thumbnail = BASE_URL + thumbnail if thumbnail.startswith('/') else None
                
                games.append({
                    'id': game_id,
                    'slug': slug,
                    'name': game_name,
                    'release_status': release_status,
                    'year': year,
                    'thumbnail': thumbnail
                })
            except Exception:
                continue
    except Exception:
        pass
    
    return games




def extract_images_from_game_page(soup: BeautifulSoup) -> Dict[str, List[str]]:
    """Extract image URLs from game images page, organized by category."""
    images_by_category = {}
    
    try:
        def add_nuxt_images():
            """Parse __NUXT_DATA__ payload to pull explicit full-size image file names.
            This is the primary/most reliable source of images."""
            nuxt_script = soup.find('script', id='__NUXT_DATA__')
            if not nuxt_script or not nuxt_script.string:
                return False

            try:
                payload = json.loads(nuxt_script.string)
            except Exception:
                return False

            if not isinstance(payload, list):
                return False

            def resolve_scalar(value):
                if isinstance(value, int) and 0 <= value < len(payload):
                    nested = payload[value]
                    if isinstance(nested, (str, int, float, bool)) or nested is None:
                        return nested
                return value

            found_any = False
            for entry in payload:
                if not isinstance(entry, dict) or 'gameImages' not in entry:
                    continue

                images_ref = entry.get('gameImages')
                if isinstance(images_ref, int) and 0 <= images_ref < len(payload):
                    image_indices = payload[images_ref]
                elif isinstance(images_ref, list):
                    image_indices = images_ref
                else:
                    continue

                if not isinstance(image_indices, list):
                    continue

                for idx in image_indices:
                    if not isinstance(idx, int) or not (0 <= idx < len(payload)):
                        continue
                    image_entry = payload[idx]
                    if not isinstance(image_entry, dict):
                        continue

                    category_name = str(resolve_scalar(image_entry.get('imageTypeName', ''))).strip().lower()
                    full_name = resolve_scalar(image_entry.get('fullGameImageFileName'))

                    if not category_name or not full_name:
                        continue

                    category_key = category_name
                    if category_key not in images_by_category:
                        images_by_category[category_key] = []

                    full_url = f"https://images.launchbox-app.com//{full_name}"
                    
                    # Only add full-size URL
                    if full_url not in images_by_category[category_key]:
                        images_by_category[category_key].append(full_url)
                        found_any = True

                # Only need the first matching entry that has gameImages
                break
            
            return found_any

        # Try NUXT data first (most reliable source)
        if add_nuxt_images():
            # NUXT data found images, use those exclusively
            return images_by_category
        
        # Fallback: Parse embedded JSON in the GameDetailsEntity script for full-size image URLs
        pattern = re.compile(r'"([0-9a-f-]{8}-[0-9a-f-]{4}-[0-9a-f-]{4}-[0-9a-f-]{4}-[0-9a-f-]{12}\.(?:png|jpg))","([^"]+)",(\d+),(\d+),"([^"]+)","([0-9a-f-]{8}-[0-9a-f-]{4}-[0-9a-f-]{4}-[0-9a-f-]{4}-[0-9a-f-]{12}\.(?:png|jpg))",(\d+),(\d+),(\d+)')
        for script in soup.find_all('script'):
            if not script.string or 'GameDetailsEntity' not in script.string:
                continue
            text = script.string
            for m in pattern.finditer(text):
                thumb, category, w, h, region, full, size, full_w, full_h = m.groups()
                category_key = category.strip().lower()
                full_url = f"https://images.launchbox-app.com//{full}"

                if category_key not in images_by_category:
                    images_by_category[category_key] = []
                # Only add full-size URL to avoid duplicate downloads of different resolutions
                if full_url not in images_by_category[category_key]:
                    images_by_category[category_key].append(full_url)
            # Only need first matching script
            break
    
    except Exception:
        pass
    
    return images_by_category


def extract_game_metadata(soup: BeautifulSoup, game_info: Dict) -> Optional[Dict]:
    """Extract game metadata from LaunchBox details page.
    Returns dict with: name, nb_players, editor, year, description (with language keys).
    """
    try:
        if not soup:
            return None
        
        metadata = {
            'name': game_info.get('name', ''),
            'nb_players': 1,
            'editor': 'Unknown',
            'year': 0,
            'description': {
                'en': '',
                'fr': '',
                'de': '',
                'es': '',
                'it': ''
            }
        }
        
        def resolve_nuxt_value(value, data_array):
            """Recursively resolve NUXT data index references."""
            if isinstance(value, int) and 0 <= value < len(data_array):
                return resolve_nuxt_value(data_array[value], data_array)
            return value
        
        # Extract from NUXT data (primary method)
        try:
            nuxt_script = soup.find('script', id='__NUXT_DATA__')
            if nuxt_script and nuxt_script.string:
                data = json.loads(nuxt_script.string)
                
                # Find game detail entry (usually around index 5-10)
                for entry in data[:20]:
                    if isinstance(entry, dict) and 'gameDevelopers' in entry:
                        # Extract year
                        if 'releaseYear' in entry:
                            year = resolve_nuxt_value(entry['releaseYear'], data)
                            if isinstance(year, int):
                                metadata['year'] = year
                        
                        # Extract max players
                        if 'maxPlayers' in entry:
                            players = resolve_nuxt_value(entry['maxPlayers'], data)
                            if isinstance(players, int):
                                metadata['nb_players'] = players
                        
                        # Extract developer name
                        if 'gameDevelopers' in entry:
                            dev_list = resolve_nuxt_value(entry['gameDevelopers'], data)
                            if isinstance(dev_list, list) and dev_list:
                                first_dev = resolve_nuxt_value(dev_list[0], data)
                                if isinstance(first_dev, dict) and 'name' in first_dev:
                                    dev_name = resolve_nuxt_value(first_dev['name'], data)
                                    if dev_name and dev_name != 'Unknown':
                                        metadata['editor'] = dev_name
                        
                        # If no developer, try publisher
                        if metadata['editor'] == 'Unknown' and 'gamePublishers' in entry:
                            pub_list = resolve_nuxt_value(entry['gamePublishers'], data)
                            if isinstance(pub_list, list) and pub_list:
                                first_pub = resolve_nuxt_value(pub_list[0], data)
                                if isinstance(first_pub, dict) and 'name' in first_pub:
                                    pub_name = resolve_nuxt_value(first_pub['name'], data)
                                    if pub_name and pub_name != 'Unknown':
                                        metadata['editor'] = pub_name
                        
                        break
        except Exception:
            pass
        
        # Extract Overview description
        try:
            overview_heading = soup.find('h2', string=lambda s: s and 'Overview' in s)
            if overview_heading:
                description_parts = []
                current = overview_heading.find_next('p')
                while current:
                    if current.name == 'h2' or current.name == 'h3':
                        break
                    if current.name == 'p':
                        text = current.get_text(strip=True)
                        if text:
                            description_parts.append(text)
                    current = current.find_next_sibling()
                
                overview_text = '\n\n'.join(description_parts)
                overview_text = ' '.join(overview_text.split())
                
                for lang in metadata['description']:
                    metadata['description'][lang] = overview_text
        except Exception:
            pass
        
        return metadata
    
    except Exception:
        return None


def save_game_metadata(metadata: Dict, unique_name: str, output_dir: Path) -> bool:
    """Save game metadata to JSON file.
    Skips if file already exists.
    Returns True if successful.
    """
    try:
        output_path = output_dir / f"{unique_name}.json"
        
        if output_path.exists():
            return True
        
        output_dir.mkdir(parents=True, exist_ok=True)
        
        with open(output_path, 'w', encoding='utf-8') as f:
            json.dump(metadata, f, indent=2, ensure_ascii=False)
        
        return True
    
    except Exception:
        return False


def match_category(category: str, mapping: Dict[str, str]) -> Optional[Tuple[str, str]]:
    """Match category name with mapping (case-insensitive partial matching).
    Returns tuple of (matched_key, suffix) or None.
    Prefers RECONSTRUCTED versions."""
    category_lower = category.lower()
    
    # First pass: look for reconstructed versions
    for key, suffix in mapping.items():
        if key in category_lower and 'reconstructed' in category_lower:
            return (key, suffix)
    
    # Second pass: match without reconstructed requirement
    for key, suffix in mapping.items():
        if key in category_lower:
            return (key, suffix)
    
    return None


def download_game_images(game_info: Dict, unique_name: str, game_checksums: Dict[str, List[str]], output_dir: Path) -> bool:
    """Download all images for a game to specified output directory. Returns True if successful."""
    try:
        def pick_first_image(image_urls: List[str]) -> Optional[Image.Image]:
            """Download and return the first available image."""
            for candidate_url in image_urls:
                random_delay()
                img = download_image(candidate_url)
                if img:
                    return img
            return None

        print(f"         Fetching images page for {game_info['name']}...", flush=True)
        # Visit game images page
        images_url = f"{BASE_URL}/games/images/{game_info['id']}-{game_info['slug']}"
        random_delay()
        images_soup = get_page(images_url)

        print("         Fetching details page for metadata...", flush=True)
        # Also visit details page to get embedded JSON that contains full-size URLs
        details_url = f"{BASE_URL}/games/details/{game_info['id']}-{game_info['slug']}"
        random_delay()
        details_soup = get_page(details_url)
        
        if not images_soup and not details_soup:
            print("         No images or details available.", flush=True)
            return False
        
        # Extract and save game metadata from details page
        if details_soup:
            metadata = extract_game_metadata(details_soup, game_info)
            if metadata:
                save_game_metadata(metadata, unique_name, output_dir)
        
        # Get or initialize game checksum list
        if unique_name not in game_checksums:
            game_checksums[unique_name] = []
        
        checksums_list = game_checksums[unique_name]
        
        # Extract images by category from both pages and merge (put full-size from details first, then append from images)
        images_by_category = {}
        # Process details_soup first so full-size URLs from embedded JSON are placed first
        for source_soup in (details_soup, images_soup):
            if not source_soup:
                continue
            extracted = extract_images_from_game_page(source_soup)
            for cat, urls in extracted.items():
                if cat not in images_by_category:
                    # First source to contribute to this category: insert first URL at front, rest after
                    images_by_category[cat] = []
                    for u in urls:
                        if u not in images_by_category[cat]:
                            images_by_category[cat].append(u)
                else:
                    # Category already exists from prior source: only add new URLs at the end
                    for u in urls:
                        if u not in images_by_category[cat]:
                            images_by_category[cat].append(u)
        
        # Track gameplay screenshots count
        gameplay_count = 0
        
        # Group categories by their mapping suffix, preferring reconstructed versions
        category_to_process = {}  # suffix -> category to use
        for category, image_urls in images_by_category.items():
            match = match_category(category, CATEGORY_MAPPING)
            if not match:
                continue
            
            key, suffix = match
            
            # For screenshots, process all; for others, keep only the best (reconstructed or highest quality)
            if suffix == "_snap":
                # Screenshots are handled separately below
                pass
            else:
                # For non-screenshot categories, prefer reconstructed versions
                if suffix not in category_to_process:
                    category_to_process[suffix] = category
                else:
                    # If this category is reconstructed and current isn't, or if current beats existing, update
                    existing_cat = category_to_process[suffix]
                    if 'reconstructed' in category.lower() and 'reconstructed' not in existing_cat.lower():
                        category_to_process[suffix] = category
        
        # Process selected categories
        for category, image_urls in images_by_category.items():
            match = match_category(category, CATEGORY_MAPPING)
            if not match:
                continue
            
            key, suffix = match
            print(f"         Category '{category}' ({len(image_urls)} urls) -> suffix '{suffix}'", flush=True)
            
            # Special handling for gameplay screenshots - process all
            if suffix == "_snap":
                for img_url in image_urls:
                    random_delay()
                    img = download_image(img_url)
                    if img:
                        # Compute checksum of this image
                        checksum = compute_image_checksum(img)
                        
                        # Skip if we've already saved this exact image for this game
                        if checksum and checksum in checksums_list:
                            continue
                        
                        # Increment counter and check if file already exists
                        gameplay_count += 1
                        snap_path = output_dir / f"{unique_name}_snap{gameplay_count}.png"
                        if snap_path.exists():
                            continue
                        
                        # Save the screenshot
                        save_image(img, snap_path, suffix="_snap")
                        
                        # Cache the checksum
                        if checksum:
                            checksums_list.append(checksum)
            else:
                # For other categories, only process if this is the selected category for this suffix
                if category_to_process.get(suffix) == category and image_urls:
                    # Check if main file already exists
                    main_filepath = output_dir / f"{unique_name}{suffix}"
                    img = None
                    if not main_filepath.exists():
                        img = pick_first_image(image_urls)
                        if img:
                            save_image(img, main_filepath, suffix=suffix)
                    
                    # Load image if not already loaded
                    if img is None and main_filepath.exists():
                        img = load_image_from_path(main_filepath)
                    
                    if img:
                        # Create derivatives based on suffix
                        if suffix == "_big_overlay.png":
                            print(f"         Creating overlay composite for {unique_name}...", flush=True)
                            if composite_overlay(img, unique_name, output_dir):
                                print(f"         ✓ Overlay composite created", flush=True)
                            else:
                                print(f"         ✗ Overlay composite failed", flush=True)
                        elif suffix == ".png":  # box - front
                            save_image(img.copy(), output_dir / f"{unique_name}_small.png", suffix="_small.png")
        
        return True
    
    except Exception:
        return False


def load_game_pages(max_pages: int = 20) -> Dict[int, List[Dict]]:
    """Fetch game list pages once and cache results until a page returns no games."""
    cache: Dict[int, List[Dict]] = {}
    print("Caching LaunchBox game pages...")
    for page in range(1, max_pages + 1):
        print(f"  -> Fetching page {page}", flush=True)
        url = f"{PLATFORM_URL}?page={page}"
        soup = get_page(url)
        if not soup:
            print("     (stop: failed to fetch page)", flush=True)
            break

        games = extract_games_from_page(soup)
        if not games:
            print("     (stop: no games on this page)", flush=True)
            break

        cache[page] = games
        print(f"     cached {len(games)} games", flush=True)
        random_delay()

    print(f"Cached {sum(len(v) for v in cache.values())} games across {len(cache)} pages")
    return cache


def search_for_game_by_name(game_name: str, page_cache: Dict[int, List[Dict]]) -> Optional[Dict]:
    """Search for a game by name using cached pages. Returns game_info or None."""
    print(f"  Searching cache for: {game_name}...", flush=True)
    candidates = generate_search_variants(game_name)

    for idx, candidate in enumerate(candidates):
        if idx > 0:
            print(f"    -> Retry with alternate title: {candidate}", flush=True)

        for page, games in page_cache.items():
            print(f"    -> Page {page} (cached, {len(games)} games)", flush=True)
            
            # Look for exact or partial match
            for game in games:
                if (game['name'].lower() == candidate.lower() or 
                    candidate.lower() in game['name'].lower() or 
                    game['name'].lower() in candidate.lower()):
                    print(f" ✓ Found: {game['name']}")
                    return game

    print(" ✗ Not found in cached pages")
    return None


def main():
    """Main scraping function for ROM directory."""
    parser = argparse.ArgumentParser(description="Scrape Mattel Intellivision game images from LaunchBox Games Database")
    parser.add_argument('--roms-dir', type=str, default=None, help='Path to ROMs directory (default: prompts user or ./roms/)')
    parser.add_argument('-v', '--version', action='store_true', help='Print script version and exit')
    args = parser.parse_args()

    # Handle version flag
    if getattr(args, 'version', False):
        print(f"create_sprint_files.py {VERSION}")
        return
    
    # Get ROMs directory
    if args.roms_dir:
        roms_dir = Path(args.roms_dir)
    else:
        roms_dir = get_roms_directory()
    
    if not roms_dir.exists():
        print(f"ERROR: ROMs directory does not exist: {roms_dir}")
        return
    
    # Get ROM files
    rom_files = get_rom_files(roms_dir)
    if not rom_files:
        print(f"No ROM files found in {roms_dir}")
        print(f"Supported extensions: .bin, .int, .rom, .zip, .7z")
        return
    
    print(f"Found {len(rom_files)} ROM files")
    print()
    
    # Load memory map for filename -> title matching
    memory_map = load_memory_map()

    # Cache all LaunchBox pages once so we don't fetch non-existent pages repeatedly
    page_cache = load_game_pages(max_pages=20)
    if not page_cache:
        print("ERROR: Could not cache any LaunchBox pages")
        return
    
    # Load asset cache
    asset_cache = load_asset_cache()
    print(f"Loaded asset cache with {len(asset_cache)} cached games\n")

    # Initialize tracking
    game_checksums = {}
    games_found = 0
    games_no_assets = []  # Track games where assets couldn't be found
    games_not_found = []  # Track games not resolved on LaunchBox
    rom_count = 0
    
    # Process each ROM
    for rom_path in rom_files:
        rom_count += 1
        rom_name = normalize_rom_name(rom_path)
        output_name = sanitize_filename(rom_path.stem)
        
        print(f"[{rom_count}] Processing: {rom_path.name}")
        
        # Choose search title based on memory map when available
        mapped_info = resolve_rom_to_title(rom_path, memory_map)
        if mapped_info:
            search_title, mapper_num = mapped_info
            print(f"       Using memory map title: {search_title} (Mapper {mapper_num})")
        else:
            search_title = rom_name
            mapper_num = 0  # Default mapper
            print(f"       Using normalized filename: {search_title} (Default Mapper {mapper_num})")

        # Search for game on LaunchBox
        game_info = search_for_game_by_name(search_title, page_cache)
        
        if not game_info:
            print(f"       WARNING: Not found on LaunchBox, skipping")
            games_not_found.append(rom_path.name)
            continue
        
        # Use ROM filename (stem) for asset naming
        unique_name = output_name
        print(f"       Downloading images for: {game_info['name']} -> saving as '{unique_name}'")
        
        # Check if we have cached assets for this game
        game_title_key = game_info['name']
        if game_title_key in asset_cache:
            print(f"       ✓ Found in asset cache, copying cached files...", flush=True)
            copied = copy_cached_assets(asset_cache[game_title_key], unique_name, roms_dir)
            if copied or assets_complete(unique_name, roms_dir):
                if copied:
                    print(f"       ✓ Assets copied from cache")
                else:
                    print(f"       ✓ Assets already present; skipping download")
                games_found += 1
                # Generate .cfg file
                if generate_cfg_file(unique_name, mapper_num, roms_dir):
                    print(f"       ✓ Complete (with .cfg)")
                else:
                    print(f"       ✓ Complete (no .cfg)")
                continue
            else:
                print(f"       WARNING: Cache entry found but assets missing; will download remaining assets")
        
        # Download images to ROM directory
        success = download_game_images(game_info, unique_name, game_checksums, roms_dir)
        
        if success:
            games_found += 1
            # Register the assets in the cache
            register_game_assets(game_title_key, unique_name, roms_dir, asset_cache)
            # Generate .cfg file
            if generate_cfg_file(unique_name, mapper_num, roms_dir):
                print(f"       ✓ Complete (with .cfg)")
            else:
                print(f"       ✓ Complete (no .cfg)")
        else:
            print(f"       ✗ Failed to download images")
            games_no_assets.append(game_info['name'])
    
    # Save asset cache before exiting
    save_asset_cache(asset_cache)
    
    print(f"\n=== PROCESSING COMPLETE ===")
    print(f"Total ROMs processed: {rom_count}")
    print(f"Games found and processed: {games_found}")
    print(f"Output directory: {roms_dir}")
    print(f"Asset cache saved with {len(asset_cache)} entries")
    
    if games_not_found:
        print(f"\n⚠ Games not found on LaunchBox ({len(games_not_found)}):")
        for rom_name in games_not_found:
            print(f"  - {rom_name}")
    
    if games_no_assets:
        print(f"\n⚠ Games with no assets found ({len(games_no_assets)}):")
        for game_name in games_no_assets:
            print(f"  - {game_name}")


if __name__ == "__main__":
    main()