run_scrapers.py

import sys
import argparse
import importlib
from core import config
from core.notifier import send_discord_scraper_started, send_discord_scraper_finished


def run_scrapers(city=None, max_price=None, scrapers=None):
    city = city or config.CITY
    max_price = max_price or config.MAX_PRICE
    scrapers = scrapers or config.get_enabled_scrapers()
    
    print(f"=== Housing Scraper ===")
    print(f"City: {city}")
    print(f"Max price: €{max_price}")
    print(f"Scrapers: {', '.join(scrapers)}")
    print()
    
    # send start notification
    if config.NOTIFICATIONS_ENABLED and config.NOTIFY_START:
        try:
            send_discord_scraper_started()
        except Exception as e:
            if config.DEBUG_MODE:
                print(f"Failed to send start notification: {e}")
    
    total_new = 0
    results = {}
    
    # run each enabled scraper
    for scraper_name in scrapers:
        if scraper_name not in config.SCRAPERS or not config.SCRAPERS[scraper_name]:
            print(f"Skipping {scraper_name} (disabled)")
            continue
        
        try:
            # dynamic import
            scraper_module = importlib.import_module(f"scrapers.{scraper_name}")
            new_count = scraper_module.scrape(city, max_price)
            
            results[scraper_name] = new_count
            total_new += new_count
            
        except ImportError:
            print(f"Scraper {scraper_name} not available")
            results[scraper_name] = 0
        except Exception as e:
            print(f"Error running {scraper_name}: {e}")
            results[scraper_name] = 0
    
    print()
    print("=== Summary ===")
    print(f"Total new listings: {total_new}")
    for scraper_name, count in results.items():
        print(f"  {scraper_name}: {count}")
    
    if config.NOTIFICATIONS_ENABLED and config.NOTIFY_FINISH:
        try:
            send_discord_scraper_finished()
        except Exception as e:
            if config.DEBUG_MODE:
                print(f"Failed to send finish notification: {e}")
    
    return total_new


def main():
    parser = argparse.ArgumentParser(description="Housing Scraper")
    parser.add_argument("--city", help="City to search in")
    parser.add_argument("--max-price", type=int, help="Maximum price")
    parser.add_argument("--scrapers", nargs="+", help="Specific scrapers to run")
    parser.add_argument("--list", action="store_true", help="List available scrapers")
    parser.add_argument("--debug", action="store_true", help="Enable debug mode")
    
    args = parser.parse_args()
    
    if args.debug:
        config.DEBUG_MODE = True
    
    if args.list:
        print("Available scrapers:")
        for name, enabled in config.SCRAPERS.items():
            status = "enabled" if enabled else "disabled"
            print(f"  {name} ({status})")
        return
    
    try:
        total = run_scrapers(args.city, args.max_price, args.scrapers)
        if total > 0:
            print(f"\n✅ Found {total} new listings!")
        else:
            print("\nℹ️ No new listings found.")
    except KeyboardInterrupt:
        print("\n⏹️ Interrupted by user")
        sys.exit(1)
    except Exception as e:
        print(f"\n❌ Error: {e}")
        if config.DEBUG_MODE:
            import traceback
            traceback.print_exc()
        sys.exit(1)


if __name__ == "__main__":
    main()