data_gathering_plan.json

{
  "metadata": {
    "generated": "2025-11-14 16:02:55 UTC",
    "purpose": "Hypothesis validation data gathering",
    "source_analysis": "devils_advocate_analysis.md"
  },
  "firecrawl_searches": [
    {
      "outbreak": "Measles Outbreak in the United States",
      "search_queries": [
        {
          "query": "Measles case reporting changes 2025 USA",
          "purpose": "Identify changes in reporting practices",
          "expected_data": "Documentation of reporting changes",
          "priority": "high"
        },
        {
          "query": "Measles surveillance bias USA 2025",
          "purpose": "Assess surveillance bias",
          "expected_data": "Comparative data from different jurisdictions",
          "priority": "medium"
        },
        {
          "query": "Measles misclassification viral exanthems USA",
          "purpose": "Investigate misclassification of cases",
          "expected_data": "Reports of misdiagnosis",
          "priority": "medium"
        },
        {
          "query": "Measles laboratory contamination false positives",
          "purpose": "Check for laboratory errors",
          "expected_data": "Laboratory audit reports",
          "priority": "high"
        },
        {
          "query": "Measles media coverage influence 2025 USA",
          "purpose": "Analyze media influence on reporting",
          "expected_data": "Correlation between media coverage and case reports",
          "priority": "medium"
        },
        {
          "query": "Measles vaccination records USA 2025",
          "purpose": "Verify vaccination status accuracy",
          "expected_data": "Vaccination records and case correlation",
          "priority": "high"
        },
        {
          "query": "Measles vs other viral infections USA 2025",
          "purpose": "Compare trends with other infections",
          "expected_data": "Comparative infection data",
          "priority": "medium"
        },
        {
          "query": "Measles geospatial analysis USA 2025",
          "purpose": "Identify non-disease-related patterns",
          "expected_data": "Geospatial distribution of cases",
          "priority": "low"
        }
      ]
    },
    {
      "outbreak": "Monkeypox Outbreak in Central and Eastern Africa",
      "search_queries": [
        {
          "query": "Monkeypox historical case data Africa",
          "purpose": "Assess underreporting in previous years",
          "expected_data": "Historical case data",
          "priority": "high"
        },
        {
          "query": "Environmental changes wildlife interactions Africa",
          "purpose": "Investigate environmental changes",
          "expected_data": "Environmental impact reports",
          "priority": "medium"
        },
        {
          "query": "Healthcare access monkeypox reporting Africa",
          "purpose": "Correlate healthcare access with reporting",
          "expected_data": "Healthcare access data",
          "priority": "medium"
        },
        {
          "query": "Monkeypox cross-reactivity testing Africa",
          "purpose": "Assess cross-reactivity in testing",
          "expected_data": "Laboratory test results",
          "priority": "high"
        },
        {
          "query": "Economic incentives monkeypox reporting Africa",
          "purpose": "Identify economic reporting biases",
          "expected_data": "Funding and resource allocation reports",
          "priority": "low"
        },
        {
          "query": "Wildlife population surveys Africa",
          "purpose": "Assess changes in wildlife populations",
          "expected_data": "Wildlife survey data",
          "priority": "medium"
        }
      ]
    },
    {
      "outbreak": "Avian Influenza in Mongolia",
      "search_queries": [
        {
          "query": "Avian influenza seasonal patterns Mongolia",
          "purpose": "Identify seasonal migration patterns",
          "expected_data": "Historical seasonal data",
          "priority": "high"
        },
        {
          "query": "Avian influenza diagnostic test validation Mongolia",
          "purpose": "Validate diagnostic test accuracy",
          "expected_data": "Test sensitivity and specificity data",
          "priority": "high"
        },
        {
          "query": "Poultry trade patterns Mongolia",
          "purpose": "Analyze trade dynamics",
          "expected_data": "Trade data and outbreak correlation",
          "priority": "medium"
        },
        {
          "query": "Weather patterns avian influenza Mongolia",
          "purpose": "Correlate weather with outbreak",
          "expected_data": "Weather data and outbreak timing",
          "priority": "medium"
        },
        {
          "query": "Wild bird monitoring avian influenza Mongolia",
          "purpose": "Monitor migratory bird populations",
          "expected_data": "Bird population and virus prevalence data",
          "priority": "medium"
        }
      ]
    },
    {
      "outbreak": "Tuberculosis Surge in Indonesia",
      "search_queries": [
        {
          "query": "TB diagnostic capacity Indonesia 2025",
          "purpose": "Evaluate diagnostic capabilities",
          "expected_data": "Diagnostic and screening practice data",
          "priority": "high"
        },
        {
          "query": "Population movement TB Indonesia 2025",
          "purpose": "Correlate migration with TB trends",
          "expected_data": "Migration and TB case data",
          "priority": "medium"
        },
        {
          "query": "Healthcare policy TB reporting Indonesia",
          "purpose": "Assess policy impact on reporting",
          "expected_data": "Policy change and reporting data",
          "priority": "medium"
        },
        {
          "query": "Economic incentives TB reporting Indonesia",
          "purpose": "Investigate economic reporting biases",
          "expected_data": "Economic data and reporting trends",
          "priority": "low"
        },
        {
          "query": "TB vs other respiratory illnesses Indonesia",
          "purpose": "Compare TB with other illnesses",
          "expected_data": "Comparative clinical data",
          "priority": "medium"
        }
      ]
    }
  ],
  "urls_to_scrape": [
    {
      "outbreak": "Measles Outbreak in the United States",
      "urls": [
        {
          "url": "https://www.cdc.gov/measles/index.html",
          "source_type": "CDC",
          "data_type": "Official case reports and guidelines",
          "validates": "Data reporting artifacts",
          "scraping_notes": "Check for updates on reporting practices"
        },
        {
          "url": "https://www.who.int/news-room/fact-sheets/detail/measles",
          "source_type": "WHO",
          "data_type": "Global measles data and trends",
          "validates": "Surveillance bias",
          "scraping_notes": "Compare global vs. US trends"
        },
        {
          "url": "https://promedmail.org/",
          "source_type": "ProMED",
          "data_type": "Outbreak reports and alerts",
          "data_type": "Real-time outbreak monitoring",
          "validates": "Geospatial analysis",
          "scraping_notes": "Map case distribution"
        },
        {
          "url": "https://www.ncbi.nlm.nih.gov/pmc/",
          "source_type": "Academic",
          "data_type": "Research articles on measles",
          "validates": "Laboratory contamination",
          "scraping_notes": "Focus on diagnostic accuracy studies"
        },
        {
          "url": "https://www.nytimes.com/section/health",
          "source_type": "News",
          "data_type": "Media coverage analysis",
          "validates": "Media influence",
          "scraping_notes": "Track coverage timeline"
        },
        {
          "url": "https://www.census.gov/data.html",
          "source_type": "Government",
          "data_type": "Demographic and population data",
          "validates": "Geospatial analysis",
          "scraping_notes": "Correlate with case data"
        }
      ]
    },
    {
      "outbreak": "Monkeypox Outbreak in Central and Eastern Africa",
      "urls": [
        {
          "url": "https://www.who.int/emergencies/disease-outbreak-news",
          "source_type": "WHO",
          "data_type": "Official outbreak news",
          "validates": "Underreporting in previous years",
          "scraping_notes": "Check historical outbreak reports"
        },
        {
          "url": "https://www.cdc.gov/poxvirus/monkeypox/index.html",
          "source_type": "CDC",
          "data_type": "Monkeypox case data and guidelines",
          "validates": "Cross-reactivity in testing",
          "scraping_notes": "Focus on testing protocols"
        },
        {
          "url": "https://journals.plos.org/plosntds/",
          "source_type": "Academic",
          "data_type": "Research on neglected tropical diseases",
          "validates": "Environmental changes",
          "scraping_notes": "Search for environmental impact studies"
        },
        {
          "url": "https://www.afro.who.int/health-topics/monkeypox",
          "source_type": "WHO Regional Office for Africa",
          "data_type": "Regional case data",
          "validates": "Healthcare access",
          "scraping_notes": "Compare regional healthcare access"
        },
        {
          "url": "https://www.worldbank.org/en/topic/health",
          "source_type": "Government",
          "data_type": "Healthcare funding and resources",
          "validates": "Economic incentives",
          "scraping_notes": "Analyze funding structures"
        }
      ]
    },
    {
      "outbreak": "Avian Influenza in Mongolia",
      "urls": [
        {
          "url": "https://www.oie.int/en/animal-health-in-the-world/avian-influenza-portal/",
          "source_type": "OIE",
          "data_type": "Avian influenza reports",
          "validates": "Seasonal migration patterns",
          "scraping_notes": "Check for seasonal data"
        },
        {
          "url": "https://www.fao.org/ag/againfo/programmes/en/empres/avianflu/home.html",
          "source_type": "FAO",
          "data_type": "Poultry trade and biosecurity",
          "validates": "Poultry trade dynamics",
          "scraping_notes": "Focus on trade reports"
        },
        {
          "url": "https://journals.asm.org/journal/jvi",
          "source_type": "Academic",
          "data_type": "Virology research articles",
          "validates": "Diagnostic sensitivity",
          "scraping_notes": "Search for test validation studies"
        },
        {
          "url": "https://www.wmo.int/pages/index_en.html",
          "source_type": "Government",
          "data_type": "Weather and climate data",
          "validates": "Climate variability",
          "scraping_notes": "Correlate weather with outbreaks"
        },
        {
          "url": "https://www.birdlife.org/",
          "source_type": "NGO",
          "data_type": "Wild bird population data",
          "validates": "Wild bird monitoring",
          "scraping_notes": "Focus on migratory patterns"
        }
      ]
    },
    {
      "outbreak": "Tuberculosis Surge in Indonesia",
      "urls": [
        {
          "url": "https://www.who.int/teams/global-tuberculosis-programme",
          "source_type": "WHO",
          "data_type": "Global TB data and guidelines",
          "validates": "Improved detection",
          "scraping_notes": "Check for diagnostic advancements"
        },
        {
          "url": "https://www.cdc.gov/tb/statistics/default.htm",
          "source_type": "CDC",
          "data_type": "TB statistics and trends",
          "validates": "Population movement",
          "scraping_notes": "Correlate with migration data"
        },
        {
          "url": "https://www.ncbi.nlm.nih.gov/pmc/",
          "source_type": "Academic",
          "data_type": "Research on TB and respiratory illnesses",
          "validates": "Non-TB respiratory illnesses",
          "scraping_notes": "Focus on misclassification studies"
        },
        {
          "url": "https://www.worldbank.org/en/country/indonesia",
          "source_type": "Government",
          "data_type": "Economic and healthcare policy data",
          "validates": "Healthcare policy changes",
          "scraping_notes": "Analyze policy impact on reporting"
        },
        {
          "url": "https://www.unaids.org/en/regionscountries/countries/indonesia",
          "source_type": "NGO",
          "data_type": "Healthcare access and utilization",
          "validates": "Healthcare access survey",
          "scraping_notes": "Focus on access trends"
        }
      ]
    }
  ],
  "validation_data_requirements": {
    "baseline_data": [
      "Historical measles case data from CDC and WHO",
      "Historical monkeypox case data from WHO",
      "Seasonal avian influenza data from OIE",
      "Historical TB data from WHO and CDC"
    ],
    "control_data": [
      "Measles data from regions without enhanced surveillance",
      "Monkeypox data from regions with stable healthcare access",
      "Avian influenza data from regions with stable poultry trade",
      "TB data from regions with stable population movement"
    ],
    "temporal_data": [
      "Time-series data for measles case reports in 2025",
      "Time-series data for monkeypox case reports in 2025",
      "Time-series data for avian influenza outbreaks in 2025",
      "Time-series data for TB case reports in 2025"
    ],
    "laboratory_data": [
      "Measles diagnostic test validation reports",
      "Monkeypox cross-reactivity test results",
      "Avian influenza diagnostic test performance data",
      "TB diagnostic capacity and screening practice data"
    ]
  },
  "scraping_strategy": {
    "priority_order": [
      "CDC and WHO official pages",
      "ProMED and HealthMap reports",
      "Academic journals and preprint servers",
      "Government statistics databases",
      "News aggregators for media analysis",
      "Environmental and demographic data sources"
    ],
    "frequency": "Weekly for outbreak updates, monthly for historical data",
    "depth": "Crawl linked pages up to 2 levels deep",
    "filters": "Exclude unrelated news articles and non-peer-reviewed content"
  }
}