From 57b4cc4bbdc063800514906a39e5e02c59dcd109 Mon Sep 17 00:00:00 2001 From: Gitanaskhan26 <87142370+Gitanaskhan26@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:24:46 +0530 Subject: [PATCH] Add News dataset simulator generation notebook --- news/News_Dataset_Generation.ipynb | 311 +++++++++++++++++++++++++++++ 1 file changed, 311 insertions(+) create mode 100644 news/News_Dataset_Generation.ipynb diff --git a/news/News_Dataset_Generation.ipynb b/news/News_Dataset_Generation.ipynb new file mode 100644 index 0000000..04764ca --- /dev/null +++ b/news/News_Dataset_Generation.ipynb @@ -0,0 +1,311 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# News Dataset Generation\n", + "\n", + "This notebook outlines the data preparation process for the `news_nytimes_lda.npz` file, which is used by the `NewsDataset` simulator in `pgmpy`. \n", + "\n", + "The dataset simulates a media consumer's reading experience of news articles under different viewing devices (desktop vs. mobile), implementing the benchmark introduced by Johansson et al. (2016).\n", + "\n", + "The process includes:\n", + "1. Downloading the NY Times bag-of-words corpus from the UCI Machine Learning Repository (300,000 articles).\n", + "2. Parsing the data into memory-efficient sparse matrices.\n", + "3. Fitting Latent Dirichlet Allocation (LDA) models for $K=50$ and $K=200$ topics.\n", + "4. Extracting the vocabulary and visualizing the learned topics.\n", + "5. Saving the required arrays into an `.npz` format for hosting.\n", + "\n", + "*Note: Generating this dataset requires fitting LDA on 300,000 documents. Running this notebook sequentially requires approximately 4-8 GB of RAM and may take 30-60 minutes depending on hardware.*\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import gzip\n", + "import logging\n", + "import os\n", + "import time\n", + "import urllib.error\n", + "import urllib.request\n", + "from pathlib import Path\n", + "\n", + "import numpy as np\n", + "from scipy import sparse\n", + "from sklearn.decomposition import LatentDirichletAllocation\n", + "\n", + "logging.basicConfig(level=logging.INFO, format=\"%(asctime)s [%(levelname)s] %(message)s\")\n", + "log = logging.getLogger(__name__)\n", + "\n", + "# Constants\n", + "UCI_BASE = \"https://archive.ics.uci.edu/ml/machine-learning-databases/bag-of-words\"\n", + "DOCWORD_URL = f\"{UCI_BASE}/docword.nytimes.txt.gz\"\n", + "VOCAB_URL = f\"{UCI_BASE}/vocab.nytimes.txt\"\n", + "CACHE_DIR = Path(\"_cache_news_prep\")\n", + "\n", + "SEED = 42\n", + "OUTPUT_FILE = \"news_nytimes_lda.npz\" \n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Download Raw Data\n", + "Fetch the raw gzipped word counts and vocabulary list from the UCI Machine Learning Repository.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def download_file(url: str, dest: Path) -> Path:\n", + " if dest.exists():\n", + " log.info(f\"Using cached: {dest}\")\n", + " return dest\n", + " dest.parent.mkdir(parents=True, exist_ok=True)\n", + " log.info(f\"Downloading {url} ...\")\n", + " try:\n", + " urllib.request.urlretrieve(url, dest)\n", + " except urllib.error.URLError as e:\n", + " if \"CERTIFICATE_VERIFY_FAILED\" in str(e):\n", + " import ssl\n", + " log.warning(\"SSL verification failed, retrying with unverified context ...\")\n", + " ctx = ssl.create_default_context()\n", + " ctx.check_hostname = False\n", + " ctx.verify_mode = ssl.CERT_NONE\n", + " opener = urllib.request.build_opener(urllib.request.HTTPSHandler(context=ctx))\n", + " with opener.open(url) as response, open(dest, \"wb\") as out_file:\n", + " import shutil\n", + " shutil.copyfileobj(response, out_file)\n", + " else:\n", + " raise\n", + " log.info(f\"Saved to {dest} ({dest.stat().st_size / 1e6:.1f} MB)\")\n", + " return dest\n", + "\n", + "docword_path = download_file(DOCWORD_URL, CACHE_DIR / \"docword.nytimes.txt.gz\")\n", + "vocab_path = download_file(VOCAB_URL, CACHE_DIR / \"vocab.nytimes.txt\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Parsing the Dataset\n", + "The UCI docword format is parsed directly into a `scipy.sparse.csr_matrix` for memory efficiency.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def load_vocabulary(vocab_path: Path) -> np.ndarray:\n", + " log.info(\"Loading vocabulary ...\")\n", + " with open(vocab_path) as f:\n", + " words = [line.strip() for line in f if line.strip()]\n", + " log.info(f\"Vocabulary size: {len(words)}\")\n", + " return np.array(words)\n", + "\n", + "def load_docword_sparse(docword_path: Path) -> sparse.csr_matrix:\n", + " log.info(f\"Parsing sparse matrix from {docword_path} ...\")\n", + " t0 = time.time()\n", + " open_fn = gzip.open if str(docword_path).endswith(\".gz\") else open\n", + " \n", + " with open_fn(docword_path, \"rt\") as f:\n", + " D = int(f.readline().strip())\n", + " W = int(f.readline().strip())\n", + " NNZ = int(f.readline().strip())\n", + " log.info(f\" D={D:,}, W={W:,}, NNZ={NNZ:,}\")\n", + " \n", + " rows = np.empty(NNZ, dtype=np.int32)\n", + " cols = np.empty(NNZ, dtype=np.int32)\n", + " vals = np.empty(NNZ, dtype=np.int32)\n", + " \n", + " for i, line in enumerate(f):\n", + " parts = line.split()\n", + " rows[i] = int(parts[0]) - 1\n", + " cols[i] = int(parts[1]) - 1\n", + " vals[i] = int(parts[2])\n", + " if (i + 1) % 20_000_000 == 0:\n", + " log.info(f\" ... parsed {i + 1:,}/{NNZ:,} entries\")\n", + " \n", + " mat = sparse.coo_matrix((vals, (rows, cols)), shape=(D, W)).tocsr()\n", + " elapsed = time.time() - t0\n", + " log.info(f\" Sparse matrix shape: {mat.shape}, nnz: {mat.nnz:,}, parsed in {elapsed:.1f}s\")\n", + " return mat\n", + "\n", + "full_vocab = load_vocabulary(vocab_path)\n", + "word_counts_full = load_docword_sparse(docword_path)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Topic Modeling (LDA)\n", + "Latent Dirichlet Allocation (LDA) models are fit to the dataset.\n", + "* **$K=50$**: The standard hyperparameter used in the original benchmarking paper.\n", + "* **$K=200$**: Pre-computed to allow `pgmpy` to use agglomerative clustering for dynamically merging topics on the fly (supporting $K \\le 200$).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def fit_lda(word_counts: sparse.csr_matrix, n_components: int, seed: int) -> tuple:\n", + " log.info(f\"Fitting LDA with K={n_components} (batch mode, random_state={seed}) ...\")\n", + " t0 = time.time()\n", + " lda = LatentDirichletAllocation(\n", + " n_components=n_components,\n", + " learning_method=\"batch\",\n", + " random_state=seed,\n", + " n_jobs=1,\n", + " verbose=10,\n", + " )\n", + " topic_dists = lda.fit_transform(word_counts)\n", + " elapsed = time.time() - t0\n", + " log.info(f\" LDA K={n_components} fit in {elapsed:.1f}s ({elapsed / 60:.1f} min)\")\n", + " return topic_dists, lda.components_\n", + "\n", + "topic_dists_k50, lda_components_k50 = fit_lda(word_counts_full, n_components=50, seed=SEED)\n", + "topic_dists_k200, lda_components_k200 = fit_lda(word_counts_full, n_components=200, seed=SEED)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Vocabulary Selection\n", + "The covariate vocabulary is constructed by taking the union of the top 100 words from each of the 50 baseline topics.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def select_vocabulary(lda_components: np.ndarray, top_n_words: int = 100) -> np.ndarray:\n", + " top_words_per_topic = np.argsort(lda_components, axis=1)[:, -top_n_words:]\n", + " vocab_indices = np.unique(top_words_per_topic.flatten())\n", + " log.info(\n", + " f\" Selected {len(vocab_indices)} unique words from top-{top_n_words} across {lda_components.shape[0]} topics\"\n", + " )\n", + " return vocab_indices\n", + "\n", + "vocab_indices = select_vocabulary(lda_components_k50, top_n_words=100)\n", + "vocab_3477 = full_vocab[vocab_indices]\n", + "word_counts_3477 = word_counts_full[:, vocab_indices]\n", + "\n", + "log.info(f\"Final Vocabulary shape: {vocab_3477.shape}\")\n", + "log.info(f\"Subset word counts shape: {word_counts_3477.shape}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Visualizing the Learned Topics\n", + "Display the top words for the first 10 topics to verify semantic coherence.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def display_topics(components, feature_names, no_top_words, n_topics=10):\n", + " for topic_idx, topic in enumerate(components[:n_topics]):\n", + " print(f\"Topic {topic_idx + 1}:\")\n", + " print(\" \".join([feature_names[i] for i in topic.argsort()[:-no_top_words - 1:-1]]))\n", + " print()\n", + "\n", + "display_topics(lda_components_k50, full_vocab, 15, n_topics=10)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. Saving the NPZ File\n", + "The data is packaged into a compressed `.npz` file. Since `np.savez_compressed` does not natively support `scipy.sparse` matrices, they are deconstructed into their underlying CSR arrays (`data`, `indices`, `indptr`).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Convert to float32 for storage efficiency\n", + "topic_dists_k50 = topic_dists_k50.astype(np.float32)\n", + "topic_dists_k200 = topic_dists_k200.astype(np.float32)\n", + "lda_components_k50 = lda_components_k50.astype(np.float32)\n", + "lda_components_k200 = lda_components_k200.astype(np.float32)\n", + "\n", + "log.info(f\"Saving to {OUTPUT_FILE} ...\")\n", + "t0 = time.time()\n", + "\n", + "np.savez_compressed(\n", + " OUTPUT_FILE,\n", + " # Precomputed topic distributions\n", + " topic_distributions_k50=topic_dists_k50,\n", + " topic_distributions_k200=topic_dists_k200,\n", + " # LDA components\n", + " lda_components_k50=lda_components_k50,\n", + " lda_components_k200=lda_components_k200,\n", + " # Precomputed vocabulary\n", + " vocab_3477=vocab_3477,\n", + " # Word counts at defaults\n", + " word_counts_3477_data=word_counts_3477.data if sparse.issparse(word_counts_3477) else word_counts_3477,\n", + " word_counts_3477_indices=word_counts_3477.indices if sparse.issparse(word_counts_3477) else np.array([]),\n", + " word_counts_3477_indptr=word_counts_3477.indptr if sparse.issparse(word_counts_3477) else np.array([]),\n", + " word_counts_3477_shape=np.array(word_counts_3477.shape),\n", + " # Full word counts\n", + " word_counts_full_data=word_counts_full.data,\n", + " word_counts_full_indices=word_counts_full.indices,\n", + " word_counts_full_indptr=word_counts_full.indptr,\n", + " word_counts_full_shape=np.array(word_counts_full.shape),\n", + " # Full vocabulary\n", + " full_vocab=full_vocab,\n", + ")\n", + "\n", + "file_size = os.path.getsize(OUTPUT_FILE)\n", + "elapsed = time.time() - t0\n", + "log.info(f\"Saved {OUTPUT_FILE} ({file_size / 1e6:.1f} MB) in {elapsed:.1f}s\")\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.0" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file