document_loaders.py

import os
from typing import List
import trafilatura

def load_pdf(file_path: str) -> str:
    from pypdf import PdfReader
    
    reader = PdfReader(file_path)
    text_parts = []
    
    for page in reader.pages:
        page_text = page.extract_text()
        if page_text:
            text_parts.append(page_text)
    
    return "\n\n".join(text_parts)

def load_docx(file_path: str) -> str:
    from docx import Document
    
    doc = Document(file_path)
    text_parts = []
    
    for para in doc.paragraphs:
        if para.text.strip():
            text_parts.append(para.text)
    
    for table in doc.tables:
        for row in table.rows:
            row_text = " | ".join(cell.text.strip() for cell in row.cells if cell.text.strip())
            if row_text:
                text_parts.append(row_text)
    
    return "\n\n".join(text_parts)

def load_csv(file_path: str) -> str:
    import pandas as pd
    
    df = pd.read_csv(file_path)
    
    text_parts = []
    for idx, row in df.iterrows():
        row_text = " | ".join(f"{col}: {val}" for col, val in row.items() if pd.notna(val))
        if row_text:
            text_parts.append(row_text)
    
    return "\n\n".join(text_parts)

def load_txt(file_path: str) -> str:
    with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
        return f.read()

def load_urls(urls: List[str]) -> str:
    text_parts = []
    
    for url in urls:
        try:
            downloaded = trafilatura.fetch_url(url)
            if downloaded:
                text = trafilatura.extract(downloaded)
                if text:
                    text_parts.append(f"Source: {url}\n\n{text}")
        except Exception as e:
            print(f"Error loading URL {url}: {e}")
            continue
    
    return "\n\n---\n\n".join(text_parts)