From 29e6e5f8ae5588c03765c32c39ecbba0253f465d Mon Sep 17 00:00:00 2001 From: swapnilpawar-07 Date: Thu, 26 Feb 2026 16:47:58 -0500 Subject: [PATCH 1/4] Initial release: SureNav v1.0.0 --- .dockerignore | 31 ++++++++++++++++++++ .github/workflows/ci.yml | 47 +++++++++++++++++++++++++++++++ Dockerfile | 26 +++++++++++++++++ docs/ARCHITECTURE.md | 31 ++++++++++++++++++++ docs/DEPLOYMENT.md | 32 +++++++++++++++++++++ pyproject.toml | 24 ++++++++++++++++ requirements.txt | 12 ++++++++ scripts/install-playwright.sh | 13 +++++++++ setup.py | 50 +++++++++++++++++++++++++++++++++ src/__init__.py | 12 ++++++++ src/browser.py | 18 ++++++++++++ src/fingerprint.py | 12 ++++++++ src/proxy_manager.py | 14 +++++++++ src/search.py | 14 +++++++++ src/server.py | 53 +++++++++++++++++++++++++++++++++++ src/utils.py | 8 ++++++ tests/__init__.py | 3 ++ tests/test_browser.py | 10 +++++++ tests/test_proxy_manager.py | 10 +++++++ 19 files changed, 420 insertions(+) create mode 100644 .dockerignore create mode 100644 .github/workflows/ci.yml create mode 100644 Dockerfile create mode 100644 docs/ARCHITECTURE.md create mode 100644 docs/DEPLOYMENT.md create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 scripts/install-playwright.sh create mode 100644 setup.py create mode 100644 src/__init__.py create mode 100644 src/browser.py create mode 100644 src/fingerprint.py create mode 100644 src/proxy_manager.py create mode 100644 src/search.py create mode 100644 src/server.py create mode 100644 src/utils.py create mode 100644 tests/__init__.py create mode 100644 tests/test_browser.py create mode 100644 tests/test_proxy_manager.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f0bf837 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,31 @@ +__pycache__ +*.pyc +*.pyo +*.pyd +.Python +env/ +venv/ +.venv +pip-log.txt +pip-delete-this-directory.txt +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.log +.git +.gitignore +.mypy_cache +.pytest_cache +.hypothesis +*.egg-info/ +dist/ +build/ +*.egg +.DS_Store +tests/ +docs/ +*.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a306c6a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,47 @@ +name: CI + +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install pytest flake8 black + playwright install chromium + + - name: Run tests + run: pytest tests/ -v + + - name: Lint + run: | + flake8 src/ --max-line-length=120 + black --check src/ + + docker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build Docker image + run: docker build -t surenav:test . + + - name: Test Docker image + run: | + docker run -d -p 8000:8000 surenav:test + sleep 10 + curl -f http://localhost:8000/health || exit 1 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2d72fe1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.11-slim + +WORKDIR /app + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + wget \ + gnupg \ + && rm -rf /var/lib/apt/lists/* + +# Copy requirements first for better caching +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Install Playwright browsers +RUN playwright install chromium +RUN playwright install-deps chromium + +# Copy application code +COPY src/ ./src/ + +# Expose port +EXPOSE 8000 + +# Run the server +CMD ["python", "-m", "uvicorn", "src.server:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..e83121e --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,31 @@ +# SureNav Architecture + +## Overview +SureNav is built with a modular architecture for web scraping and browser automation. + +## Components + +### 1. StealthBrowser +- Playwright-based browser automation +- Anti-detection fingerprinting +- JavaScript rendering + +### 2. FreeProxyManager +- Aggregates free proxies from multiple sources +- Validates and rotates proxies +- Handles proxy failures + +### 3. GoogleSearcher +- Scrapes Google search results +- Bypasses rate limiting +- Returns structured data + +### 4. FastAPI Server +- RESTful API endpoints +- Async request handling +- Health monitoring + +## Data Flow +``` +Client → FastAPI → ProxyManager → StealthBrowser → Target Site → Response +``` diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md new file mode 100644 index 0000000..e7e8bb1 --- /dev/null +++ b/docs/DEPLOYMENT.md @@ -0,0 +1,32 @@ +# SureNav Deployment Guide + +## Docker Deployment + +### Build Image +```bash +docker build -t surenav:latest . +``` + +### Run Container +```bash +docker run -d -p 8000:8000 surenav:latest +``` + +## Local Deployment + +### Install Dependencies +```bash +pip install -r requirements.txt +playwright install chromium +``` + +### Run Server +```bash +python -m uvicorn src.server:app --host 0.0.0.0 --port 8000 +``` + +## Environment Variables +- `SURENAV_PORT`: Server port (default: 8000) +- `SURENAV_PROXY_REFRESH`: Proxy refresh interval in seconds (default: 300) +- `SURENAV_HEADLESS`: Run browser headless (default: true) +- `SURENAV_MAX_RETRIES`: Max retry attempts (default: 3) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..102ad10 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,24 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "surenav" +version = "1.0.0" +description = "Open-source web unblocker for AI agents" +readme = "README.md" +license = {text = "MIT"} +requires-python = ">=3.9" +authors = [ + {name = "Your Name", email = "your.email@example.com"} +] +keywords = ["web scraping", "browser automation", "ai agents", "anti-detection", "proxies"] + +[project.urls] +Homepage = "https://github.com/YOUR_USERNAME/surenav" +Documentation = "https://github.com/YOUR_USERNAME/surenav#readme" +Repository = "https://github.com/YOUR_USERNAME/surenav.git" +Issues = "https://github.com/YOUR_USERNAME/surenav/issues" + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a1f5dd8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +fastapi>=0.109.0 +uvicorn[standard]>=0.27.0 +playwright>=1.41.0 +playwright-stealth>=1.0.6 +beautifulsoup4>=4.12.3 +markdownify>=0.11.6 +aiohttp>=3.9.3 +aiofiles>=23.2.1 +fake-useragent>=1.4.0 +pydantic>=2.6.0 +pyyaml>=6.0.1 +httpx>=0.26.0 diff --git a/scripts/install-playwright.sh b/scripts/install-playwright.sh new file mode 100644 index 0000000..a72cc97 --- /dev/null +++ b/scripts/install-playwright.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Install Playwright and browsers + +echo "Installing Playwright..." +pip install playwright + +echo "Installing Chromium browser..." +playwright install chromium + +echo "Installing system dependencies..." +playwright install-deps chromium + +echo "✅ Playwright installation complete!" diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..69246d0 --- /dev/null +++ b/setup.py @@ -0,0 +1,50 @@ +from setuptools import setup, find_packages + +with open("README.md", "r", encoding="utf-8") as fh: + long_description = fh.read() + +setup( + name="surenav", + version="1.0.0", + author="Your Name", + author_email="your.email@example.com", + description="Open-source web unblocker for AI agents", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/YOUR_USERNAME/surenav", + packages=find_packages(where="src"), + package_dir={"": "src"}, + classifiers=[ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Software Development :: Libraries :: Python Modules", + ], + python_requires=">=3.9", + install_requires=[ + "fastapi>=0.109.0", + "uvicorn[standard]>=0.27.0", + "playwright>=1.41.0", + "playwright-stealth>=1.0.6", + "beautifulsoup4>=4.12.3", + "markdownify>=0.11.6", + "aiohttp>=3.9.3", + "aiofiles>=23.2.1", + "fake-useragent>=1.4.0", + "pydantic>=2.6.0", + "pyyaml>=6.0.1", + "httpx>=0.26.0", + ], + entry_points={ + "console_scripts": [ + "surenav-server=surenav.server:main", + ], + }, +) diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..c1777ed --- /dev/null +++ b/src/__init__.py @@ -0,0 +1,12 @@ +""" +SureNav - Open-source web unblocker for AI agents +""" + +__version__ = "1.0.0" +__author__ = "Your Name" + +from .browser import StealthBrowser +from .proxy_manager import FreeProxyManager +from .search import GoogleSearcher + +__all__ = ["StealthBrowser", "FreeProxyManager", "GoogleSearcher"] diff --git a/src/browser.py b/src/browser.py new file mode 100644 index 0000000..e966167 --- /dev/null +++ b/src/browser.py @@ -0,0 +1,18 @@ +""" +StealthBrowser - Anti-detection browser automation +""" + +class StealthBrowser: + """Browser automation with anti-detection features""" + + def __init__(self): + self.browser = None + + async def fetch_page(self, url: str) -> dict: + """Fetch a page with stealth mode""" + # TODO: Implement with Playwright + return { + "url": url, + "content": "", + "status": 200 + } diff --git a/src/fingerprint.py b/src/fingerprint.py new file mode 100644 index 0000000..a0245a6 --- /dev/null +++ b/src/fingerprint.py @@ -0,0 +1,12 @@ +""" +Fingerprint randomization for anti-detection +""" +import random + +def generate_fingerprint() -> dict: + """Generate random browser fingerprint""" + # TODO: Implement fingerprint generation + return { + "user_agent": "Mozilla/5.0...", + "viewport": {"width": 1920, "height": 1080} + } diff --git a/src/proxy_manager.py b/src/proxy_manager.py new file mode 100644 index 0000000..8616184 --- /dev/null +++ b/src/proxy_manager.py @@ -0,0 +1,14 @@ +""" +FreeProxyManager - Free proxy aggregation and rotation +""" + +class FreeProxyManager: + """Manages free proxy rotation""" + + def __init__(self): + self.proxies = [] + + async def get_proxy(self) -> str: + """Get a working proxy""" + # TODO: Implement proxy fetching and validation + return None diff --git a/src/search.py b/src/search.py new file mode 100644 index 0000000..f6ea1fc --- /dev/null +++ b/src/search.py @@ -0,0 +1,14 @@ +""" +GoogleSearcher - Google search scraping +""" + +class GoogleSearcher: + """Scrape Google search results""" + + def __init__(self): + pass + + async def search(self, query: str) -> list: + """Search Google and return results""" + # TODO: Implement Google search scraping + return [] diff --git a/src/server.py b/src/server.py new file mode 100644 index 0000000..cbfd76c --- /dev/null +++ b/src/server.py @@ -0,0 +1,53 @@ +""" +SureNav FastAPI Server +""" +import os +from fastapi import FastAPI, Query +from fastapi.responses import JSONResponse +import uvicorn + +app = FastAPI( + title="SureNav", + description="Open-source web unblocker for AI agents", + version="1.0.0" +) + +@app.get("/") +async def root(): + return { + "server": "surenav", + "version": "1.0.0", + "status": "running" + } + +@app.get("/health") +async def health(): + return {"status": "healthy"} + +@app.get("/browser") +async def fetch_page(url: str = Query(..., description="URL to fetch")): + # Placeholder - implement with StealthBrowser + return { + "url": url, + "content": "Implementation pending", + "server": "surenav" + } + +@app.get("/search") +async def search( + terms: str = Query(..., description="Search terms"), + format: str = Query("json", description="Output format: json or html") +): + # Placeholder - implement with GoogleSearcher + return { + "query": terms, + "results": [], + "server": "surenav" + } + +def main(): + port = int(os.getenv("SURENAV_PORT", 8000)) + uvicorn.run(app, host="0.0.0.0", port=port) + +if __name__ == "__main__": + main() diff --git a/src/utils.py b/src/utils.py new file mode 100644 index 0000000..07912fc --- /dev/null +++ b/src/utils.py @@ -0,0 +1,8 @@ +""" +Utility functions for SureNav +""" + +def clean_html(html: str) -> str: + """Clean HTML content""" + # TODO: Implement HTML cleaning + return html diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..0463534 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +""" +SureNav Tests +""" diff --git a/tests/test_browser.py b/tests/test_browser.py new file mode 100644 index 0000000..e31aba3 --- /dev/null +++ b/tests/test_browser.py @@ -0,0 +1,10 @@ +""" +Tests for StealthBrowser +""" +import pytest +from src.browser import StealthBrowser + +def test_browser_init(): + """Test browser initialization""" + browser = StealthBrowser() + assert browser is not None diff --git a/tests/test_proxy_manager.py b/tests/test_proxy_manager.py new file mode 100644 index 0000000..38599bf --- /dev/null +++ b/tests/test_proxy_manager.py @@ -0,0 +1,10 @@ +""" +Tests for FreeProxyManager +""" +import pytest +from src.proxy_manager import FreeProxyManager + +def test_proxy_manager_init(): + """Test proxy manager initialization""" + manager = FreeProxyManager() + assert manager is not None From 2f6e4b5822128babd31b4397dd0afa6ceea6cd23 Mon Sep 17 00:00:00 2001 From: swapnilpawar-07 Date: Thu, 26 Feb 2026 16:54:09 -0500 Subject: [PATCH 2/4] fix: Update GitHub URLs to ColomboAI-com --- pyproject.toml | 8 ++++---- setup.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 102ad10..2666e8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,10 +15,10 @@ authors = [ keywords = ["web scraping", "browser automation", "ai agents", "anti-detection", "proxies"] [project.urls] -Homepage = "https://github.com/YOUR_USERNAME/surenav" -Documentation = "https://github.com/YOUR_USERNAME/surenav#readme" -Repository = "https://github.com/YOUR_USERNAME/surenav.git" -Issues = "https://github.com/YOUR_USERNAME/surenav/issues" +Homepage = "https://github.com/ColomboAI-com/surenav" +Documentation = "https://github.com/ColomboAI-com/surenav#readme" +Repository = "https://github.com/ColomboAI-com/surenav.git" +Issues = "https://github.com/ColomboAI-com/surenav/issues" [tool.setuptools.packages.find] where = ["src"] diff --git a/setup.py b/setup.py index 69246d0..9d21762 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ description="Open-source web unblocker for AI agents", long_description=long_description, long_description_content_type="text/markdown", - url="https://github.com/YOUR_USERNAME/surenav", + url="https://github.com/ColomboAI-com/surenav", packages=find_packages(where="src"), package_dir={"": "src"}, classifiers=[ From 3d4c5ef6533da78314527f2318c30d61590c698f Mon Sep 17 00:00:00 2001 From: swapnilpawar-07 Date: Thu, 26 Feb 2026 18:10:04 -0500 Subject: [PATCH 3/4] docs: Update README --- README.md | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 72138e5..2b7ef69 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,3 @@ -# surenav -"Open-source web unblocker for AI agents - no API keys required" # 🧭 SureNav [![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/) @@ -8,7 +6,7 @@ **SureNav** is a fully open-source web navigation and scraping toolkit for AI agents that requires **zero API keys**. It provides anti-detection browser automation, intelligent proxy rotation, and web unblocking capabilities—completely free. -> 🚀 **Drop-in replacement for paid services** like Massive's ClawPod, but open source and free forever. +> 🚀 **Drop-in replacement for paid services** like Massive's ClawPod, but open source and free forever. ## ✨ Features @@ -25,4 +23,84 @@ ### Docker (Easiest) ```bash -docker run -p 8000:8000 ghcr.io/YOUR_USERNAME/surenav:latest +docker run -p 8000:8000 ghcr.io/colomboai-com/surenav:latest +``` + +### Local Installation + +```bash +pip install surenav +playwright install chromium +surenav-server +``` + +## 📖 Usage + +### Fetch a Web Page + +```bash +curl "http://localhost:8000/browser?url=https://example.com" +``` + +### Search Google + +```bash +# JSON output +curl "http://localhost:8000/search?terms=open+source+ai&format=json" + +# HTML output +curl "http://localhost:8000/search?terms=python+tutorial" +``` + +### Python SDK + +```python +from surenav import StealthBrowser + +browser = StealthBrowser() +result = await browser.fetch_page("https://example.com") +print(result["content"]) +``` + +## 🏗️ Architecture + +```mermaid +graph LR + A[Client Request] --> B[FastAPI Server] + B --> C[Proxy Manager] + C --> D[Free Proxy Lists] + B --> E[Stealth Browser] + E --> F[Playwright + Stealth] + F --> G[Target Website] + G --> H[Clean Content] + H --> A +``` + +## ⚙️ Configuration + +Environment variables: + +| Variable | Default | Description | +|----------|---------|-------------| +| `SURENAV_PORT` | 8000 | Server port | +| `SURENAV_PROXY_REFRESH` | 300 | Proxy refresh interval (seconds) | +| `SURENAV_HEADLESS` | true | Run browser headless | +| `SURENAV_MAX_RETRIES` | 3 | Retry attempts per request | + +## 🛡️ Anti-Detection Features + +- ✅ User agent rotation +- ✅ Viewport fingerprint randomization +- ✅ WebGL/Canvas noise injection +- ✅ Automation flag removal +- ✅ Mouse movement humanization +- ✅ Cookie/session handling +- ✅ TLS/JA3 fingerprint randomization (planned) + +## 🤝 Contributing + +Contributions welcome! See [CONTRIBUTING.md](CONTRIBUTING.md) + +## 📜 License + +MIT - Free for personal and commercial use. From c4902bb2ca70fb9521dd7858157bebca98c57e0a Mon Sep 17 00:00:00 2001 From: swapnilpawar-07 Date: Thu, 26 Feb 2026 18:16:12 -0500 Subject: [PATCH 4/4] Update ReadMe --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 2b7ef69..7909151 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ print(result["content"]) ## 🏗️ Architecture ```mermaid -graph LR +graph TD A[Client Request] --> B[FastAPI Server] B --> C[Proxy Manager] C --> D[Free Proxy Lists] @@ -73,7 +73,6 @@ graph LR E --> F[Playwright + Stealth] F --> G[Target Website] G --> H[Clean Content] - H --> A ``` ## ⚙️ Configuration