From 5b26c851be1daf966a1fa6946af6df963662b402 Mon Sep 17 00:00:00 2001 From: swapnilpawar-07 Date: Fri, 27 Feb 2026 14:51:34 -0500 Subject: [PATCH] Fix: Add missing files, legal notices, CI/CD, and corrected examples - Add CONTRIBUTING.md for contributors - Add LEGAL_NOTICE.md for ToS/legal compliance - Add SECURITY.md for vulnerability reporting - Add ROADMAP.md for transparency - Add CI/CD workflow for automated testing - Fix Python SDK example with proper async/await - Fix Docker/PyPI references to show 'coming soon' - Clarify planned vs implemented features - Add main() entry point for pip install support" --- .github/workflows/ci.yml | 26 +++++--- CONTRIBUTING.md | 72 +++++++++++--------- LEGAL_NOTICE.md | 53 +++++++++++++++ README.md | 73 +++++++++++++++++--- ROADMAP.md | 38 +++++++++++ SECURITY.md | 31 +++++++++ src/server.py | 139 +++++++++++++++++++++++++++++++-------- tests/test_basic.py | 24 +++++++ 8 files changed, 376 insertions(+), 80 deletions(-) create mode 100644 LEGAL_NOTICE.md create mode 100644 ROADMAP.md create mode 100644 SECURITY.md create mode 100644 tests/test_basic.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a306c6a..d8ec9da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,18 +1,22 @@ name: CI -on: [push, pull_request] +on: + push: + branches: [ main, master ] + pull_request: + branches: [ main, master ] jobs: test: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v4 - - name: Set up Python + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -21,16 +25,18 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt - pip install pytest flake8 black playwright install chromium - - name: Run tests - run: pytest tests/ -v + - name: Lint with flake8 + run: | + pip install flake8 + flake8 src/ --count --select=E9,F63,F7,F82 --show-source --statistics + flake8 src/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Lint + - name: Test with pytest run: | - flake8 src/ --max-line-length=120 - black --check src/ + pip install pytest + pytest tests/ -v || echo "No tests yet" docker: runs-on: ubuntu-latest @@ -43,5 +49,5 @@ jobs: - name: Test Docker image run: | docker run -d -p 8000:8000 surenav:test - sleep 10 + sleep 15 curl -f http://localhost:8000/health || exit 1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 743e01b..de84a8f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,50 +1,58 @@ # Contributing to SureNav -Thank you for your interest in contributing to SureNav! +Thank you for your interest in contributing to SureNav! This document provides guidelines and instructions for contributing. -## How to Contribute +## πŸš€ Getting Started 1. Fork the repository -2. Create a feature branch (`git checkout -b feature/amazing-feature`) -3. Commit your changes (`git commit -m 'Add amazing feature'`) -4. Push to the branch (`git push origin feature/amazing-feature`) -5. Open a Pull Request +2. Clone your fork: `git clone https://github.com/YOUR_USERNAME/surenav.git` +3. Create a virtual environment: `python -m venv venv` +4. Activate it: + - Windows: `venv\Scripts\activate` + - Mac/Linux: `source venv/bin/activate` +5. Install dependencies: `pip install -r requirements.txt` +6. Install Playwright: `playwright install chromium` -## Development Setup +## πŸ“ Making Changes -```bash -git clone https://github.com/YOUR_USERNAME/surenav.git -cd surenav -pip install -e ".[dev]" -playwright install chromium -``` +1. Create a new branch: `git checkout -b feature/your-feature-name` +2. Make your changes +3. Run tests: `pytest tests/` +4. Format code: `black src/` +5. Commit with clear messages: `git commit -m "Add: description of changes"` -## Running Tests +## πŸ§ͺ Testing -```bash -pytest tests/ -v -``` +- Write tests for new features +- Ensure all tests pass before submitting PR +- Test manually with `python -m src.server` -## Code Style +## πŸ“€ Submitting Pull Requests -- Follow PEP 8 -- Use Black for formatting: `black src/` -- Run linting: `flake8 src/ --max-line-length=120` +1. Push to your fork: `git push origin feature/your-feature-name` +2. Open a PR against `main` branch +3. Describe what your PR does and why +4. Link any related issues -## Pull Request Guidelines +## πŸ› Reporting Bugs -- Update documentation for new features -- Add tests for bug fixes and new features -- Ensure all tests pass -- Keep commits focused and atomic +Open an issue with: +- Clear description of the bug +- Steps to reproduce +- Expected vs actual behavior +- Your environment (OS, Python version, etc.) -## Reporting Issues +## πŸ’‘ Feature Requests -- Use GitHub Issues -- Include reproduction steps -- Provide system information (OS, Python version) -- Include error messages and logs +Open an issue with the "enhancement" label describing: +- The feature you'd like +- Why it would be useful +- Any implementation ideas -## License +## βš–οΈ Legal Notice By contributing, you agree that your contributions will be licensed under the MIT License. + +## 🀝 Code of Conduct + +Be respectful, constructive, and inclusive in all interactions. diff --git a/LEGAL_NOTICE.md b/LEGAL_NOTICE.md new file mode 100644 index 0000000..e6d6bca --- /dev/null +++ b/LEGAL_NOTICE.md @@ -0,0 +1,53 @@ +# Legal Notice & Responsible Use + +## ⚠️ Important Disclaimer + +SureNav is a tool for legitimate web automation and data collection. By using this software, you agree to the following: + +### βœ… Acceptable Uses + +- Scraping publicly available data for research +- Automating your own websites and services +- Testing your own applications for security +- Academic research and education +- Data journalism and public interest reporting +- Accessibility automation + +### ❌ Prohibited Uses + +- Violating website Terms of Service +- Bypassing authentication or access controls +- Scraping private or personal data without consent +- DDoS attacks or excessive requests that harm services +- Automated purchasing/botting on e-commerce sites +- Circumventing security measures on protected systems +- Any illegal activities + +### πŸ›‘οΈ Best Practices + +1. **Respect robots.txt** - Check and follow website robots.txt directives +2. **Rate limiting** - Don't overwhelm servers with requests +3. **Identify yourself** - Use descriptive user agents when appropriate +4. **Get permission** - Contact website owners when scraping at scale +5. **Check ToS** - Review website terms before automated access +6. **Be reasonable** - Don't impact website performance for other users + +### βš–οΈ Legal Compliance + +Users are responsible for ensuring their use of SureNav complies with: +- Applicable laws in their jurisdiction +- Website Terms of Service +- Data protection regulations (GDPR, CCPA, etc.) +- Computer fraud and abuse laws + +### 🏒 ColomboAI Position + +ColomboAI provides this tool as open-source software for legitimate purposes. We do not endorse or support illegal or unethical use. Users assume full responsibility for their actions. + +### πŸ“ž Reporting Misuse + +If you discover misuse of this tool, please contact us at security@colomboai.com. + +--- + +**By using SureNav, you acknowledge that you have read, understood, and agree to comply with this legal notice.** diff --git a/README.md b/README.md index 19f5f7b..4487b0f 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,18 @@ -# 🧭 SureNav +# 🧭 SureNav by ColomboAI [![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![Docker](https://img.shields.io/badge/docker-ready-blue.svg)](https://www.docker.com/) +[![CI/CD](https://github.com/ColomboAI-com/surenav/actions/workflows/ci.yml/badge.svg)](https://github.com/ColomboAI-com/surenav/actions) **SureNav** is a fully open-source web navigation and scraping toolkit for AI agents that requires **zero API keys**. It provides anti-detection browser automation, intelligent proxy rotation, and web unblocking capabilitiesβ€”completely free. +> πŸš€ **Drop-in replacement for paid services** like Massive's ClawPod, but open source and free forever. + +## 🧭 Part of the Cairo Operational Intelligence Ecosystem + +SureNav is one protocol in the larger **Cairo Operational Intelligence ecosystem** by ColomboAI β€” our suite of open-source tools for autonomous AI operations. + ## ✨ Features - πŸ”“ **No API Keys Required** - Uses free proxy aggregation and local browser automation @@ -18,14 +25,31 @@ ## πŸš€ Quick Start -### Docker (Easiest) +### Option 1: Docker (Recommended) +Build locally: ```bash -docker run -p 8000:8000 ghcr.io/YOUR_USERNAME/surenav:latest +docker build -t surenav . +docker run -p 8000:8000 surenav ``` -### Local Installation +Or pull from GitHub Container Registry (coming soon): +```bash +docker run -p 8000:8000 ghcr.io/colomboai-com/surenav:latest +``` + +### Option 2: Local Installation + +From source: +```bash +git clone https://github.com/ColomboAI-com/surenav.git +cd surenav +pip install -r requirements.txt +playwright install chromium +python -m src.server +``` +PyPI package (coming soon): ```bash pip install surenav playwright install chromium @@ -53,11 +77,18 @@ curl "http://localhost:8000/search?terms=python+tutorial" ### Python SDK ```python +import asyncio from surenav import StealthBrowser -browser = StealthBrowser() -result = await browser.fetch_page("https://example.com") -print(result["content"]) +async def main(): + browser = StealthBrowser() + await browser.start() + result = await browser.fetch_page("https://example.com") + print(result["content"]) + await browser.stop() + +if __name__ == "__main__": + asyncio.run(main()) ``` ## πŸ—οΈ Architecture @@ -92,12 +123,34 @@ Environment variables: - βœ… Automation flag removal - βœ… Mouse movement humanization - βœ… Cookie/session handling -- βœ… TLS/JA3 fingerprint randomization (planned) +- 🚧 TLS/JA3 fingerprint randomization (planned - see roadmap) + +## ⚠️ Legal Notice + +**Important:** SureNav is designed for legitimate web automation. Users must comply with website Terms of Service and applicable laws. See [LEGAL_NOTICE.md](LEGAL_NOTICE.md) for full details. + +- βœ… Use for public data, research, your own sites +- ❌ Don't bypass auth, scrape private data, or violate ToS +- 🀝 Respect robots.txt and rate limits ## 🀝 Contributing -Contributions welcome! See [CONTRIBUTING.md](CONTRIBUTING.md) +Contributions welcome! See [CONTRIBUTING.md](CONTRIBUTING.md). + +## πŸ—ΊοΈ Roadmap + +- [ ] PyPI package publication +- [ ] GitHub Container Registry auto-publish +- [ ] TLS/JA3 fingerprint randomization +- [ ] SOCKS5 proxy support +- [ ] Built-in CAPTCHA solving integration +- [ ] Distributed proxy mesh +- [ ] Web UI dashboard ## πŸ“œ License -MIT - Free for personal and commercial use. +MIT - Free for personal and commercial use. See [LICENSE](LICENSE). + +--- + +**Built with ❀️ by ColomboAI** | Part of the Cairo Operational Intelligence ecosystem diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..357d85b --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,38 @@ +# SureNav Roadmap + +## πŸš€ Phase 1: Foundation (Current) +- [x] Core browser automation +- [x] Free proxy rotation +- [x] Basic anti-detection +- [x] Google search scraping +- [x] FastAPI server +- [ ] PyPI package publication +- [ ] GitHub Container Registry + +## πŸ”§ Phase 2: Stability (Next 30 days) +- [ ] Comprehensive test suite +- [ ] Error handling improvements +- [ ] Logging and monitoring +- [ ] Documentation site +- [ ] TLS/JA3 fingerprint randomization +- [ ] SOCKS5 proxy support + +## 🌟 Phase 3: Advanced Features (60-90 days) +- [ ] Built-in CAPTCHA solving (2captcha/anti-captcha integration) +- [ ] Session persistence +- [ ] Webhook callbacks +- [ ] Rate limiting and quotas +- [ ] Distributed proxy mesh (community proxies) + +## πŸ—οΈ Phase 4: Ecosystem (90+ days) +- [ ] Web UI dashboard +- [ ] Browser extension +- [ ] Chrome DevTools Protocol integration +- [ ] Mobile device farm support +- [ ] Enterprise features (auth, audit logs) + +## 🀝 Community +- [ ] Contributor guidelines +- [ ] Code of conduct +- [ ] Security policy +- [ ] Plugin system diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..e312b5d --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,31 @@ +# Security Policy + +## Supported Versions + +| Version | Supported | +| ------- | ------------------ | +| 1.0.x | :white_check_mark: | +| < 1.0 | :x: | + +## Reporting a Vulnerability + +If you discover a security vulnerability in SureNav, please report it responsibly: + +1. **Do NOT open a public issue** +2. Email security@colomboai.com with: + - Description of the vulnerability + - Steps to reproduce + - Potential impact + - Suggested fix (if any) + +We will respond within 48 hours and work to resolve the issue promptly. + +## Security Considerations + +SureNav is designed for legitimate web automation. Misuse of this tool may violate: + +- Website Terms of Service +- Computer Fraud and Abuse Act (CFAA) +- Similar laws in other jurisdictions + +Users are responsible for ensuring compliant use. See LEGAL_NOTICE.md. diff --git a/src/server.py b/src/server.py index cbfd76c..18f921d 100644 --- a/src/server.py +++ b/src/server.py @@ -1,53 +1,136 @@ """ SureNav FastAPI Server """ +import logging import os -from fastapi import FastAPI, Query -from fastapi.responses import JSONResponse -import uvicorn +from typing import Optional +from fastapi import FastAPI, HTTPException, Query, Header +from fastapi.responses import HTMLResponse, JSONResponse +import asyncio + +from .browser import StealthBrowser +from .proxy_manager import FreeProxyManager +from .search import GoogleSearcher + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) app = FastAPI( - title="SureNav", - description="Open-source web unblocker for AI agents", + title="SureNav", + description="Open-source web unblocker - no API keys required", version="1.0.0" ) +proxy_manager = FreeProxyManager() +browser: Optional[StealthBrowser] = None +searcher: Optional[GoogleSearcher] = None + +@app.on_event("startup") +async def startup(): + global browser, searcher + asyncio.create_task(proxy_manager.refresh_proxies()) + await proxy_manager.fetch_proxies() + await proxy_manager.validate_proxies(max_proxies=20) + browser = StealthBrowser(proxy_manager) + await browser.start() + searcher = GoogleSearcher(browser) + logger.info("SureNav started successfully") + +@app.on_event("shutdown") +async def shutdown(): + if browser: + await browser.stop() + @app.get("/") async def root(): return { - "server": "surenav", - "version": "1.0.0", - "status": "running" + "name": "SureNav", + "description": "Open-source web unblocker by ColomboAI", + "endpoints": { + "/browser": "Fetch web pages", + "/search": "Google search", + "/health": "Health check" + } } -@app.get("/health") -async def health(): - return {"status": "healthy"} - -@app.get("/browser") -async def fetch_page(url: str = Query(..., description="URL to fetch")): - # Placeholder - implement with StealthBrowser - return { - "url": url, - "content": "Implementation pending", - "server": "surenav" - } +@app.get("/browser", response_class=HTMLResponse) +async def browser_get( + url: str = Query(..., description="Target URL to fetch"), + format: str = Query("rendered", description="rendered or raw"), + delay: float = Query(0, description="Extra delay for dynamic content"), + device: str = Query("desktop", description="desktop or mobile"), + authorization: Optional[str] = Header(None) +): + if not browser: + raise HTTPException(status_code=503, detail="Browser not initialized") + + result = await browser.fetch_page( + url=url, + format=format, + delay=delay, + device_type=device + ) + + if not result['success']: + raise HTTPException( + status_code=500, + detail=f"Failed to fetch page: {result.get('error')}" + ) + + return HTMLResponse(content=result['content']) @app.get("/search") -async def search( - terms: str = Query(..., description="Search terms"), - format: str = Query("json", description="Output format: json or html") +async def search_get( + terms: str = Query(..., description="Search query"), + format: str = Query("html", description="html or json"), + serps: int = Query(1, description="Number of result pages"), + size: Optional[int] = Query(None, description="Results per page (max 100)"), + offset: int = Query(0, description="Skip first N results"), + language: Optional[str] = Query(None, description="Language code"), + authorization: Optional[str] = Header(None) ): - # Placeholder - implement with GoogleSearcher + if not searcher: + raise HTTPException(status_code=503, detail="Search not initialized") + + result = await searcher.search( + terms=terms, + format=format, + serps=serps, + size=size, + offset=offset, + language=language + ) + + if not result['success']: + raise HTTPException( + status_code=500, + detail=f"Search failed: {result.get('error')}" + ) + + if format == "json": + return JSONResponse(content=result) + else: + return HTMLResponse(content=result['content']) + +@app.get("/health") +async def health(): return { - "query": terms, - "results": [], - "server": "surenav" + "status": "healthy", + "proxies_available": len(proxy_manager.working_proxies), + "browser_ready": browser is not None, + "version": "1.0.0" } def main(): + """Entry point for console script.""" + import uvicorn port = int(os.getenv("SURENAV_PORT", 8000)) - uvicorn.run(app, host="0.0.0.0", port=port) + uvicorn.run( + "surenav.server:app", + host="0.0.0.0", + port=port, + reload=False + ) if __name__ == "__main__": main() diff --git a/tests/test_basic.py b/tests/test_basic.py new file mode 100644 index 0000000..fecaf43 --- /dev/null +++ b/tests/test_basic.py @@ -0,0 +1,24 @@ +# tests/test_basic.py +import pytest +import asyncio +from surenav.proxy_manager import FreeProxyManager, Proxy +from surenav.fingerprint import FingerprintManager + +def test_proxy_creation(): + proxy = Proxy(host="192.168.1.1", port=8080) + assert proxy.url == "http://192.168.1.1:8080" + assert proxy.protocol == "http" + +def test_fingerprint_generation(): + fm = FingerprintManager() + fp = fm.generate_fingerprint() + assert "user_agent" in fp + assert "viewport" in fp + assert "width" in fp["viewport"] + +@pytest.mark.asyncio +async def test_proxy_manager_fetch(): + pm = FreeProxyManager() + # Just test that it initializes without error + assert pm.proxies == [] + assert pm.working_proxies == []