Merge pull request #100 from goldlabelapps/staging

Update GitHub API integration and versioning; standardize meta keys

Author: goldlabelapps

app/__init__.py

@@ -1,4 +1,4 @@
 """Python° - FastAPI, Postgres, tsvector"""
 
 # Current Version
-__version__ = "3.0.1"
+__version__ = "3.0.3"

app/api/github/README.md

@@ -0,0 +1,38 @@
+## GitHub
+
+We are going to use this route to get data from our GitHub account using their API. We'll be storing that info in the Postgres DB, but have not created any tables yet.
+
+### Create Table Route
+
+**POST /api/github/createtable**
+
+Hits this URL to create the GitHub tables in Postgres. Each table is created with `IF NOT EXISTS`, so it is safe to call multiple times — existing tables and data will not be affected.
+
+### Proposed Table Design
+
+1. github_accounts
+- One row per GitHub account/user profile.
+- Stores account identity fields and full raw payload.
+
+2. github_repos
+- One row per repository.
+- Stores common repo analytics fields plus raw JSON payload.
+
+3. github_gists
+- One row per gist.
+- Stores gist metadata plus raw JSON payload.
+
+4. github_projects
+- One row per project.
+- Stores project metadata plus raw JSON payload.
+
+5. github_resources
+- Generic catch-all for any future GitHub resource type.
+- Supports "everything else" from the API without migrations for every new object type.
+
+### Why this shape works
+
+- Normalized for common entities you asked for: repos, gists, projects.
+- Flexible for all additional GitHub objects through jsonb payload storage.
+- Indexes on resource type, account, and payload for filtering and search.
+

app/api/github/__init__.py

@@ -0,0 +1,9 @@
+"""GitHub Routes"""
+
+from fastapi import APIRouter
+from .github import router as _github_router
+from .sql.create_tables import router as _create_tables_router
+
+github_router = APIRouter()
+github_router.include_router(_github_router)
+github_router.include_router(_create_tables_router)

app/api/github/github.py

@@ -0,0 +1,48 @@
+from fastapi import APIRouter, Depends
+from app.utils.make_meta import make_meta
+from app.utils.db import get_db_connection_direct
+from app.utils.api_key_auth import get_api_key
+
+router = APIRouter()
+
+_TABLES = [
+    "github_accounts",
+    "github_repos",
+    "github_gists",
+    "github_projects",
+    "github_resources",
+]
+
+
+def _fetch_table(cur, table: str) -> dict:
+    cur.execute(f"SELECT COUNT(*) FROM {table};")
+    row = cur.fetchone()
+    count = row[0] if row and row[0] is not None else 0
+    cur.execute(f"SELECT * FROM {table} ORDER BY id DESC LIMIT 100;")
+    if cur.description:
+        columns = [desc[0] for desc in cur.description]
+        rows = [dict(zip(columns, r)) for r in cur.fetchall()]
+    else:
+        rows = []
+    return {"count": count, "rows": rows}
+
+
+@router.get("/github")
+def get_github(api_key: str = Depends(get_api_key)) -> dict:
+    """GET /github: Return counts and records from all GitHub tables."""
+    conn = None
+    cur = None
+    try:
+        conn = get_db_connection_direct()
+        cur = conn.cursor()
+        data = {table: _fetch_table(cur, table) for table in _TABLES}
+        return {"meta": make_meta("success", "GitHub data"), "data": data}
+    except Exception as e:
+        return {"meta": make_meta("error", f"DB error: {str(e)}"), "data": {}}
+    finally:
+        if cur is not None:
+            cur.close()
+        if conn is not None:
+            conn.close()
+
+

app/api/github/sql/create_tables.py

@@ -0,0 +1,156 @@
+from fastapi import APIRouter, HTTPException, Depends
+from app.utils.make_meta import make_meta
+from app.utils.db import get_db_connection_direct
+from app.utils.api_key_auth import get_api_key
+
+router = APIRouter()
+
+_SQL = [
+    """
+    CREATE TABLE IF NOT EXISTS github_accounts (
+        id SERIAL PRIMARY KEY,
+        github_user_id BIGINT UNIQUE,
+        login TEXT UNIQUE,
+        name TEXT,
+        email TEXT,
+        company TEXT,
+        blog TEXT,
+        location TEXT,
+        bio TEXT,
+        avatar_url TEXT,
+        html_url TEXT,
+        payload JSONB NOT NULL DEFAULT '{}'::jsonb,
+        last_synced_at TIMESTAMPTZ,
+        created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+        updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+    );
+    """,
+    """
+    CREATE TABLE IF NOT EXISTS github_repos (
+        id BIGSERIAL PRIMARY KEY,
+        github_repo_id BIGINT UNIQUE NOT NULL,
+        account_login TEXT,
+        name TEXT NOT NULL,
+        full_name TEXT,
+        private BOOLEAN NOT NULL DEFAULT FALSE,
+        fork BOOLEAN NOT NULL DEFAULT FALSE,
+        archived BOOLEAN NOT NULL DEFAULT FALSE,
+        disabled BOOLEAN NOT NULL DEFAULT FALSE,
+        default_branch TEXT,
+        language TEXT,
+        stargazers_count INTEGER,
+        watchers_count INTEGER,
+        forks_count INTEGER,
+        open_issues_count INTEGER,
+        size_kb INTEGER,
+        pushed_at TIMESTAMPTZ,
+        created_at_github TIMESTAMPTZ,
+        updated_at_github TIMESTAMPTZ,
+        html_url TEXT,
+        api_url TEXT,
+        payload JSONB NOT NULL DEFAULT '{}'::jsonb,
+        synced_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+        created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+        updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+    );
+    """,
+    """
+    CREATE TABLE IF NOT EXISTS github_gists (
+        id BIGSERIAL PRIMARY KEY,
+        gist_id TEXT UNIQUE NOT NULL,
+        owner_login TEXT,
+        description TEXT,
+        public BOOLEAN,
+        files_count INTEGER,
+        comments_count INTEGER,
+        html_url TEXT,
+        api_url TEXT,
+        created_at_github TIMESTAMPTZ,
+        updated_at_github TIMESTAMPTZ,
+        payload JSONB NOT NULL DEFAULT '{}'::jsonb,
+        synced_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+        created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+        updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+    );
+    """,
+    """
+    CREATE TABLE IF NOT EXISTS github_projects (
+        id BIGSERIAL PRIMARY KEY,
+        github_project_id BIGINT UNIQUE NOT NULL,
+        owner_login TEXT,
+        owner_type TEXT,
+        name TEXT NOT NULL,
+        body TEXT,
+        state TEXT,
+        number INTEGER,
+        html_url TEXT,
+        api_url TEXT,
+        created_at_github TIMESTAMPTZ,
+        updated_at_github TIMESTAMPTZ,
+        payload JSONB NOT NULL DEFAULT '{}'::jsonb,
+        synced_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+        created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+        updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+    );
+    """,
+    """
+    CREATE TABLE IF NOT EXISTS github_resources (
+        id BIGSERIAL PRIMARY KEY,
+        account_login TEXT,
+        resource_type TEXT NOT NULL,
+        resource_id TEXT NOT NULL,
+        resource_name TEXT,
+        resource_url TEXT,
+        is_private BOOLEAN,
+        state TEXT,
+        created_at_github TIMESTAMPTZ,
+        updated_at_github TIMESTAMPTZ,
+        payload JSONB NOT NULL DEFAULT '{}'::jsonb,
+        synced_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+        created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+        updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+        UNIQUE (resource_type, resource_id)
+    );
+    """,
+    "CREATE INDEX IF NOT EXISTS idx_github_resources_type ON github_resources (resource_type);",
+    "CREATE INDEX IF NOT EXISTS idx_github_resources_account ON github_resources (account_login);",
+    "CREATE INDEX IF NOT EXISTS idx_github_resources_payload ON github_resources USING GIN (payload);",
+    "CREATE INDEX IF NOT EXISTS idx_github_repos_payload ON github_repos USING GIN (payload);",
+    "CREATE INDEX IF NOT EXISTS idx_github_gists_payload ON github_gists USING GIN (payload);",
+    "CREATE INDEX IF NOT EXISTS idx_github_projects_payload ON github_projects USING GIN (payload);",
+]
+
+
+@router.post("/api/github/createtable")
+def create_github_tables(api_key: str = Depends(get_api_key)) -> dict:
+    """POST /api/github/createtable: Create GitHub ingestion tables."""
+    conn = None
+    cur = None
+    try:
+        conn = get_db_connection_direct()
+        cur = conn.cursor()
+        for statement in _SQL:
+            cur.execute(statement)
+        conn.commit()
+        meta = make_meta("success", "GitHub tables created")
+        return {
+            "meta": meta,
+            "data": {
+                "tables": [
+                    "github_accounts",
+                    "github_repos",
+                    "github_gists",
+                    "github_projects",
+                    "github_resources",
+                ]
+            },
+        }
+    except Exception as e:
+        if conn is not None:
+            conn.rollback()
+        raise HTTPException(status_code=500, detail=str(e))
+    finally:
+        if cur is not None:
+            cur.close()
+        if conn is not None:
+            conn.close()