diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..c56639f --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,65 @@ +name: Docs + +on: + push: + branches: [main] + paths: + - "docs/**" + - "mkdocs.yml" + - ".github/workflows/docs.yml" + workflow_dispatch: + +# GITHUB_TOKEN permissions required by actions/deploy-pages. +permissions: + contents: read + pages: write + id-token: write + +# Allow only one in-flight deploy at a time so a fast follow-up commit +# doesn't race the previous build. +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + name: Build site + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: latest + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install docs dependencies + run: uv sync --extra docs + + - name: Build site (strict) + run: uv run mkdocs build --strict + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: site + + deploy: + name: Deploy to GitHub Pages + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Configure Pages + uses: actions/configure-pages@v5 + + - name: Deploy + id: deployment + uses: actions/deploy-pages@v4 diff --git a/docs/assets/extra.css b/docs/assets/extra.css new file mode 100644 index 0000000..a3f04bb --- /dev/null +++ b/docs/assets/extra.css @@ -0,0 +1,132 @@ +/* + * Agent Library docs — Arcade brand overrides. + * + * Brand palette pulled directly from arcade.dev: + * - background: #0f0f0f (near-black, primary) + * - surface: #1a1a2e (dark navy, raised) + * - accent: #00e0e0 (signature cyan) + * - foreground: #ffffff + */ + +:root { + --arcade-cyan: #00e0e0; + --arcade-cyan-dim: #00b0b0; + --arcade-bg: #0f0f0f; + --arcade-surface: #1a1a2e; + --arcade-surface-2: #232342; + --arcade-fg: #ffffff; + --arcade-fg-muted: #c8c8d4; +} + +/* Dark scheme — primary brand experience. */ +[data-md-color-scheme="slate"] { + --md-default-bg-color: var(--arcade-bg); + --md-default-bg-color--light: var(--arcade-surface); + --md-default-fg-color: var(--arcade-fg); + --md-default-fg-color--light: var(--arcade-fg-muted); + --md-default-fg-color--lighter: rgba(255, 255, 255, 0.6); + --md-default-fg-color--lightest: rgba(255, 255, 255, 0.07); + + --md-primary-fg-color: var(--arcade-bg); + --md-primary-fg-color--light: var(--arcade-surface); + --md-primary-fg-color--dark: #050505; + --md-primary-bg-color: var(--arcade-fg); + --md-primary-bg-color--light: var(--arcade-fg-muted); + + --md-accent-fg-color: var(--arcade-cyan); + --md-accent-fg-color--transparent: rgba(0, 224, 224, 0.12); + --md-accent-bg-color: var(--arcade-bg); + + --md-code-bg-color: var(--arcade-surface); + --md-code-fg-color: var(--arcade-fg); + --md-code-hl-color: rgba(0, 224, 224, 0.18); + --md-code-hl-keyword-color: var(--arcade-cyan); + --md-code-hl-string-color: #c4ff8a; + + --md-typeset-a-color: var(--arcade-cyan); + + --md-footer-bg-color: var(--arcade-bg); + --md-footer-bg-color--dark: #050505; +} + +/* Light scheme — keep the cyan accent, white background, near-black text. */ +[data-md-color-scheme="default"] { + --md-primary-fg-color: var(--arcade-bg); + --md-primary-fg-color--light: var(--arcade-surface); + --md-primary-fg-color--dark: #050505; + --md-primary-bg-color: var(--arcade-fg); + + --md-accent-fg-color: var(--arcade-cyan-dim); + --md-accent-fg-color--transparent: rgba(0, 176, 176, 0.12); + + --md-typeset-a-color: var(--arcade-cyan-dim); + + --md-footer-bg-color: var(--arcade-bg); + --md-footer-bg-color--dark: #050505; +} + +/* Header tweaks: slightly darker than page background, hairline accent border. */ +.md-header { + background-color: var(--arcade-bg); + border-bottom: 1px solid rgba(0, 224, 224, 0.18); +} + +/* Logo: enforce a fixed height so the SVG wordmark doesn't dominate. */ +.md-header__button.md-logo img, +.md-header__button.md-logo svg { + height: 1.5rem; + width: auto; +} + +/* Brighter cyan on hover for nav links + table-of-contents items. */ +[data-md-color-scheme="slate"] .md-nav__link:hover, +[data-md-color-scheme="slate"] .md-nav__link.md-nav__link--active { + color: var(--arcade-cyan); +} + +/* Code: navy surface tone, body color text. Earlier draft tinted inline + * `code` cyan, which made every backticked term in prose loud and wore + * down the eye fast. Cyan stays reserved for links and accents. */ +[data-md-color-scheme="slate"] .md-typeset code { + background-color: var(--arcade-surface-2); + color: var(--arcade-fg); + border-radius: 3px; +} +[data-md-color-scheme="slate"] .md-typeset pre > code { + background-color: var(--arcade-surface); + color: var(--arcade-fg); +} + +/* Admonitions: tint the title bar with cyan for note/tip; keep warning/danger. */ +[data-md-color-scheme="slate"] .md-typeset .admonition.note, +[data-md-color-scheme="slate"] .md-typeset .admonition.tip, +[data-md-color-scheme="slate"] .md-typeset .admonition.info { + border-color: var(--arcade-cyan); +} +[data-md-color-scheme="slate"] .md-typeset .admonition.note > .admonition-title, +[data-md-color-scheme="slate"] .md-typeset .admonition.tip > .admonition-title, +[data-md-color-scheme="slate"] .md-typeset .admonition.info > .admonition-title { + background-color: rgba(0, 224, 224, 0.15); +} + +/* Buttons: cyan accent, dark text. */ +[data-md-color-scheme="slate"] .md-button--primary { + background-color: var(--arcade-cyan); + border-color: var(--arcade-cyan); + color: var(--arcade-bg); +} +[data-md-color-scheme="slate"] .md-button--primary:hover { + background-color: var(--arcade-fg); + border-color: var(--arcade-fg); + color: var(--arcade-bg); +} + +/* Search bar: cyan focus ring. */ +[data-md-color-scheme="slate"] .md-search__input { + background-color: var(--arcade-surface); + color: var(--arcade-fg); +} +[data-md-color-scheme="slate"] .md-search__input:focus { + background-color: var(--arcade-surface-2); + box-shadow: 0 0 0 2px rgba(0, 224, 224, 0.4); +} diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000..8348be9 --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,196 @@ +# CLI reference + +Every command Agent Library exposes, with copy-paste examples. The binary name is `librarian` (with `libr` as a shorter alias). Examples below use `librarian`. + +!!! note "Reminder" + These all assume you've set up the `librarian` alias from the [Quickstart](quickstart.md). If you haven't, prepend each command with: + + ``` + uvx --from "agent-library[all]==0.13.0" + ``` + +--- + +## `librarian add` + +Add a file or directory as a source and index its contents. + +```bash +librarian add ~/notes +``` + +**Options:** + +| Flag | What it does | +|---|---| +| `-n NAME` / `--name` | Give the source a friendly name (defaults to the directory name) | +| `-d N` / `--depth` | Limit recursion. `0` = the directory itself only. Default: unlimited | +| `-p PATTERN` / `--pattern` | Glob filter (e.g. `'notes/*.md'`) | +| `-e PATTERN` / `--exclude` | Glob to skip. Can be passed multiple times | +| `--dry-run` | Show what would be indexed without doing anything | +| `-v` / `--verbose` | Print every file as it's indexed | + +```bash +# Index just the top-level files of a folder +librarian add ~/notes --depth 0 + +# Index only Python files in a project, excluding tests +librarian add ~/code/myproject --pattern '**/*.py' --exclude '**/tests/**' + +# Preview without writing +librarian add ~/notes --dry-run +``` + +--- + +## `librarian list` + +Show every source you've added, with document counts. + +```bash +librarian list +``` + +**Options:** + +| Flag | What it does | +|---|---| +| `-a` / `--all` | Include sources marked hidden/test | +| `--json` | Output as JSON (good for scripting) | + +--- + +## `librarian rm` + +Remove a source from the index. Files on disk are **not** deleted — only the database entries. + +```bash +librarian rm notes +``` + +**Options:** + +| Flag | What it does | +|---|---| +| `-f` / `--force` | Skip the confirmation prompt | +| `--path PATH` | Disambiguate if two sources share the same name | + +--- + +## `librarian search` + +Search across everything you've indexed. + +```bash +librarian search "retry policy" +``` + +**Options:** + +| Flag | What it does | +|---|---| +| `-l N` / `--limit` | Max results (default 10) | +| `-m MODE` / `--mode` | `hybrid` (default), `semantic`, `vector` (alias for semantic), or `keyword` | +| `-s NAME` / `--source` | Search within a single source | +| `-t TIMEFRAME` / `--timeframe` | `today`, `yesterday`, `week`, `month`, `year` | +| `-f FORMAT` / `--format` | Output as `table` (default), `json`, or `paths` | +| `-v` / `--verbose` | Include the matched content snippet inline | +| `-o` / `--open` | Open the top result in your editor | +| `-c` / `--copy` | Copy the top result's content to clipboard | +| `--code` | Search code files only | +| `--images` | Search images only (uses CLIP if vision is installed) | +| `--type TYPE` | Filter by asset type: `text`, `code`, `pdf`, `image` | + +```bash +# Find conceptual matches, not just keyword +librarian search "graceful degradation" --mode semantic + +# Just the file paths, one per line, useful for piping +librarian search "deploy" --format paths | xargs cat | less + +# Things from this week, code only +librarian search "TODO" --timeframe week --code +``` + +--- + +## `librarian serve` + +Start the MCP server. This is what your AI assistant invokes — you usually configure it once in Claude/Cursor and never run it by hand. But it's available for testing. + +```bash +librarian serve stdio # for Claude Desktop, Claude Code, Cursor +librarian serve http --port 7878 # for HTTP-based MCP clients +``` + +**Options:** + +| Flag | What it does | +|---|---| +| `--host HOST` / `-h` | HTTP-only: bind address | +| `--port N` / `-p` | HTTP-only: port number | +| `--log-level LEVEL` | `debug`, `info`, `warning` (default), or `error` | + +--- + +## Subcommand groups + +Three command groups bundle less-common operations: + +### `librarian config` + +The most useful of the three. **Persist config changes** to `~/.librarian/settings.json` so they survive across sessions: + +```bash +librarian config show # table of every setting with source attribution +librarian config get HYBRID_ALPHA # one value +librarian config set HYBRID_ALPHA 0.5 +librarian config path # show the four config-file locations +librarian config edit # open settings.json in your editor +librarian config models # check / download embedding models +librarian config reset # back to defaults +``` + +See [Configuration](configuration.md) for the full list of settable keys. + +### `librarian index` + +Lower-level index operations: + +```bash +librarian index build # rebuild the entire index from scratch +librarian index clean # remove all indexed data (keeps sources list) +librarian index clobber # remove everything and reinitialize the database +``` + +### `librarian docs` + +Per-document operations: + +```bash +librarian docs list # list every indexed document +librarian docs search "title text" # search by title only (not contents) +``` + +--- + +## Environment variables + +Override defaults without editing config files: + +| Variable | Default | Effect | +|---|---|---| +| `DATABASE_PATH` | `~/.librarian/index.db` | Where the search index is stored | +| `DOCUMENTS_PATH` | `./documents` | Default directory used when no `path` is given | +| `EMBEDDING_PROVIDER` | `local` | Switch to `openai` for hosted embeddings | +| `EMBEDDING_MODEL` | `all-MiniLM-L6-v2` | Sentence-transformers model name | +| `HYBRID_ALPHA` | `0.7` | Vector ↔ keyword blend (0 = keyword only, 1 = vector only) | +| `MMR_LAMBDA` | `0.7` | Relevance ↔ diversity (0 = max diversity, 1 = max relevance) | +| `SEARCH_LIMIT` | `10` | Default result count | + +```bash +# One-off override for a single search +DATABASE_PATH=/tmp/test.db librarian search "anything" +``` + +For permanent overrides, add them to your shell profile or a `.env` file in the directory you launch `librarian` from. diff --git a/docs/concepts.md b/docs/concepts.md new file mode 100644 index 0000000..a4e0dd9 --- /dev/null +++ b/docs/concepts.md @@ -0,0 +1,81 @@ +# Concepts + +Optional reading. The library works fine if you skip this page — but understanding the moving parts helps when you want to tune results. + +## Asset types + +Agent Library tags every file with an **asset type**: + +| Asset type | What's in it | Parser | +|---|---|---| +| `text` | Markdown, plain text | built-in | +| `code` | Source code (Python, JS, TS, Go, Rust, …) | regex-based symbol extractor | +| `pdf` | PDF documents | requires `pypdf` (in `[all]` extras) | +| `image` | PNG, JPG, GIF, WEBP | requires `Pillow` (in `[all]` extras) | +| `multimodal` | Reserved for documents that mix modalities (none of today's parsers emit this — but the enum value exists for forward compatibility, and `--asset_type=multimodal` is accepted as a filter) | — | + +When you search, the asset type is preserved on every result. You can filter on it: + +```bash +librarian search "encrypt" --type code +``` + +## Search modes + +Three modes, chosen with `--mode`: + +- **`keyword`** — pure full-text search, BM25-ranked. Best for exact phrases or unique tokens. Fast. +- **`semantic`** — pure embedding similarity (cosine distance against a sentence-transformer model). Finds meaning matches even when the wording differs. Slower; loads ~100 MB of model on first use. +- **`hybrid`** *(default)* — runs both and merges. Each modality normalizes its scores to [0, 1]; the merger gives a small overlap bonus to chunks that match across modalities. This is what you want most of the time. + +When `ENABLE_CROSS_MODAL_SEARCH=true` (the default), `hybrid` also runs separate embedding models for code (CodeBERT) and images (CLIP) when those extras are installed. + +## Chunking + +Documents are split into **chunks** before indexing. Each chunk is what the search returns — a passage, not a whole file. This keeps results focused and gives you snippet-level scores instead of file-level. + +The chunker is asset-type aware: + +- Markdown is split by headers (`H1`/`H2`) and paragraphs +- Code is split by symbol (function, class, method) +- PDFs are split by page +- Images become a single chunk with metadata + +## Scoring & MMR + +Results are scored 0 → 1. The blend is controlled by two knobs: + +- **`HYBRID_ALPHA`** (default `0.7`): in non-cross-modal hybrid, the formula is `alpha * vector_score + (1 - alpha) * keyword_score`. Higher = lean on semantic match more. +- **`MMR_LAMBDA`** (default `0.7`): after blending, **Maximal Marginal Relevance** picks the top-K with a diversity bias. The formula is `lambda * relevance - (1 - lambda) * max_similarity_to_already_selected`. Lower = more diverse top-K (might miss the second-best answer if it looks too much like the first); higher = more relevance-focused. + +You can tweak both via environment variables or `librarian config`. + +## What lives where + +| File | What it is | +|---|---| +| `~/.librarian/index.db` | SQLite database with the FTS5 index, vector embeddings, and document metadata. Survives across sessions. | +| `~/.librarian/sources.json` | The list of registered sources (managed by `librarian add` / `rm`). | +| `~/.librarian/documents/` | Default location for content created via `add_to_library` from inside the MCP server (when no `directory` is given). | + +Delete `index.db` and re-run `librarian add ...` to rebuild from scratch. + +## MCP under the hood + +When an AI assistant calls Agent Library, it speaks the **Model Context Protocol** — a JSON-RPC convention defined by Anthropic. `librarian serve stdio` talks MCP over stdin/stdout; `librarian serve http` talks MCP over HTTP streaming. + +The server advertises 9 tools: + +| Tool | Purpose | +|---|---| +| `Librarian_SearchLibrary` | The main thing — find content | +| `Librarian_ReadFromLibrary` | Read a full document by path | +| `Librarian_AddToLibrary` | Save new content into the library | +| `Librarian_UpdateLibraryDoc` | Replace a document's content | +| `Librarian_RemoveFromLibrary` | Drop a document from the index | +| `Librarian_ListLibraryContents` | List indexed documents | +| `Librarian_IndexDirectoryToLibrary` | Bulk-index a directory | +| `Librarian_GetLibraryOverview` | Inspect the library (sections / stats / tree) | +| `Librarian_SuggestLibraryLocation` | Recommend where new content belongs | + +Each takes typed arguments, returns typed JSON. The MCP host (Claude, Cursor) shows them under the server's name in its tool picker. diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..fc86fc9 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,345 @@ +# Configuration + +Every knob Agent Library exposes, what it does, what its default is, and **how to change it**. If you only want one or two tweaks, jump straight to [How to change a setting](#how-to-change-a-setting); the rest is reference. + +--- + +## How to change a setting + +There are four ways to apply a setting. Pick the one that matches your situation. + +=== "`librarian config set` (recommended)" + + The simplest path. Settings are persisted to `~/.librarian/settings.json` and survive across sessions: + + ```bash + librarian config set EMBEDDING_MODEL "BAAI/bge-base-en-v1.5" + librarian config set EMBEDDING_DIMENSION 768 + librarian config set HYBRID_ALPHA 0.5 + ``` + + Inspect the current state: + + ```bash + librarian config show # table of every setting + where each came from + librarian config get HYBRID_ALPHA + librarian config path # show the four config-file paths + librarian config edit # open settings.json in your editor + librarian config reset # back to defaults + ``` + + Restart `librarian serve` (or your AI client) after changing anything. + +=== "From the terminal (one-off)" + + Prefix any `librarian` command with an env var. Useful for trying a setting without committing to it: + + ```bash + HYBRID_ALPHA=0.5 librarian search "deploy notes" + ``` + +=== "Inside Claude Desktop / Cursor / Claude Code" + + The MCP server is a subprocess, so settings the AI host should know about live in the `env` block of the MCP config: + + ```json + { + "mcpServers": { + "librarian": { + "command": "uvx", + "args": [ + "--from", "agent-library[all]==0.13.0", + "librarian", "serve", "stdio" + ], + "env": { + "EMBEDDING_MODEL": "BAAI/bge-base-en-v1.5", + "EMBEDDING_DIMENSION": "768", + "MMR_LAMBDA": "0.5" + } + } + } + } + ``` + + Restart Claude / Cursor after editing. + +=== "From a `.env` file" + + Drop a `.env` file in the directory you launch `librarian` from: + + ``` + EMBEDDING_MODEL=BAAI/bge-base-en-v1.5 + HYBRID_ALPHA=0.6 + DATABASE_PATH=/Users/me/work/librarian.db + ``` + + Agent Library reads it automatically on startup. + +=== "Permanent shell var" + + Less recommended (env is invisible to GUI apps), but works if everything you care about is terminal-only. Add to `~/.zshrc` or `~/.bashrc`: + + ```bash + export DATABASE_PATH="$HOME/Documents/librarian.db" + export EMBEDDING_MODEL="BAAI/bge-base-en-v1.5" + ``` + +!!! info "Precedence (highest wins)" + 1. Process env vars (`HYBRID_ALPHA=0.5 librarian ...`) + 2. `.env` file in CWD + 3. `librarian config set` (in `~/.librarian/settings.json`) + 4. Built-in defaults + +!!! warning "All values must be strings inside JSON" + JSON env blocks expect `"true"` and `"0.7"`, not `true` or `0.7`. Boolean values that count as "true": `true`, `1`, `yes`, `on` (case-insensitive). Anything else is false. + +--- + +## Storage + +| Variable | Default | What it does | +|---|---|---| +| `DATABASE_PATH` | `~/.librarian/index.db` | SQLite file with the FTS index, vectors, and document metadata | +| `DOCUMENTS_PATH` | `./documents` | Default directory used when no `path` is given to a tool | +| `SOURCES_CONFIG_PATH` | `~/.librarian/sources.json` | List of registered sources (managed by `librarian add` / `rm`) | + +Set these per-project to keep work and personal libraries separate. + +--- + +## Text embeddings + +The library uses an **embedding model** to map text into vectors so semantic search can find meaning matches. The default is fast and small; bigger models give better results at the cost of disk space and CPU. + +| Variable | Default | What it does | +|---|---|---| +| `EMBEDDING_PROVIDER` | `local` | Either `local` (sentence-transformers on your machine) or `openai` (any OpenAI-compatible endpoint) | +| `EMBEDDING_MODEL` | `all-MiniLM-L6-v2` | The model to load. See the supported list below | +| `EMBEDDING_DIMENSION` | `384` | Vector dimension. Must match the chosen model — see the supported list | +| `EMBEDDING_QUERY_INSTRUCTION` | `"Given a query, return relevant information from documents."` | Used by instruction-tuned models (E5, BGE) to bias the encoding toward retrieval | + +### Supported text models + +All of these are loaded via `sentence-transformers`. To switch, set `EMBEDDING_MODEL` and `EMBEDDING_DIMENSION` to the matching pair from the table. + +| Model | Dim | Size | Notes | HF link | +|---|---|---|---|---| +| `all-MiniLM-L6-v2` *(default)* | 384 | 80 MB | Fast, decent quality, ships everywhere | [→](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) | +| `all-mpnet-base-v2` | 768 | 420 MB | The classic sentence-transformers default. Higher quality, ~5× slower | [→](https://huggingface.co/sentence-transformers/all-mpnet-base-v2) | +| `BAAI/bge-small-en-v1.5` | 384 | 130 MB | BGE small — drop-in replacement for MiniLM with stronger retrieval | [→](https://huggingface.co/BAAI/bge-small-en-v1.5) | +| `BAAI/bge-base-en-v1.5` | 768 | 440 MB | BGE base — what most retrieval benchmarks use | [→](https://huggingface.co/BAAI/bge-base-en-v1.5) | +| `BAAI/bge-large-en-v1.5` | 1024 | 1.3 GB | BGE large — best quality of this family, slowest | [→](https://huggingface.co/BAAI/bge-large-en-v1.5) | +| `intfloat/e5-small-v2` | 384 | 130 MB | E5 small — strong retrieval baseline | [→](https://huggingface.co/intfloat/e5-small-v2) | +| `intfloat/e5-base-v2` | 768 | 440 MB | E5 base | [→](https://huggingface.co/intfloat/e5-base-v2) | +| `intfloat/e5-large-v2` | 1024 | 1.3 GB | E5 large | [→](https://huggingface.co/intfloat/e5-large-v2) | +| `mixedbread-ai/mxbai-embed-large-v1` | 1024 | 1.3 GB | Newer model with strong English retrieval scores | [→](https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1) | + +!!! danger "Re-index when you change models" + Vectors from one model can't be searched with another. After switching `EMBEDDING_MODEL`, delete `~/.librarian/index.db` and re-run `librarian add ...` so your existing content is re-embedded. + +### Using OpenAI-compatible APIs + +If you'd rather offload embedding to a hosted service (OpenAI, vLLM, llama.cpp's server, etc.), switch the provider: + +| Variable | Default | What it does | +|---|---|---| +| `EMBEDDING_PROVIDER` | `local` | Set to `openai` | +| `OPENAI_API_BASE` | `http://localhost:7171/v1` | Endpoint URL (point at OpenAI, vLLM, llama.cpp, etc.) | +| `OPENAI_API_KEY` | `not-needed` | Your API key (or `not-needed` for local servers that don't auth) | +| `OPENAI_EMBEDDING_MODEL` | `qwen3-embedding-06b` | Model identifier the endpoint serves | +| `OPENAI_EMBEDDING_DIMENSION` | `1024` | Vector dimension for that model | +| `OPENAI_EMBEDDING_BATCH_SIZE` | `64` | How many texts to embed per API call | + +--- + +## Code embeddings + +When `ENABLE_CODE_EMBEDDINGS=true` (the default), source code files are embedded with a code-specific model in addition to the regular text embedder. This makes "find the function that handles retries" work even when "retry" isn't in the comments. + +| Variable | Default | What it does | +|---|---|---| +| `ENABLE_CODE_EMBEDDINGS` | `true` | Turn the code path on/off | +| `CODE_EMBEDDING_MODEL` | `microsoft/codebert-base` | The code embedding model | +| `CODE_EMBEDDING_DIMENSION` | `768` | Vector dimension | +| `CODE_EMBEDDING_PROVIDER` | `local` | `local` or `openai` | + +### Supported code models + +The code path activates when the model name contains `codebert` or `codellama`. Any other model falls back to the regular text path. + +| Model | Dim | Size | Notes | HF link | +|---|---|---|---|---| +| `microsoft/codebert-base` *(default)* | 768 | 500 MB | Multi-language, balanced speed/quality | [→](https://huggingface.co/microsoft/codebert-base) | +| `microsoft/graphcodebert-base` | 768 | 500 MB | Better at structural code matches (data flow / graph aware) | [→](https://huggingface.co/microsoft/graphcodebert-base) | + +!!! tip "Don't have any code in your library?" + Set `ENABLE_CODE_EMBEDDINGS=false` to skip loading the model entirely. Saves ~500 MB and a couple seconds at startup. + +--- + +## Vision embeddings + +Image files (PNG, JPG, GIF, WEBP) get a separate visual embedding so semantic search works across diagrams and screenshots. This uses CLIP — a model that maps images and text into the same vector space, so a query like "auth flow" finds matching diagrams. + +| Variable | Default | What it does | +|---|---|---| +| `ENABLE_VISION_EMBEDDINGS` | `true` | Turn the vision path on/off | +| `VISION_EMBEDDING_MODEL` | `clip-ViT-B-32` | The CLIP-family model | +| `VISION_EMBEDDING_DIMENSION` | `512` | Vector dimension | + +### Supported vision models + +The vision path activates when the model name contains `clip` or `siglip`. + +| Model | Dim | Size | Notes | HF link | +|---|---|---|---|---| +| `clip-ViT-B-32` *(default)* | 512 | 600 MB | Original CLIP base, fast | [→](https://huggingface.co/sentence-transformers/clip-ViT-B-32) | +| `clip-ViT-B-16` | 512 | 600 MB | Higher resolution patches than B-32 — slightly better, slightly slower | [→](https://huggingface.co/sentence-transformers/clip-ViT-B-16) | +| `clip-ViT-L-14` | 768 | 1.7 GB | Large CLIP — best image quality, expensive | [→](https://huggingface.co/sentence-transformers/clip-ViT-L-14) | + +!!! tip "Indexing screenshots only?" + `clip-ViT-B-32` is plenty. The L-14 variants only pay off with photographic content where fine detail matters. + +--- + +## OCR (extracting text from images) + +Tesseract-based OCR runs over indexed images so the text inside a screenshot is still searchable. + +| Variable | Default | What it does | +|---|---|---| +| `ENABLE_OCR` | `true` | Toggle OCR on indexed images | +| `OCR_LANGUAGE` | `eng` | Tesseract language code(s). Multiple langs use `+` (e.g. `eng+spa`) | +| `OCR_CONFIG` | `--psm 3` | Tesseract page-segmentation mode | +| `OCR_MIN_CONFIDENCE` | `0` | Drop OCR'd text below this confidence (0–100, 0 = no filter) | + +OCR requires `tesseract` installed on the system (separate from Python deps): `brew install tesseract` on macOS, `apt install tesseract-ocr` on Debian/Ubuntu. + +--- + +## Image captioning (optional) + +When enabled, every indexed image also gets a free-text caption generated by an image-to-text model. Off by default since most users don't need it. + +| Variable | Default | What it does | +|---|---|---| +| `IMAGE_GENERATE_CAPTIONS` | `false` | Turn captioning on | +| `IMAGE_CAPTION_MODEL` | `blip-base` | The captioning model | + +| Model | Notes | HF link | +|---|---|---| +| `Salesforce/blip-image-captioning-base` *(default — `blip-base` is the short alias)* | BLIP base; fast, decent captions | [→](https://huggingface.co/Salesforce/blip-image-captioning-base) | +| `Salesforce/blip-image-captioning-large` | BLIP large; slower, better captions | [→](https://huggingface.co/Salesforce/blip-image-captioning-large) | + +--- + +## Chunking + +How documents get split into searchable chunks before indexing. + +| Variable | Default | What it does | +|---|---|---| +| `CHUNK_SIZE` | `512` | Target chunk length in tokens (≈ words) | +| `CHUNK_OVERLAP` | `50` | Tokens of overlap between adjacent chunks (preserves context across boundaries) | +| `MIN_CHUNK_SIZE` | `50` | Drop chunks shorter than this | +| `CODE_CHUNK_STRATEGY` | `code_blocks` | `code_blocks` (split by function/class) or `fixed` (fixed size) | +| `CODE_INCLUDE_CONTEXT` | `true` | Include surrounding lines as context when chunking code | +| `CODE_CONTEXT_LINES` | `5` | How many context lines on each side | +| `PDF_CHUNK_STRATEGY` | `pages` | `pages` (one chunk per page) or `sections` (split by headings) | + +!!! tip "Larger chunks = more context per result, fewer results overall" + Bumping `CHUNK_SIZE` to 1024 makes each result longer but reduces the total number of chunks. Good for technical docs where context matters; bad for short notes where you want fine-grained matching. + +--- + +## Search behavior + +| Variable | Default | What it does | +|---|---|---| +| `SEARCH_LIMIT` | `10` | Default result count | +| `HYBRID_ALPHA` | `0.7` | In hybrid mode, the blend: `alpha · vector_score + (1 - alpha) · keyword_score`. Higher = more semantic | +| `MMR_LAMBDA` | `0.7` | Maximal Marginal Relevance: `lambda · relevance - (1 - lambda) · max_similarity_to_already_picked`. Higher = relevance-focused, lower = diverse | +| `ENABLE_CROSS_MODAL_SEARCH` | `true` | Search text + code + image vectors in parallel and merge fairly | +| `CROSS_MODAL_SIMILARITY_THRESHOLD` | `0.7` | Drop cross-modal matches below this similarity | +| `MODALITY_WEIGHT_TEXT` | `1.0` | Per-modality weight in the cross-modal merge | +| `MODALITY_WEIGHT_CODE` | `1.0` | | +| `MODALITY_WEIGHT_VISION` | `1.0` | | +| `MODALITY_WEIGHT_FTS` | `1.0` | Keyword search modality weight | + +!!! tip "Tuning the search" + - **Results feel scattered?** Lower `MMR_LAMBDA` toward 0.3 to push for more diverse top-K, or raise it toward 0.9 to lock in on the most relevant. + - **Hybrid is missing exact-keyword matches?** Lower `HYBRID_ALPHA` toward 0.3 to weight keyword score higher. + - **Code matches dominating text matches?** Lower `MODALITY_WEIGHT_CODE` to 0.5. + +--- + +## Codebase indexing + +| Variable | Default | What it does | +|---|---|---| +| `CODEBASE_AUTO_DETECT` | `true` | Detect language from file extensions automatically | +| `CODEBASE_INDEX_TESTS` | `true` | Include `tests/` and `*_test.*` files | +| `CODEBASE_MAX_FILE_SIZE_KB` | `500` | Skip files larger than this | +| `DEFAULT_ASSET_TYPES` | `text,code` | Comma-separated asset types to index by default | + +--- + +## PDF processing + +| Variable | Default | What it does | +|---|---|---| +| `ENABLE_PDF_PROCESSING` | `true` | Toggle PDF parsing | +| `PDF_OCR_ENABLED` | `false` | Run OCR on PDF pages (slow; only useful for scanned PDFs without an embedded text layer) | + +--- + +## Server + +| Variable | Default | What it does | +|---|---|---| +| `LIBRARIAN_HOST` | `127.0.0.1` | HTTP transport bind address | +| `LIBRARIAN_PORT` | `8000` | HTTP transport port | +| `LIBRARIAN_ENABLE_OPTIONAL_TOOLS` | `true` | Whether `get_library_overview` and `suggest_library_location` are advertised | + +--- + +## Tool behavior (advanced) + +| Variable | Default | What it does | +|---|---|---| +| `TOOL_SEARCH_DEFAULT_LIMIT` | `10` | Default `limit` arg when the AI calls `search_library` without specifying | +| `TOOL_MAX_CONTEXT_LINES` | `10` | Max lines of context surfaced in tool output | +| `CODE_MAX_DEPENDENCY_DEPTH` | `3` | Max depth for code dependency walks | +| `CODE_MAX_REFERENCES` | `50` | Max references returned per query | +| `INDEX_POLL_INTERVAL` | `60.0` | Seconds between background re-index polls | +| `INDEX_START_DELAY` | `5.0` | Seconds before the first background poll on startup | + +--- + +## Putting it together — a real example + +Say you want a project-specific library that uses a stronger embedder, weights keyword matches higher than the default, and lives next to your code rather than in `~/.librarian/`. Add this to your Cursor `mcp.json` or Claude Desktop config: + +```json +{ + "mcpServers": { + "librarian": { + "command": "uvx", + "args": [ + "--from", "agent-library[all]==0.13.0", + "librarian", "serve", "stdio" + ], + "env": { + "DATABASE_PATH": "${workspaceFolder}/.librarian/index.db", + "DOCUMENTS_PATH": "${workspaceFolder}", + "EMBEDDING_MODEL": "BAAI/bge-base-en-v1.5", + "EMBEDDING_DIMENSION": "768", + "HYBRID_ALPHA": "0.5", + "ENABLE_VISION_EMBEDDINGS": "false" + } + } + } +} +``` + +After saving, reload Cursor (or restart Claude). Index once with `librarian add .` — and your AI now searches that project's content with the stronger model. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..e28ae67 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,40 @@ +# Agent Library + +**A personal knowledge library for your AI assistant.** Drop your notes, code, PDFs, and screenshots into it once. From then on, Claude (or Cursor, or any other MCP-compatible AI) can search and read your stuff to answer your questions. + +You don't need to know what an embedding is. You don't need to know what MCP is. You just need to follow three steps. + +--- + +## What you can do with it + +- **Ask Claude about anything in your notes.** "What did I write about the API redesign last month?" — Claude searches your library, finds the relevant note, and reads it back. +- **Have your AI work over your codebase.** Index a project folder once. Cursor or Claude Code can then search the code by meaning ("find the function that handles retry logic") instead of just by filename. +- **Search PDFs and screenshots alongside your notes.** Indexes contracts, papers, diagrams. The library treats them like first-class search results. + +It runs entirely on your machine. Nothing leaves your laptop. + +--- + +## How to get started + +Three pages, in order: + +1. **[Install uv](install.md)** — `uv` is the tool that runs Agent Library. One copy-paste command. +2. **[Quickstart](quickstart.md)** — index a folder and search it from your terminal. Verify it works. +3. **Connect it to your AI** — pick the one you use: + - **[Claude Desktop](integrations/claude-desktop.md)** — the regular Claude app + - **[Claude Code](integrations/claude-code.md)** — the terminal-based Claude + - **[Cursor](integrations/cursor.md)** — the IDE + +After that: + +- **[CLI reference](cli.md)** — every `librarian` command with a copy-paste example +- **[Concepts](concepts.md)** — what's actually happening under the hood (optional reading) +- **[Troubleshooting](troubleshooting.md)** — when something doesn't work + +--- + +## Why "Agent Library"? + +Most AI assistants forget your work the moment a conversation ends. Agent Library is the long-lived shelf they can reach for. You write a note once; six conversations from now your assistant still finds it. diff --git a/docs/install.md b/docs/install.md new file mode 100644 index 0000000..09d24e1 --- /dev/null +++ b/docs/install.md @@ -0,0 +1,72 @@ +# Installing uv + +`uv` is the tool that runs Agent Library. It's a fast Python package manager — think of it as the "app store" we'll fetch Agent Library from. + +You only have to do this once. + +## macOS + +Open the **Terminal** app (Spotlight → "Terminal") and paste this: + +```bash +curl -LsSf https://astral.sh/uv/install.sh | sh +``` + +Press Enter. After it finishes (a few seconds), close and reopen Terminal so the new command is on your path. + +Verify it worked: + +```bash +uv --version +``` + +You should see something like `uv 0.5.x`. If you do, you're done — go to **[Quickstart](quickstart.md)**. + +??? tip "Prefer Homebrew?" + If you already use Homebrew: + + ```bash + brew install uv + ``` + +## Windows + +Open **PowerShell** (Start menu → "PowerShell") and paste this: + +```powershell +powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex" +``` + +Close and reopen PowerShell, then verify: + +```powershell +uv --version +``` + +## Linux + +```bash +curl -LsSf https://astral.sh/uv/install.sh | sh +``` + +Verify: + +```bash +uv --version +``` + +--- + +## Troubleshooting + +!!! warning "`uv: command not found`" + The installer sets up your `PATH` in a shell profile that only reloads on a new shell. **Close your terminal entirely and reopen it**, then try `uv --version` again. If it still fails, run `source ~/.zshrc` (macOS default) or `source ~/.bashrc` (Linux) and try once more. + +!!! warning "`Could not connect to astral.sh`" + Corporate networks sometimes block the install script. If you have Python installed already, you can fall back to: + + ```bash + pip install --user uv + ``` + +When `uv --version` prints a version number, head to **[Quickstart](quickstart.md)**. diff --git a/docs/integrations/claude-code.md b/docs/integrations/claude-code.md new file mode 100644 index 0000000..732a788 --- /dev/null +++ b/docs/integrations/claude-code.md @@ -0,0 +1,98 @@ +# Claude Code (CLI) + +Claude Code is the terminal-based version of Claude. It uses the same MCP server interface as Claude Desktop, but you configure it from the shell. + +## Prerequisites + +- You ran the [Quickstart](../quickstart.md) and have at least one folder indexed. +- Claude Code is installed: . + +## Add Agent Library + +Claude Code has a one-line command to register an MCP server. The `--` separator tells Claude where the command-and-args to invoke begin: + +```bash +claude mcp add librarian -- uvx --from "agent-library[all]==0.13.0" librarian serve stdio +``` + +That writes the entry into `~/.claude.json` (or your project-local equivalent depending on `--scope`). Verify: + +```bash +claude mcp list +``` + +You should see `librarian` listed. + +!!! tip "Editing the JSON directly" + If you'd rather edit `~/.claude/settings.json` by hand, the `mcpServers` block looks the same as the Claude Desktop one: + + ```json + { + "mcpServers": { + "librarian": { + "command": "uvx", + "args": [ + "--from", "agent-library[all]==0.13.0", + "librarian", "serve", "stdio" + ] + } + } + } + ``` + +??? tip "Already ran `uv tool install`?" + Claude Code is a terminal app, so it inherits your shell `PATH`. That means once you've done `uv tool install`, the simpler form works too: + + ```json + "librarian": { + "command": "librarian", + "args": ["serve", "stdio"] + } + ``` + + `uvx` is still our default in the docs because it's hermetic and doesn't depend on the install state. + +## Use it + +Start (or restart) Claude Code. In your conversation: + +> *"Search my library for the retry policy notes."* + +The first response triggers the MCP tool. You'll be prompted to allow `Librarian_SearchLibrary` — accept it once and Claude Code remembers. + +## Scope: local, project, or user + +`claude mcp add` accepts a `--scope` flag (default: `local`): + +| Scope | Where it's stored | When you'd use it | +|---|---|---| +| `local` *(default)* | `.claude.json` for the current project — only you see it | Trying things out, personal experiments | +| `project` | `.mcp.json` at the repo root, checked into git | Sharing a server config with the whole team | +| `user` | `~/.claude.json` — visible from every directory | One library you use across all your projects | + +For a personal library available everywhere: + +```bash +claude mcp add librarian --scope user -- uvx --from "agent-library[all]==0.13.0" librarian serve stdio +``` + +For a project-specific library with its own database: + +```bash +claude mcp add librarian \ + -e "DATABASE_PATH=$(pwd)/.librarian/index.db" \ + -e "DOCUMENTS_PATH=$(pwd)" \ + -- uvx --from "agent-library[all]==0.13.0" librarian serve stdio +``` + +Pass `-e KEY=VALUE` once per env var (it can be repeated). + +## Removing it + +```bash +claude mcp remove librarian +``` + +--- + +Problems? See [Troubleshooting](../troubleshooting.md). diff --git a/docs/integrations/claude-desktop.md b/docs/integrations/claude-desktop.md new file mode 100644 index 0000000..8a04e6e --- /dev/null +++ b/docs/integrations/claude-desktop.md @@ -0,0 +1,129 @@ +# Claude Desktop + +Connect your Agent Library to the Claude Desktop app so Claude can search your notes during chats. + +## Prerequisites + +- You ran the [Quickstart](../quickstart.md) and have at least one folder indexed. +- Claude Desktop is installed: . + +## 1. Find the config file + +Claude Desktop reads its tool list from a single JSON file. The path depends on your OS: + +=== "macOS" + + ``` + ~/Library/Application Support/Claude/claude_desktop_config.json + ``` + + Open it in TextEdit or your editor of choice: + + ```bash + open -a TextEdit "$HOME/Library/Application Support/Claude/claude_desktop_config.json" + ``` + + If TextEdit complains the file doesn't exist, create it first: + + ```bash + mkdir -p "$HOME/Library/Application Support/Claude" + touch "$HOME/Library/Application Support/Claude/claude_desktop_config.json" + ``` + +=== "Windows" + + ``` + %APPDATA%\Claude\claude_desktop_config.json + ``` + + Open it in Notepad: + + ```powershell + notepad "$env:APPDATA\Claude\claude_desktop_config.json" + ``` + +=== "Linux" + + ``` + ~/.config/Claude/claude_desktop_config.json + ``` + +## 2. Add the Agent Library entry + +If the file is empty, paste this whole block in: + +```json +{ + "mcpServers": { + "librarian": { + "command": "uvx", + "args": [ + "--from", "agent-library[all]==0.13.0", + "librarian", "serve", "stdio" + ] + } + } +} +``` + +If the file already has content, just add the `"librarian": { ... }` block inside the existing `mcpServers` object. The full file should look like a valid JSON document with `mcpServers` at the top level. + +Save the file. + +??? tip "Already ran `uv tool install`?" + If you went through the [Quickstart](../quickstart.md) and installed Agent Library globally with `uv tool install`, you can swap the `uvx` form for the installed binary directly. **You'll need to use the absolute path** because Claude Desktop is launched by macOS's `launchd` and doesn't inherit your terminal's `PATH`: + + ```json + "librarian": { + "command": "/Users/YOUR-USERNAME/.local/bin/librarian", + "args": ["serve", "stdio"] + } + ``` + + Find your absolute path with `which librarian` in the terminal. The `uvx` form doesn't have this caveat, which is why we recommend it as the default. + +## 3. Restart Claude Desktop + +Quit Claude entirely (right-click the dock icon → Quit, or Cmd+Q) and reopen it. Claude only re-reads the config on launch. + +## 4. Try it + +Start a new chat with Claude and ask something that requires your notes: + +> *"Search my library for notes about retry policy."* + +Claude will call the `Librarian_SearchLibrary` tool. The first time, you'll see a permission prompt — click **Allow**. The results appear inline. + +You can verify the connection by checking the bottom of the Claude window — when an MCP server is connected, a small plug icon shows "1 server connected". + +--- + +## What if something doesn't work? + +!!! warning "Claude says it has no tools" + The most common cause is that Claude was already running when you saved the config. Quit it completely (Cmd+Q, not just close window) and reopen. + +!!! warning "Claude says the tool errored out" + Open `~/Library/Logs/Claude/mcp-server-librarian.log` (macOS) or the equivalent on your OS. The first run downloads ~2 GB of models, which can take a couple of minutes — Claude may time out before that finishes. Run the install command once at the terminal first to warm the cache: + + ```bash + librarian --help + ``` + + Then restart Claude. + +!!! tip "Want a custom storage location?" + Add an `env` block: + + ```json + "librarian": { + "command": "uvx", + "args": ["--from", "agent-library[all]==0.13.0", "librarian", "serve", "stdio"], + "env": { + "DATABASE_PATH": "/Users/me/work/librarian.db", + "DOCUMENTS_PATH": "/Users/me/work/notes" + } + } + ``` + +More problems? [Troubleshooting →](../troubleshooting.md) diff --git a/docs/integrations/cursor.md b/docs/integrations/cursor.md new file mode 100644 index 0000000..a8cc4c0 --- /dev/null +++ b/docs/integrations/cursor.md @@ -0,0 +1,95 @@ +# Cursor + +Cursor supports MCP servers natively as of late 2024. Hooking up Agent Library lets Cursor search your notes and external knowledge alongside whatever's open in your editor. + +## Prerequisites + +- You ran the [Quickstart](../quickstart.md) and have at least one folder indexed. +- A version of Cursor with built-in MCP support (visible as a **MCP** tab in Settings). + +## 1. Open the MCP settings + +In Cursor: + +1. Open the command palette: Cmd+Shift+P (or Ctrl+Shift+P on Windows/Linux). +2. Type **"MCP"** and pick **"Cursor: Open MCP Settings"** (or **"Cursor: Edit MCP Servers"** in newer builds). + +That opens `~/.cursor/mcp.json`. If the file is empty, Cursor will pre-fill the skeleton. + +## 2. Add the Agent Library entry + +If the file is empty or freshly skeletoned, paste this: + +```json +{ + "mcpServers": { + "librarian": { + "command": "uvx", + "args": [ + "--from", "agent-library[all]==0.13.0", + "librarian", "serve", "stdio" + ] + } + } +} +``` + +If you already have other MCP servers in there, add `"librarian"` as another key under `mcpServers`. Save the file. + +??? tip "Already ran `uv tool install`?" + Cursor inherits more of your environment than Claude Desktop, so the simpler form usually works once you've installed the binary globally: + + ```json + "librarian": { + "command": "librarian", + "args": ["serve", "stdio"] + } + ``` + + If Cursor reports "command not found", switch back to the `uvx` form above (or use the absolute path from `which librarian`). + +## 3. Reload Cursor + +Either fully quit and reopen Cursor, or run **"Developer: Reload Window"** from the command palette. Cursor re-reads `mcp.json` on each reload. + +## 4. Try it + +Open the Cursor chat panel (Cmd+L) and ask: + +> *"Search my library for retry policy notes."* + +Cursor's agent will call into the librarian server. The first time, you'll see a permission prompt — accept it. + +You can confirm the connection from the **MCP** tab in Cursor's settings: librarian should show as **Connected** with a green dot. + +--- + +## Project-specific libraries + +Cursor also supports a project-local `mcp.json` at the workspace root. Useful when each codebase has its own indexed knowledge: + +```bash +mkdir -p .cursor && cat > .cursor/mcp.json <<'EOF' +{ + "mcpServers": { + "librarian": { + "command": "uvx", + "args": [ + "--from", "agent-library[all]==0.13.0", + "librarian", "serve", "stdio" + ], + "env": { + "DATABASE_PATH": "${workspaceFolder}/.librarian/index.db", + "DOCUMENTS_PATH": "${workspaceFolder}" + } + } + } +} +EOF +``` + +The `${workspaceFolder}` placeholder is expanded by Cursor at launch. + +--- + +Problems? See [Troubleshooting](../troubleshooting.md). diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 0000000..5a6938d --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,87 @@ +# Quickstart + +Five minutes from zero to "my AI can search my notes". You'll do all of this from your terminal. + +!!! note "Before you start" + Make sure `uv --version` works in your terminal. If not, do **[Install uv](install.md)** first (one command). + +## 1. Install Agent Library + +Paste this into your terminal: + +```bash +uv tool install "agent-library[all]==0.13.0" +``` + +This installs Agent Library as a regular command on your machine — like installing any other CLI tool. The first run downloads the Python package and its language models (~2 GB total). You only pay this cost **once**; `uv` caches everything afterward. + +Verify it worked: + +```bash +librarian --help +``` + +You should see a help menu listing commands like `add`, `search`, `serve`. From here on, you just type `librarian ...` like any other terminal tool. + +!!! tip "Updating later" + To pick up a new release: + + ```bash + uv tool upgrade agent-library + ``` + + To remove it: + + ```bash + uv tool uninstall agent-library + ``` + +## 2. Index a folder of notes + +Pick any folder that has text-like files — notes, markdown, PDFs, code. Let's say it's `~/notes/`. + +```bash +librarian add ~/notes +``` + +Agent Library walks the folder, parses each supported file (Markdown, code in 18 languages, PDFs, images), splits the contents into searchable chunks, and stores them in `~/.librarian/index.db`. It'll print a progress summary at the end. + +!!! note "Supported file types" + Out of the box: Markdown (`.md`), text (`.txt`), 18 programming languages (`.py`, `.js`, `.ts`, `.go`, `.rs`, etc.), PDFs, and common image formats (PNG, JPG). Other file types are skipped. + +## 3. Search it + +```bash +librarian search "what was that idea about retry policy?" +``` + +Three results print, ranked by how well they match. Each has a path, a snippet, and a relevance score from 0 to 1. + +By default this is a **hybrid search** — it combines keyword matching with semantic understanding, so "retry policy" finds notes that talk about "exponential backoff" too. + +Two other modes are available when you want them: + +```bash +librarian search "exact phrase here" --mode keyword # exact-match only +librarian search "this concept" --mode semantic # meaning-based only +``` + +## 4. List what you've indexed + +```bash +librarian list +``` + +Shows every source you've added (folders or single files) with document counts. + +--- + +## You're set + +The library now has your stuff. Next: tell your AI assistant about it. + +- [Set up Claude Desktop →](integrations/claude-desktop.md) +- [Set up Claude Code →](integrations/claude-code.md) +- [Set up Cursor →](integrations/cursor.md) + +If something didn't work, head to [Troubleshooting](troubleshooting.md). diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 0000000..7d2d375 --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,125 @@ +# Troubleshooting + +The most common things that go wrong, and how to fix them. + +## `uv: command not found` + +The installer dropped `uv` into `~/.local/bin` (Linux/macOS) or `~/.cargo/bin` (older versions), but your terminal hasn't picked up the change yet. + +**Fix:** Close your terminal completely and reopen it. If that doesn't help, manually add the install dir to your PATH: + +```bash +export PATH="$HOME/.local/bin:$PATH" +``` + +Add that line to `~/.zshrc` or `~/.bashrc` to make it stick. + +## First run is taking forever + +The first `uv tool install "agent-library[all]==0.13.0"` (or `uvx ...` invocation) downloads: + +- The Agent Library package +- `sentence-transformers` and `torch` (~500 MB) +- Optional vision/code models on first search (~1 GB) + +Plan on 2–5 minutes the first time. Subsequent runs use the cached install and start in under a second. + +If it seems stuck, run with verbose output: + +```bash +uv tool install --verbose "agent-library[all]==0.13.0" +``` + +## Claude says it doesn't have any tools + +Three things to check: + +1. **Did you fully quit Claude before reopening?** A cold restart is required after editing `claude_desktop_config.json`. Close menus aren't enough — use Cmd+Q on macOS. +2. **Is the JSON valid?** Paste the whole file into to confirm. A common mistake is leaving a trailing comma after the last entry. +3. **Did the MCP subprocess crash?** Check `~/Library/Logs/Claude/mcp-server-librarian.log` (macOS) for stack traces. If you see "command not found: uvx", the issue is your PATH — see the section above. + +## "First time" timeouts inside Claude + +Claude Desktop has an internal timeout for MCP server startup. If your first run hits the ~2-minute model download, Claude may give up before the server is ready. + +**Fix:** Warm the cache once at the terminal first: + +```bash +uv tool install "agent-library[all]==0.13.0" +librarian --help +``` + +Then restart Claude. The next launch will reuse the cached install. + +## "ModuleNotFoundError: No module named 'pypdf'" / 'PIL' + +You installed without the `[all]` extras. PDFs and images are skipped silently when their parsers aren't available. + +**Fix:** Reinstall with all extras: + +```bash +uv tool install --reinstall "agent-library[all]==0.13.0" +``` + +Or, in your MCP config, change `"agent-library==0.13.0"` to `"agent-library[all]==0.13.0"`. + +## Search returns nothing + +A few likely causes: + +- **You haven't indexed anything yet.** Run `librarian list` — if it's empty, run `librarian add ` first. +- **Your query is too narrow.** Try `--mode hybrid` (the default — combines semantic and keyword) and a shorter query. Also try `--type text` if you're certain the content is text-based. +- **The index is stale.** Re-run `librarian add ` to refresh. New files are picked up automatically; modified ones are re-indexed. + +## Search results look weird / irrelevant + +Try a different `--mode`: + +- `keyword` is best for exact-phrase searches. +- `semantic` is best when you want meaning matches but no token overlap. +- `hybrid` (default) blends the two. + +If hybrid results feel diluted, you can also tune `MMR_LAMBDA`: + +```bash +MMR_LAMBDA=0.9 librarian search "your query" # heavily favor relevance over diversity +MMR_LAMBDA=0.3 librarian search "your query" # heavily favor diversity (different docs) +``` + +## I want to wipe everything and start over + +```bash +rm -rf ~/.librarian # database, sources list, settings.json +uv tool uninstall agent-library +``` + +Then reinstall via [Quickstart](quickstart.md). + +To reset just configuration (keeping your indexed content): + +```bash +librarian config reset +``` + +## I'm hitting permission errors writing to `~/.librarian` + +Override the location with an env var: + +```bash +DATABASE_PATH="$HOME/Documents/librarian.db" librarian add ~/notes +``` + +Set it permanently in your shell profile: + +```bash +echo 'export DATABASE_PATH="$HOME/Documents/librarian.db"' >> ~/.zshrc +``` + +--- + +Still stuck? Open an issue at with: + +1. What you ran +2. What you expected +3. What happened (full error output) +4. `librarian --version` output (run after `uv tool install agent-library`) diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..dbd5e1d --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,94 @@ +site_name: Agent Library +site_description: A personal knowledge library for AI agents — store and search notes, code, PDFs, and images. +site_url: https://arcadeai.github.io/agent-library/ +repo_url: https://github.com/ArcadeAI/agent-library +repo_name: ArcadeAI/agent-library +edit_uri: edit/main/docs/ + +docs_dir: docs + +theme: + name: material + # Identity is "Agent Library" (the site_name) rendered as the header text; + # the brand vibe comes from the cyan accent + dark theme defined below and + # in docs/assets/extra.css, not from a borrowed wordmark. + icon: + logo: material/bookshelf + repo: fontawesome/brands/github + features: + - navigation.instant + - navigation.tracking + - navigation.sections + - navigation.top + - search.suggest + - search.highlight + - content.code.copy + - content.action.edit + - toc.follow + # Brand-aligned palette: dark is the primary experience (matches arcade.dev), + # light is offered as an alternate. The actual hex values come from the + # custom CSS in docs/assets/extra.css — these palette entries just pin the + # right Material scheme + a brand-adjacent base color. + palette: + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: black + accent: cyan + toggle: + icon: material/weather-sunny + name: Switch to light mode + - media: "(prefers-color-scheme: light)" + scheme: default + primary: black + accent: cyan + toggle: + icon: material/weather-night + name: Switch to dark mode + font: + text: Inter + code: JetBrains Mono + +extra_css: + - assets/extra.css + +markdown_extensions: + - admonition + - attr_list + - def_list + - footnotes + - md_in_html + - tables + - toc: + permalink: true + - pymdownx.details + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true + - pymdownx.tasklist: + custom_checkbox: true + - pymdownx.keys + +nav: + - Home: index.md + - Quickstart: quickstart.md + - Install: + - Installing uv: install.md + - Use it with your AI: + - Claude Desktop: integrations/claude-desktop.md + - Claude Code (CLI): integrations/claude-code.md + - Cursor: integrations/cursor.md + - CLI reference: cli.md + - Configuration: configuration.md + - Concepts: concepts.md + - Troubleshooting: troubleshooting.md + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/ArcadeAI/agent-library diff --git a/pyproject.toml b/pyproject.toml index 7c1db66..f5bd35f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,10 @@ all = [ evals = [ "arcade-mcp[evals]>=1.14.0", ] +docs = [ + "mkdocs>=1.6.0", + "mkdocs-material>=9.5.0", +] [build-system] requires = ["hatchling"]