From eae88a1b2b69135a8d36333e26c1f044d4311edc Mon Sep 17 00:00:00 2001 From: arjunpatel7 <34616277+arjunpatel7@users.noreply.github.com> Date: Wed, 13 May 2026 21:02:20 +0000 Subject: [PATCH 1/2] [create-pull-request] automated change --- skills/pinecone-assistant/SKILL.md | 78 ++++ skills/pinecone-assistant/references/chat.md | 33 ++ .../pinecone-assistant/references/context.md | 39 ++ .../pinecone-assistant/references/create.md | 43 +++ skills/pinecone-assistant/references/list.md | 31 ++ skills/pinecone-assistant/references/sync.md | 51 +++ .../pinecone-assistant/references/upload.md | 49 +++ skills/pinecone-assistant/scripts/chat.py | 141 +++++++ skills/pinecone-assistant/scripts/context.py | 146 ++++++++ skills/pinecone-assistant/scripts/create.py | 126 +++++++ skills/pinecone-assistant/scripts/list.py | 200 ++++++++++ skills/pinecone-assistant/scripts/sync.py | 354 ++++++++++++++++++ skills/pinecone-assistant/scripts/upload.py | 222 +++++++++++ skills/pinecone-cli/SKILL.md | 156 ++++++++ .../references/command-reference.md | 237 ++++++++++++ .../references/troubleshooting.md | 136 +++++++ skills/pinecone-docs/SKILL.md | 86 +++++ .../pinecone-docs/references/data-formats.md | 81 ++++ skills/pinecone-full-text-search/SKILL.md | 354 ++++++++++++++++++ .../references/ingestion.md | 263 +++++++++++++ .../references/onboarding-walkthrough.md | 159 ++++++++ .../references/querying.md | 341 +++++++++++++++++ .../references/schema-design.md | 171 +++++++++ .../scripts/ingest.py | 281 ++++++++++++++ skills/pinecone-help/SKILL.md | 64 ++++ skills/pinecone-mcp/SKILL.md | 106 ++++++ skills/pinecone-query/SKILL.md | 84 +++++ skills/pinecone-quickstart/SKILL.md | 230 ++++++++++++ .../scripts/quickstart_complete.py | 74 ++++ skills/pinecone-quickstart/scripts/upsert.py | 47 +++ 30 files changed, 4383 insertions(+) create mode 100644 skills/pinecone-assistant/SKILL.md create mode 100644 skills/pinecone-assistant/references/chat.md create mode 100644 skills/pinecone-assistant/references/context.md create mode 100644 skills/pinecone-assistant/references/create.md create mode 100644 skills/pinecone-assistant/references/list.md create mode 100644 skills/pinecone-assistant/references/sync.md create mode 100644 skills/pinecone-assistant/references/upload.md create mode 100755 skills/pinecone-assistant/scripts/chat.py create mode 100755 skills/pinecone-assistant/scripts/context.py create mode 100755 skills/pinecone-assistant/scripts/create.py create mode 100755 skills/pinecone-assistant/scripts/list.py create mode 100644 skills/pinecone-assistant/scripts/sync.py create mode 100755 skills/pinecone-assistant/scripts/upload.py create mode 100644 skills/pinecone-cli/SKILL.md create mode 100644 skills/pinecone-cli/references/command-reference.md create mode 100644 skills/pinecone-cli/references/troubleshooting.md create mode 100644 skills/pinecone-docs/SKILL.md create mode 100644 skills/pinecone-docs/references/data-formats.md create mode 100644 skills/pinecone-full-text-search/SKILL.md create mode 100644 skills/pinecone-full-text-search/references/ingestion.md create mode 100644 skills/pinecone-full-text-search/references/onboarding-walkthrough.md create mode 100644 skills/pinecone-full-text-search/references/querying.md create mode 100644 skills/pinecone-full-text-search/references/schema-design.md create mode 100755 skills/pinecone-full-text-search/scripts/ingest.py create mode 100644 skills/pinecone-help/SKILL.md create mode 100644 skills/pinecone-mcp/SKILL.md create mode 100644 skills/pinecone-query/SKILL.md create mode 100644 skills/pinecone-quickstart/SKILL.md create mode 100644 skills/pinecone-quickstart/scripts/quickstart_complete.py create mode 100644 skills/pinecone-quickstart/scripts/upsert.py diff --git a/skills/pinecone-assistant/SKILL.md b/skills/pinecone-assistant/SKILL.md new file mode 100644 index 0000000..eeb3eb8 --- /dev/null +++ b/skills/pinecone-assistant/SKILL.md @@ -0,0 +1,78 @@ +--- +name: pinecone-assistant +description: Create, manage, and chat with Pinecone Assistants for document Q&A with citations. Handles all assistant operations - create, upload, sync, chat, context retrieval, and list. Recognizes natural language like "create an assistant from my docs", "ask my assistant about X", or "upload my docs to Pinecone". +--- + +# Pinecone Assistant + +Pinecone Assistant is a fully managed RAG service. Upload documents, ask questions, get cited answers. No embedding pipelines or infrastructure required. + +> All scripts are in `scripts/` relative to this skill directory. +> Run with: `uv run scripts/script_name.py [arguments]` + +## Operations + +| What to do | Script | Key args | +|---|---|---| +| Create an assistant | `scripts/create.py` | `--name` `--instructions` `--region` | +| Upload files | `scripts/upload.py` | `--assistant` `--source` `--patterns` | +| Sync files (incremental) | `scripts/sync.py` | `--assistant` `--source` `--delete-missing` `--dry-run` | +| Chat / ask a question | `scripts/chat.py` | `--assistant` `--message` | +| Get context snippets | `scripts/context.py` | `--assistant` `--query` `--top-k` | +| List assistants | `scripts/list.py` | `--files` `--json` | + +For full workflow details on any operation, read the relevant file in `references/`. + +--- + +## Natural Language Recognition + +Proactively handle these patterns without requiring explicit commands: + +**Create:** "create an assistant", "make an assistant called X", "set up an assistant for my docs" +→ See [references/create.md](references/create.md) + +**Upload:** "upload my docs", "add files to my assistant", "index my documentation" +→ See [references/upload.md](references/upload.md) + +**Sync:** "sync my docs", "update my assistant", "keep assistant in sync", "refresh from ./docs" +→ See [references/sync.md](references/sync.md) + +**Chat:** "ask my assistant about X", "what does my assistant know about X", "chat with X" +→ See [references/chat.md](references/chat.md) + +**Context:** "search my assistant for X", "find context about X" +→ See [references/context.md](references/context.md) + +**List:** "show my assistants", "what assistants do I have" +→ Run `uv run scripts/list.py` + +--- + +## Conversation Memory + +Track the last assistant used within the conversation: +- When a user creates or first uses an assistant, remember its name +- If user says "my assistant", "it", or "the assistant" → use the last one +- Briefly confirm which assistant you're using: "Asking docs-bot..." +- If ambiguous and multiple exist → ask the user to clarify + +--- + +## Multi-Step Requests + +Handle chained requests naturally. Example: + +> "Create an assistant called docs-bot, upload my ./docs folder, and ask what the main features are" + +1. `uv run scripts/create.py --name docs-bot` +2. `uv run scripts/upload.py --assistant docs-bot --source ./docs` +3. `uv run scripts/chat.py --assistant docs-bot --message "what are the main features?"` + +--- + +## Prerequisites + +- `PINECONE_API_KEY` must be available — terminal: `export PINECONE_API_KEY="your-key"`, or add to a `.env` file and run scripts with `uv run --env-file .env scripts/...` +- `uv` must be installed — [install uv](https://docs.astral.sh/uv/getting-started/installation/) +- Get a free API key at: https://app.pinecone.io/?sessionType=signup diff --git a/skills/pinecone-assistant/references/chat.md b/skills/pinecone-assistant/references/chat.md new file mode 100644 index 0000000..d4a984f --- /dev/null +++ b/skills/pinecone-assistant/references/chat.md @@ -0,0 +1,33 @@ +# Chat with Assistant + +Send a message to an assistant and receive a cited response. + +## Arguments + +- `--assistant` (required): Assistant name +- `--message` (required): The question or message +- `--stream` (optional flag): Enable streaming for faster perceived response + +## Workflow + +1. Parse arguments. If assistant missing, run `uv run scripts/list.py --json` and ask the user to select. +2. If message missing, prompt user for their question. +3. Execute: + ```bash + uv run scripts/chat.py \ + --assistant "assistant-name" \ + --message "user's question" + ``` +4. Display: + - Assistant's response + - Citations table: citation number, source file, page numbers, position + - Token usage statistics + +**Note:** File URLs in citations are temporary signed links (~1 hour). They are not displayed in output. + +## Troubleshooting + +**Assistant not found** — run list command, check for typos. +**No response or timeout** — verify assistant has files uploaded and status is "ready" (not "indexing"). +**Empty or poor responses** — assistant may lack relevant documents; suggest upload. +**PINECONE_API_KEY not set** — export the variable or add to a `.env` file, then restart your IDE/agent session. diff --git a/skills/pinecone-assistant/references/context.md b/skills/pinecone-assistant/references/context.md new file mode 100644 index 0000000..5208922 --- /dev/null +++ b/skills/pinecone-assistant/references/context.md @@ -0,0 +1,39 @@ +# Retrieve Context Snippets + +Get raw context snippets from an assistant's knowledge base without generating a full chat response. Useful for debugging, custom RAG workflows, or quick lookups. + +## Arguments + +- `--assistant` (required): Assistant name +- `--query` (required): Search query text +- `--top-k` (optional): Number of snippets — default `5`, max `16` +- `--snippet-size` (optional): Max tokens per snippet — default `2048` +- `--json` (optional flag): JSON output + +## Workflow + +1. Parse arguments. If missing, list assistants and prompt for selection. +2. Execute: + ```bash + uv run scripts/context.py \ + --assistant "assistant-name" \ + --query "search text" \ + --top-k 5 + ``` +3. Display snippets: file name, page numbers, relevance score, content. + +## Context vs Chat + +**Use context when:** you want raw snippets, are debugging knowledge, need source material, or are building custom workflows. +**Use chat when:** you want synthesized answers, citations in a conversational response, or multi-turn Q&A. + +## Interpreting Results + +- **Score:** Higher (closer to 1.0) = more relevant +- **Low scores (<0.5):** Weak match, assistant may need more relevant documents, or query is too broad/specific + +## Troubleshooting + +**No results** — try broader search terms; suggest uploading more documents. +**context method not available** — update SDK: `pip install --upgrade pinecone` (requires v8.0.0+). +**Assistant not found** — check name for typos, run list command. diff --git a/skills/pinecone-assistant/references/create.md b/skills/pinecone-assistant/references/create.md new file mode 100644 index 0000000..16d8c57 --- /dev/null +++ b/skills/pinecone-assistant/references/create.md @@ -0,0 +1,43 @@ +# Create Assistant + +Create a new Pinecone Assistant with custom configuration. + +## Arguments + +- `--name` (required): Unique name for the assistant +- `--instructions` (optional): Behavior directive (tone, format, language) +- `--region` (optional): `us` or `eu` — default `us` +- `--timeout` (optional): Seconds to wait for ready status — default `30` + +## Workflow + +1. Parse arguments. If name is missing, prompt the user. +2. Ask the user about region preference — US or EU. +3. Ask if user wants custom instructions. Offer examples: + - "Use professional technical tone and cite sources" + - "Respond in Spanish with formal language" +4. Execute: + ```bash + uv run scripts/create.py \ + --name "assistant-name" \ + --instructions "instructions" \ + --region "us" + ``` +5. Show assistant name, status, and host URL. +6. Offer to run upload next. + +## Naming Conventions + +Suggest: `{purpose}-{type}` — e.g. `docs-qa`, `support-bot`, `api-helper` +Avoid: `test`, `assistant1`, `my-assistant` + +## Post-Creation + +- Save the assistant host URL shown in output (needed for MCP config) +- View and manage at: https://app.pinecone.io/organizations/-/projects/-/assistant/ + +## Troubleshooting + +**Assistant name already exists** — list assistants and suggest a different name or delete the existing one. +**Timeout** — increase `--timeout 60`, check network connectivity. +**PINECONE_API_KEY not set** — export the variable or add to a `.env` file, then restart your IDE/agent session. diff --git a/skills/pinecone-assistant/references/list.md b/skills/pinecone-assistant/references/list.md new file mode 100644 index 0000000..078b387 --- /dev/null +++ b/skills/pinecone-assistant/references/list.md @@ -0,0 +1,31 @@ +# List Assistants + +List all Pinecone Assistants in the account with optional file details. + +## Arguments + +- `--files` (optional flag): Show file details for each assistant +- `--json` (optional flag): JSON output + +## Usage + +```bash +# Basic listing +uv run scripts/list.py + +# With file details +uv run scripts/list.py --files + +# JSON output +uv run scripts/list.py --json + +# JSON with files (useful for scripting) +uv run scripts/list.py --files --json +``` + +## Output + +**Without `--files`:** Table with name, region, status, host. +**With `--files`:** Adds file count column, plus detailed file tables per assistant showing file name, status, and ID. + +File status is color-coded: green = available, yellow = processing. diff --git a/skills/pinecone-assistant/references/sync.md b/skills/pinecone-assistant/references/sync.md new file mode 100644 index 0000000..504f257 --- /dev/null +++ b/skills/pinecone-assistant/references/sync.md @@ -0,0 +1,51 @@ +# Sync Files + +Incrementally sync local files to an assistant — only uploads new or changed files. Uses mtime and size to detect changes. + +## Arguments + +- `--assistant` (required): Assistant name +- `--source` (required): Local file or directory path +- `--delete-missing` (optional flag): Delete files from assistant that no longer exist locally +- `--dry-run` (optional flag): Preview changes without executing +- `--yes` / `-y` (optional flag): Skip confirmation prompt + +## Workflow + +1. Parse arguments. If missing, list assistants and prompt for selection. +2. Execute: + ```bash + uv run scripts/sync.py \ + --assistant "assistant-name" \ + --source "./docs" \ + [--delete-missing] \ + [--dry-run] \ + [--yes] + ``` +3. Script compares local files against stored metadata, shows summary, asks for confirmation (unless `--yes`). + +## Flags + +- **`--delete-missing`** — removes files from the assistant that no longer exist locally. Use when cleaning up removed content. +- **`--dry-run`** — shows exactly what would change with no side effects. Always recommend this first. +- **`--yes`** — skips confirmation. Useful for automation; combine with `--dry-run` to verify first. + +## Common Workflow + +```bash +# Preview first +uv run scripts/sync.py --assistant my-docs --source ./docs --dry-run + +# Then apply +uv run scripts/sync.py --assistant my-docs --source ./docs + +# Keep in sync after git pull +git pull +uv run scripts/sync.py --assistant my-docs --source ./docs --delete-missing +``` + +## Troubleshooting + +**Files showing as changed but content unchanged** — mtime updates on save even without content changes; harmless, file will be re-uploaded. +**Sync is slow** — each update = delete + re-upload (2 operations); use `--dry-run` first to check scope. +**No supported files found** — check source contains `.md`, `.txt`, `.pdf`, `.docx`, or `.json` files not in excluded directories. diff --git a/skills/pinecone-assistant/references/upload.md b/skills/pinecone-assistant/references/upload.md new file mode 100644 index 0000000..0c24292 --- /dev/null +++ b/skills/pinecone-assistant/references/upload.md @@ -0,0 +1,49 @@ +# Upload Files + +Upload files or directory contents to a Pinecone Assistant. + +**Supported formats:** `.md`, `.txt`, `.pdf`, `.docx`, `.json` +**Not supported:** Source code (`.py`, `.js`, `.ts`, etc.) — Assistant is optimized for natural language documents. + +## Arguments + +- `--assistant` (required): Assistant name +- `--source` (required): File path or directory to upload +- `--patterns` (optional): Comma-separated glob patterns — default: `*.md,*.txt,*.pdf,*.docx,*.json` +- `--exclude` (optional): Directories to exclude — default: `node_modules,.venv,.git,build,dist` +- `--metadata` (optional): JSON string of additional metadata + +## Workflow + +1. Parse arguments. If missing, list assistants and prompt for selection. +2. Use Glob to preview files. Show count and types. +3. **If code files detected:** Warn user and automatically filter them out: + ``` + ⚠️ Found 50 Python files. Assistant works with documents only — I'll skip the code files. + Found 25 Markdown and 8 PDF files to upload instead. + ``` +4. Confirm with the user before proceeding. +5. Execute: + ```bash + uv run scripts/upload.py \ + --assistant "assistant-name" \ + --source "./docs" \ + --patterns "*.md,*.pdf" + ``` +6. Show progress and results. Remind user files are being indexed. + +## Default Exclusions + +`node_modules`, `.venv`, `venv`, `.git`, `build`, `dist`, `__pycache__`, `.next`, `.cache` + +## Metadata Best Practices + +```bash +--metadata '{"source":"github","repo":"owner/repo","branch":"main"}' +``` + +## Troubleshooting + +**No files found** — check patterns match file types in directory; verify path exists. +**Upload failures** — check file format is supported; try smaller batches. +**>100 files** — ask user if they want to be more selective; suggest `./docs` subdirectory. diff --git a/skills/pinecone-assistant/scripts/chat.py b/skills/pinecone-assistant/scripts/chat.py new file mode 100755 index 0000000..843d3db --- /dev/null +++ b/skills/pinecone-assistant/scripts/chat.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +# /// script +# dependencies = [ +# "pinecone>=8.0.0", +# "typer>=0.15.0", +# "rich>=13.0.0", +# ] +# /// +""" +Chat with a Pinecone Assistant and receive cited responses. + +Usage: + uv run chat.py --assistant NAME --message "Your question" [--stream] + +Environment Variables: + PINECONE_API_KEY: Required Pinecone API key + +Output: + Assistant's response with citations to source documents +""" + +import os +import typer +from rich.console import Console +from rich.panel import Panel +from rich.table import Table +from pinecone import Pinecone +from pinecone_plugins.assistant.models.chat import Message + +app = typer.Typer() +console = Console() + + +@app.command() +def main( + assistant: str = typer.Option(..., "--assistant", "-a", help="Name of the assistant to chat with"), + message: str = typer.Option(..., "--message", "-m", help="Your question or message"), +): + """Chat with a Pinecone Assistant and receive answers with source citations.""" + + # Check for API key + api_key = os.environ.get("PINECONE_API_KEY") + if not api_key: + console.print("[red]Error: PINECONE_API_KEY environment variable not set[/red]") + console.print("\nGet your API key from: https://app.pinecone.io/?sessionType=signup") + raise typer.Exit(1) + + try: + # Initialize Pinecone client + pc = Pinecone(api_key=api_key,source_tag="claude_code_plugin:assistant") + asst = pc.assistant.Assistant(assistant_name=assistant) + + # Create message + user_msg = Message(role="user", content=message) + + # Display user question + console.print(Panel(f"[bold cyan]Question:[/bold cyan] {message}", border_style="cyan")) + + # Get response + with console.status("[bold blue]Thinking...[/bold blue]"): + response = asst.chat(messages=[user_msg], stream=False) + + answer_content = response.message.content + citations = response.citations if hasattr(response, 'citations') else [] + usage = response.usage if hasattr(response, 'usage') else None + + # Display assistant's response (same for both modes) + console.print("\n[bold green]Answer:[/bold green]\n") + + if answer_content: + console.print(Panel(answer_content, border_style="green", title="Assistant Response")) + else: + console.print("[yellow]No response content received[/yellow]") + + # Display citations if available + if citations and len(citations) > 0: + + console.print("\n[bold yellow]Citations:[/bold yellow]\n") + + citations_table = Table(show_header=True, header_style="bold yellow") + citations_table.add_column("#", style="dim", width=4) + citations_table.add_column("File", style="cyan", width=40) + citations_table.add_column("Pages", style="blue", width=15) + citations_table.add_column("Position", style="green", width=10) + + citation_num = 0 + for citation in citations: + # Each citation has a list of references + if hasattr(citation, 'references') and citation.references: + for reference in citation.references: + citation_num += 1 + + # Get file name + file_name = "Unknown" + if hasattr(reference, 'file') and hasattr(reference.file, 'name'): + file_name = reference.file.name + + # Get pages + pages = [] + if hasattr(reference, 'pages') and reference.pages: + pages = reference.pages + + # Format pages + if pages: + pages_str = ", ".join(str(p) for p in pages) + else: + pages_str = "N/A" + + # Get position from citation + position = getattr(citation, 'position', 'N/A') + + citations_table.add_row( + str(citation_num), + file_name, + pages_str, + str(position) + ) + + console.print(citations_table) + + # Optionally show download links + console.print("\n[dim]Tip: File URLs are temporary signed links valid for ~1 hour[/dim]") + + # Display token usage + if usage: + usage_info = f"""[dim]Tokens used:[/dim] +• Prompt: {getattr(usage, 'prompt_tokens', 'N/A')} +• Completion: {getattr(usage, 'completion_tokens', 'N/A')} +• Total: {getattr(usage, 'total_tokens', 'N/A')}""" + console.print(Panel(usage_info, border_style="dim", title="Usage Stats")) + + # Follow-up suggestion + console.print(f"\n[dim]Continue the conversation with another message using the same command[/dim]") + + except Exception as e: + console.print(f"[red]Error: {e}[/red]") + raise typer.Exit(1) + + +if __name__ == "__main__": + app() diff --git a/skills/pinecone-assistant/scripts/context.py b/skills/pinecone-assistant/scripts/context.py new file mode 100755 index 0000000..81ae65c --- /dev/null +++ b/skills/pinecone-assistant/scripts/context.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +# /// script +# dependencies = [ +# "pinecone>=8.0.0", +# "typer>=0.15.0", +# "rich>=13.0.0", +# ] +# /// +""" +Retrieve context snippets from a Pinecone Assistant's knowledge base. + +Usage: + uv run context.py --assistant NAME --query "search text" [--top-k 5] [--json] + +Environment Variables: + PINECONE_API_KEY: Required Pinecone API key + +Output: + Relevant context snippets with file sources, page numbers, and relevance scores +""" + +import os +import json as json_module +import typer +from rich.console import Console +from rich.panel import Panel +from rich.table import Table +from rich.text import Text +from pinecone import Pinecone + +app = typer.Typer() +console = Console() + + +@app.command() +def main( + assistant: str = typer.Option(..., "--assistant", "-a", help="Name of the assistant"), + query: str = typer.Option(..., "--query", "-q", help="Search query text"), + top_k: int = typer.Option(5, "--top-k", "-k", help="Number of results to return (max 16)"), + snippet_size: int = typer.Option(1024, "--snippet-size", "-s", help="Maximum tokens per snippet"), + json: bool = typer.Option(False, "--json", help="Output in JSON format"), +): + """Retrieve relevant context snippets from an assistant's knowledge base.""" + + # Check for API key + api_key = os.environ.get("PINECONE_API_KEY") + if not api_key: + console.print("[red]Error: PINECONE_API_KEY environment variable not set[/red]") + console.print("\nGet your API key from: https://app.pinecone.io/?sessionType=signup") + raise typer.Exit(1) + + try: + # Initialize Pinecone client + pc = Pinecone(api_key=api_key, source_tag="claude_code_plugin:assistant") + asst = pc.assistant.Assistant(assistant_name=assistant) + + # Display query + if not json: + console.print(Panel(f"[bold cyan]Query:[/bold cyan] {query}", border_style="cyan")) + + # Retrieve context + with console.status("[bold blue]Searching knowledge base...[/bold blue]", spinner="dots"): + response = asst.context(query=query, top_k=top_k, snippet_size=snippet_size) + + # Get snippets from response + snippets = response.snippets if hasattr(response, 'snippets') else [] + + if json: + # JSON output + results = [] + for snippet in snippets: + file_name = "Unknown" + pages = [] + if hasattr(snippet, 'reference') and snippet.reference: + ref = snippet.reference + if hasattr(ref, 'file') and hasattr(ref.file, 'name'): + file_name = ref.file.name + if hasattr(ref, 'pages') and ref.pages: + pages = ref.pages + + results.append({ + "file_name": file_name, + "pages": pages, + "content": getattr(snippet, 'content', ''), + "score": getattr(snippet, 'score', 0.0), + "type": getattr(snippet, 'type', 'text'), + }) + print(json_module.dumps({"snippets": results, "count": len(results)}, indent=2)) + else: + # Rich formatted output + if not snippets or len(snippets) == 0: + console.print("[yellow]No context found for this query[/yellow]") + return + + console.print(f"\n[bold]Found {len(snippets)} relevant snippet(s):[/bold]\n") + + for idx, snippet in enumerate(snippets, 1): + # Extract file info from reference + file_name = "Unknown" + pages = [] + if hasattr(snippet, 'reference') and snippet.reference: + ref = snippet.reference + if hasattr(ref, 'file') and hasattr(ref.file, 'name'): + file_name = ref.file.name + if hasattr(ref, 'pages') and ref.pages: + pages = ref.pages + + score = getattr(snippet, 'score', 0.0) + content = getattr(snippet, 'content', '') + + # Create header + header = f"#{idx} - {file_name}" + if pages: + pages_str = ", ".join(str(p) for p in pages) + header += f" (Page {pages_str})" + header += f" - Score: {score:.3f}" if isinstance(score, (int, float)) else f" - Score: {score}" + + console.print(Panel( + content, + title=header, + border_style="blue", + subtitle=f"[dim]Relevance: {score:.2%}[/dim]" if isinstance(score, (int, float)) else None + )) + console.print() + + # Suggest next action + next_action = f"""[bold]Next steps:[/bold] +• Ask a question: [cyan]/pinecone:assistant-chat assistant {assistant} message [your question][/cyan] +• Upload more files: [cyan]/pinecone:assistant-upload assistant {assistant} source [path][/cyan]""" + console.print(Panel(next_action, title="What's Next?", border_style="green")) + + except AttributeError as e: + # Handle case where context method doesn't exist or response structure is different + console.print(f"[red]Error: Context retrieval failed[/red]") + console.print(f"[dim]Details: {e}[/dim]") + console.print("\n[yellow]Note:[/yellow] Context API requires SDK version with assistant.context() support") + console.print("\n[yellow]Try using chat instead:[/yellow]") + console.print(f" /pinecone:assistant-chat assistant {assistant} message \"{query}\"") + raise typer.Exit(1) + except Exception as e: + console.print(f"[red]Error: {e}[/red]") + raise typer.Exit(1) + + +if __name__ == "__main__": + app() diff --git a/skills/pinecone-assistant/scripts/create.py b/skills/pinecone-assistant/scripts/create.py new file mode 100755 index 0000000..c135b56 --- /dev/null +++ b/skills/pinecone-assistant/scripts/create.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +# /// script +# dependencies = [ +# "pinecone>=8.0.0", +# "typer>=0.15.0", +# "rich>=13.0.0", +# ] +# /// +""" +Create a new Pinecone Assistant. + +Usage: + uv run create.py --name ASSISTANT_NAME [--instructions TEXT] [--region us|eu] [--timeout SECONDS] + +Environment Variables: + PINECONE_API_KEY: Required Pinecone API key + +Output: + Success message with assistant details including host URL for MCP configuration +""" + +import os +import typer +from rich.console import Console +from rich.panel import Panel +from rich.table import Table +from pinecone import Pinecone + +app = typer.Typer() +console = Console() + + +@app.command() +def main( + name: str = typer.Option(..., "--name", "-n", help="Unique name for the assistant"), + instructions: str = typer.Option( + "", + "--instructions", + "-i", + help="Instructions for assistant behavior (max 16KB)", + ), + region: str = typer.Option( + "us", + "--region", + "-r", + help="Deployment region: 'us' or 'eu'", + ), + timeout: int = typer.Option( + 30, + "--timeout", + "-t", + help="Seconds to wait for ready status", + ), +): + """Create a new Pinecone Assistant for document Q&A with citations.""" + + # Validate region + if region not in ["us", "eu"]: + console.print("[red]Error: Region must be 'us' or 'eu'[/red]") + raise typer.Exit(1) + + # Check for API key + api_key = os.environ.get("PINECONE_API_KEY") + if not api_key: + console.print("[red]Error: PINECONE_API_KEY environment variable not set[/red]") + console.print("\nGet your API key from: https://app.pinecone.io/?sessionType=signup") + raise typer.Exit(1) + + try: + # Initialize Pinecone client + with console.status(f"[bold blue]Creating assistant '{name}'...[/bold blue]"): + pc = Pinecone(api_key=api_key, source_tag="claude_code_plugin:assistant") + + # Create assistant + assistant = pc.assistant.create_assistant( + assistant_name=name, + instructions=instructions if instructions else None, + region=region, + timeout=timeout, + metadata={"agentic-ide-source":"claude-code-plugin"} + ) + + # Success message + console.print(f"\n[bold green]✓ Assistant '{name}' created successfully![/bold green]\n") + + # Display assistant details in a table + table = Table(show_header=False, box=None) + table.add_column("Property", style="cyan") + table.add_column("Value", style="white") + + table.add_row("Name", assistant.name) + table.add_row("Region", region) + table.add_row("Status", f"[yellow]{assistant.status}[/yellow]") + table.add_row("Host", getattr(assistant, "host", "N/A")) + if instructions: + instructions_preview = instructions[:80] + "..." if len(instructions) > 80 else instructions + table.add_row("Instructions", instructions_preview) + + console.print(table) + + # MCP configuration info + host = getattr(assistant, "host", "") + if host: + mcp_info = f"""[bold]MCP Endpoint:[/bold] +{host}/mcp/assistants/{name} + +[bold]Set environment variable:[/bold] +export PINECONE_ASSISTANT_HOST="{host}" +""" + console.print(Panel(mcp_info, title="MCP Configuration", border_style="blue")) + + # Next steps + next_steps = f"""[bold]Next steps:[/bold] +1. Upload files: [cyan]/pinecone:assistant-upload assistant {name} source [path][/cyan] +2. Chat: [cyan]/pinecone:assistant-chat assistant {name} message [your question][/cyan] +3. Get context: [cyan]/pinecone:assistant-context assistant {name} query [search][/cyan]""" + + console.print(Panel(next_steps, title="What's Next?", border_style="green")) + + except Exception as e: + console.print(f"[red]Error creating assistant: {e}[/red]") + raise typer.Exit(1) + + +if __name__ == "__main__": + app() diff --git a/skills/pinecone-assistant/scripts/list.py b/skills/pinecone-assistant/scripts/list.py new file mode 100755 index 0000000..bd8a373 --- /dev/null +++ b/skills/pinecone-assistant/scripts/list.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +# /// script +# dependencies = [ +# "pinecone>=8.0.0", +# "typer>=0.15.0", +# "rich>=13.0.0", +# ] +# /// +""" +List all Pinecone Assistants in the account. + +Usage: + uv run list.py [--json] [--files] + +Environment Variables: + PINECONE_API_KEY: Required Pinecone API key + +Output: + Formatted table or JSON list of assistants with name, region, status, and host + Optionally include files for each assistant with --files flag +""" + +import os +import sys +import json +import typer +from rich.console import Console +from rich.table import Table +from rich.panel import Panel +from pinecone import Pinecone + +app = typer.Typer() +console = Console() + + +@app.command() +def main( + json_output: bool = typer.Option(False, "--json", help="Output in JSON format"), + files: bool = typer.Option(False, "--files", "-f", help="Include file listing for each assistant"), +): + """List all Pinecone Assistants in your account.""" + + # Check for API key + api_key = os.environ.get('PINECONE_API_KEY') + if not api_key: + console.print("[red]Error: PINECONE_API_KEY environment variable not set[/red]") + console.print("\nGet your API key from: https://app.pinecone.io/?sessionType=signup") + raise typer.Exit(1) + + try: + # Initialize Pinecone client + pc = Pinecone(api_key=api_key, source_tag="claude_code_plugin:assistant") + + # List assistants + assistants = pc.assistant.list_assistants() + + if not assistants: + if json_output: + print(json.dumps({"assistants": [], "count": 0})) + else: + console.print("[yellow]No assistants found.[/yellow]\n") + console.print("Create your first assistant with:") + console.print(" [cyan]/pinecone:assistant-create name [assistant-name][/cyan]") + return + + if json_output: + # JSON output + assistants_data = [] + for asst in assistants: + asst_data = { + "name": asst.name, + "region": getattr(asst, 'region', 'unknown'), + "status": asst.status, + "host": getattr(asst, 'host', ''), + } + + if files: + # Get files for this assistant + try: + assistant_instance = pc.assistant.Assistant(assistant_name=asst.name) + file_list = assistant_instance.list_files() + asst_data["files"] = [ + { + "name": f.name, + "id": f.id, + "status": f.status, + "metadata": getattr(f, 'metadata', {}), + } + for f in file_list + ] + asst_data["file_count"] = len(file_list) + except Exception as e: + asst_data["files"] = [] + asst_data["file_count"] = 0 + asst_data["file_error"] = str(e) + + assistants_data.append(asst_data) + + result = { + "assistants": assistants_data, + "count": len(assistants) + } + print(json.dumps(result, indent=2)) + else: + # Rich table output + console.print(f"\n[bold]Found {len(assistants)} assistant(s):[/bold]\n") + + # Assistants table + table = Table(show_header=True, header_style="bold cyan") + table.add_column("Name", style="green", width=30) + table.add_column("Region", style="blue", width=10) + table.add_column("Status", style="yellow", width=15) + if files: + table.add_column("Files", style="magenta", width=10) + table.add_column("Host", style="dim", width=40 if files else 50) + + for asst in assistants: + name = asst.name + region = getattr(asst, 'region', 'unknown') + status = asst.status + host = getattr(asst, 'host', '') + + # Color code status + if status == 'ready': + status_display = f"[green]{status}[/green]" + elif status == 'indexing': + status_display = f"[yellow]{status}[/yellow]" + else: + status_display = status + + if files: + # Get file count for this assistant + try: + assistant_instance = pc.assistant.Assistant(assistant_name=asst.name) + file_list = assistant_instance.list_files() + file_count = str(len(file_list)) + except Exception: + file_count = "?" + + table.add_row(name, region, status_display, file_count, host) + else: + table.add_row(name, region, status_display, host) + + console.print(table) + console.print() + + # If --files flag is set, show detailed file listing for each assistant + if files: + console.print("[bold]File Details:[/bold]\n") + for asst in assistants: + try: + assistant_instance = pc.assistant.Assistant(assistant_name=asst.name) + file_list = assistant_instance.list_files() + + if file_list: + # Create a table for this assistant's files + file_table = Table(show_header=True, header_style="bold blue", title=f"[cyan]{asst.name}[/cyan]") + file_table.add_column("#", style="dim", width=4) + file_table.add_column("File Name", style="green", width=50) + file_table.add_column("Status", style="yellow", width=15) + file_table.add_column("ID", style="dim", width=30) + + for idx, file_obj in enumerate(file_list, 1): + file_name = file_obj.name + file_id = file_obj.id + file_status = file_obj.status + + # Color code file status + if file_status == 'available': + file_status_display = f"[green]{file_status}[/green]" + elif file_status == 'processing': + file_status_display = f"[yellow]{file_status}[/yellow]" + else: + file_status_display = file_status + + file_table.add_row(str(idx), file_name, file_status_display, file_id) + + console.print(file_table) + console.print() + else: + console.print(f"[dim]{asst.name}: No files uploaded[/dim]\n") + except Exception as e: + console.print(f"[red]Error listing files for {asst.name}: {e}[/red]\n") + + # Next steps panel + next_steps = """[bold]Next steps:[/bold] +• List with files: [cyan]/pinecone:assistant-list --files[/cyan] +• Chat: [cyan]/pinecone:assistant-chat assistant [name] message [your question][/cyan] +• Upload: [cyan]/pinecone:assistant-upload assistant [name] source [path][/cyan] +• Context: [cyan]/pinecone:assistant-context assistant [name] query [search][/cyan]""" + + console.print(Panel(next_steps, title="Available Commands", border_style="blue")) + + except Exception as e: + console.print(f"[red]Error listing assistants: {e}[/red]") + raise typer.Exit(1) + + +if __name__ == "__main__": + app() diff --git a/skills/pinecone-assistant/scripts/sync.py b/skills/pinecone-assistant/scripts/sync.py new file mode 100644 index 0000000..13c98a5 --- /dev/null +++ b/skills/pinecone-assistant/scripts/sync.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +# /// script +# dependencies = [ +# "pinecone>=8.0.0", +# "typer>=0.15.0", +# "rich>=13.0.0", +# ] +# /// +""" +Sync local files to a Pinecone Assistant, only uploading new or changed files. + +Usage: + uv run sync.py --assistant NAME --source PATH [--delete-missing] [--dry-run] + +Environment Variables: + PINECONE_API_KEY: Required Pinecone API key + +Output: + Shows files to add, update, and optionally delete, with confirmation prompt +""" + +import os +import hashlib +from pathlib import Path +from datetime import datetime, timezone +import typer +from rich.console import Console +from rich.panel import Panel +from rich.table import Table +from rich.progress import Progress, SpinnerColumn, TextColumn +from pinecone import Pinecone + +app = typer.Typer() +console = Console() + +# Supported file extensions +SUPPORTED_EXTENSIONS = {'.md', '.txt', '.pdf', '.docx', '.json'} + +# Directories to exclude +EXCLUDE_DIRS = {'node_modules', '.venv', '.git', 'build', 'dist', '__pycache__', '.pytest_cache'} + + +def should_exclude_path(path: Path, source_root: Path) -> bool: + """Check if path should be excluded based on directory patterns.""" + try: + rel_path = path.relative_to(source_root) + for part in rel_path.parts: + if part in EXCLUDE_DIRS or part.startswith('.'): + return True + except ValueError: + return True + return False + + +def find_files(source_path: Path) -> list[Path]: + """Find all supported files in source directory, excluding common build/dependency dirs.""" + files = [] + + if source_path.is_file(): + if source_path.suffix.lower() in SUPPORTED_EXTENSIONS: + return [source_path] + else: + return [] + + for file_path in source_path.rglob('*'): + if file_path.is_file(): + if file_path.suffix.lower() in SUPPORTED_EXTENSIONS: + if not should_exclude_path(file_path, source_path): + files.append(file_path) + + return sorted(files) + + +def get_file_info(file_path: Path): + """Get file modification time and size.""" + stat = file_path.stat() + return { + 'mtime': stat.st_mtime, + 'size': stat.st_size, + } + + +def file_changed(local_info: dict, remote_metadata: dict) -> bool: + """Check if local file differs from remote using mtime and size.""" + remote_mtime = remote_metadata.get('mtime') + remote_size = remote_metadata.get('size') + + if remote_mtime is None or remote_size is None: + # No stored metadata, assume changed + return True + + return (local_info['mtime'] != float(remote_mtime) or + local_info['size'] != int(remote_size)) + + +@app.command() +def main( + assistant: str = typer.Option(..., "--assistant", "-a", help="Name of the assistant"), + source: str = typer.Option(..., "--source", "-s", help="Local file or directory path"), + delete_missing: bool = typer.Option(False, "--delete-missing", help="Delete files from assistant that don't exist locally"), + dry_run: bool = typer.Option(False, "--dry-run", help="Show what would change without making changes"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), +): + """Sync local files to Pinecone Assistant, only uploading new or changed files.""" + + # Check for API key + api_key = os.environ.get("PINECONE_API_KEY") + if not api_key: + console.print("[red]Error: PINECONE_API_KEY environment variable not set[/red]") + console.print("\nGet your API key from: https://app.pinecone.io/?sessionType=signup") + raise typer.Exit(1) + + # Validate source path + source_path = Path(source).resolve() + if not source_path.exists(): + console.print(f"[red]Error: Source path does not exist: {source}[/red]") + raise typer.Exit(1) + + try: + # Initialize Pinecone client + pc = Pinecone(api_key=api_key, source_tag="claude_code_plugin:assistant") + asst = pc.assistant.Assistant(assistant_name=assistant) + + console.print(Panel( + f"[bold cyan]Assistant:[/bold cyan] {assistant}\n" + f"[bold cyan]Source:[/bold cyan] {source_path}", + title="Sync Configuration", + border_style="cyan" + )) + + # Step 1: Get current files in assistant + with console.status("[bold blue]Fetching assistant files...[/bold blue]", spinner="dots"): + remote_files = asst.list_files() + + # Build map of file_path -> file object + remote_file_map = {} + for f in remote_files: + metadata = getattr(f, 'metadata', {}) or {} + file_path = metadata.get('file_path', f.name) + remote_file_map[file_path] = { + 'file_obj': f, + 'metadata': metadata + } + + console.print(f"[dim]Found {len(remote_files)} file(s) in assistant[/dim]\n") + + # Step 2: Find local files + with console.status("[bold blue]Scanning local files...[/bold blue]", spinner="dots"): + local_files = find_files(source_path) + + if not local_files: + console.print("[yellow]No supported files found in source path[/yellow]") + console.print(f"Supported extensions: {', '.join(sorted(SUPPORTED_EXTENSIONS))}") + raise typer.Exit(0) + + console.print(f"[dim]Found {len(local_files)} local file(s)[/dim]\n") + + # Step 3: Determine what needs syncing + to_upload = [] # New files + to_update = [] # Changed files (delete + re-upload) + to_delete = [] # Files in assistant but not local + unchanged = [] # Files that match + + # Track which remote files we've seen + seen_remote_paths = set() + + for local_file in local_files: + # Get relative path from source root + if source_path.is_file(): + rel_path = local_file.name + else: + rel_path = str(local_file.relative_to(source_path)) + + local_info = get_file_info(local_file) + + if rel_path in remote_file_map: + # File exists remotely, check if changed + seen_remote_paths.add(rel_path) + remote_info = remote_file_map[rel_path] + + if file_changed(local_info, remote_info['metadata']): + to_update.append({ + 'local_path': local_file, + 'rel_path': rel_path, + 'remote_file_id': remote_info['file_obj'].id, + 'local_info': local_info + }) + else: + unchanged.append(rel_path) + else: + # New file + to_upload.append({ + 'local_path': local_file, + 'rel_path': rel_path, + 'local_info': local_info + }) + + # Find files to delete (in remote but not local) + if delete_missing: + for rel_path, remote_info in remote_file_map.items(): + if rel_path not in seen_remote_paths: + to_delete.append({ + 'rel_path': rel_path, + 'remote_file_id': remote_info['file_obj'].id + }) + + # Step 4: Show summary + console.print("[bold]Sync Summary:[/bold]\n") + + summary_table = Table(show_header=True, header_style="bold cyan") + summary_table.add_column("Action", style="yellow", width=15) + summary_table.add_column("Count", style="green", width=10) + + summary_table.add_row("New files", str(len(to_upload))) + summary_table.add_row("Updated files", str(len(to_update))) + if delete_missing: + summary_table.add_row("Deleted files", str(len(to_delete))) + summary_table.add_row("Unchanged", str(len(unchanged))) + + console.print(summary_table) + console.print() + + # Show details if there are changes + if to_upload: + console.print("[bold green]Files to upload:[/bold green]") + for item in to_upload[:10]: # Show first 10 + console.print(f" + {item['rel_path']}") + if len(to_upload) > 10: + console.print(f" ... and {len(to_upload) - 10} more") + console.print() + + if to_update: + console.print("[bold yellow]Files to update:[/bold yellow]") + for item in to_update[:10]: + console.print(f" ~ {item['rel_path']}") + if len(to_update) > 10: + console.print(f" ... and {len(to_update) - 10} more") + console.print() + + if to_delete: + console.print("[bold red]Files to delete:[/bold red]") + for item in to_delete[:10]: + console.print(f" - {item['rel_path']}") + if len(to_delete) > 10: + console.print(f" ... and {len(to_delete) - 10} more") + console.print() + + # If no changes, exit early + if not (to_upload or to_update or to_delete): + console.print("[green]✓ All files are up to date![/green]") + return + + # Dry run mode + if dry_run: + console.print("[yellow]Dry run mode: No changes made[/yellow]") + return + + # Confirmation prompt + if not yes: + proceed = typer.confirm("\nProceed with sync?") + if not proceed: + console.print("[yellow]Sync cancelled[/yellow]") + return + + console.print() + + # Step 5: Execute sync + uploaded_count = 0 + updated_count = 0 + deleted_count = 0 + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console + ) as progress: + + # Upload new files + if to_upload: + task = progress.add_task(f"Uploading {len(to_upload)} new file(s)...", total=len(to_upload)) + for item in to_upload: + try: + asst.upload_file( + file_path=str(item['local_path']), + metadata={ + 'file_path': item['rel_path'], + 'mtime': item['local_info']['mtime'], + 'size': item['local_info']['size'], + 'uploaded_at': datetime.now(timezone.utc).isoformat(), + 'source': 'sync_script', + }, + timeout=None + ) + uploaded_count += 1 + progress.advance(task) + except Exception as e: + console.print(f"[red]Failed to upload {item['rel_path']}: {e}[/red]") + + # Update changed files (delete old + upload new) + if to_update: + task = progress.add_task(f"Updating {len(to_update)} file(s)...", total=len(to_update) * 2) + for item in to_update: + try: + # Delete old version + asst.delete_file(file_id=item['remote_file_id']) + progress.advance(task) + + # Upload new version + asst.upload_file( + file_path=str(item['local_path']), + metadata={ + 'file_path': item['rel_path'], + 'mtime': item['local_info']['mtime'], + 'size': item['local_info']['size'], + 'uploaded_at': datetime.now(timezone.utc).isoformat(), + 'source': 'sync_script', + }, + timeout=None + ) + updated_count += 1 + progress.advance(task) + except Exception as e: + console.print(f"[red]Failed to update {item['rel_path']}: {e}[/red]") + + # Delete missing files + if to_delete: + task = progress.add_task(f"Deleting {len(to_delete)} file(s)...", total=len(to_delete)) + for item in to_delete: + try: + asst.delete_file(file_id=item['remote_file_id']) + deleted_count += 1 + progress.advance(task) + except Exception as e: + console.print(f"[red]Failed to delete {item['rel_path']}: {e}[/red]") + + # Final summary + console.print() + console.print(Panel( + f"[green]✓ Sync complete![/green]\n\n" + f"Uploaded: {uploaded_count}\n" + f"Updated: {updated_count}\n" + + (f"Deleted: {deleted_count}\n" if delete_missing else "") + + f"Unchanged: {len(unchanged)}", + title="Results", + border_style="green" + )) + + except Exception as e: + console.print(f"[red]Error: {e}[/red]") + raise typer.Exit(1) + + +if __name__ == "__main__": + app() diff --git a/skills/pinecone-assistant/scripts/upload.py b/skills/pinecone-assistant/scripts/upload.py new file mode 100755 index 0000000..4484e75 --- /dev/null +++ b/skills/pinecone-assistant/scripts/upload.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +# /// script +# dependencies = [ +# "pinecone>=8.0.0", +# "typer>=0.15.0", +# "rich>=13.0.0", +# ] +# /// +""" +Upload files or repository contents to a Pinecone Assistant. + +IMPORTANT: Only uploads DOCUMENTATION and DATA files. +Supported: DOCX (.docx), JSON (.json), Markdown (.md), PDF (.pdf), Text (.txt) +Code files are NOT supported by Pinecone Assistant. + +Usage: + uv run upload.py --assistant NAME --source PATH [--patterns "*.md,*.pdf,*.docx"] + +Environment Variables: + PINECONE_API_KEY: Required Pinecone API key + +Output: + Progress updates and summary of uploaded files +""" + +import os +import glob +from pathlib import Path +from typing import List +from datetime import datetime, timezone +import typer +from rich.console import Console +from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn +from rich.table import Table +from rich.panel import Panel +from pinecone import Pinecone + +app = typer.Typer() +console = Console() + +# Default file patterns - DOCUMENTATION ONLY +# Assistant supports: DOCX, JSON, Markdown, PDF, Text +DEFAULT_PATTERNS = ["**/*.md", "**/*.txt", "**/*.pdf", "**/*.docx", "**/*.json"] + +# Default directories to exclude +DEFAULT_EXCLUDES = ["node_modules", ".venv", "venv", ".git", "build", "dist", "__pycache__", ".next", ".cache"] + + +def find_files(source_path: str, patterns: List[str], excludes: List[str]) -> List[Path]: + """Find files matching patterns, excluding certain directories.""" + source = Path(source_path) + + if not source.exists(): + console.print(f"[red]Error: Path '{source_path}' does not exist[/red]") + raise typer.Exit(1) + + # If it's a single file, return it + if source.is_file(): + return [source] + + # Otherwise, scan directory + files = [] + for pattern in patterns: + matched = glob.glob(str(source / pattern), recursive=True) + files.extend([Path(f) for f in matched]) + + # Filter out excluded directories + filtered_files = [] + for file_path in files: + # Check if any exclude pattern is in the path + if not any(excl in str(file_path) for excl in excludes): + filtered_files.append(file_path) + + return sorted(set(filtered_files)) + + +@app.command() +def main( + assistant: str = typer.Option(..., "--assistant", "-a", help="Name of the assistant to upload to"), + source: str = typer.Option(..., "--source", "-s", help="File or directory path to upload"), + patterns: str = typer.Option( + ",".join(DEFAULT_PATTERNS), + "--patterns", + "-p", + help="Comma-separated glob patterns for documentation files (e.g., '*.md,*.pdf')", + ), + exclude: str = typer.Option( + ",".join(DEFAULT_EXCLUDES), + "--exclude", + "-e", + help="Comma-separated directories to exclude", + ), + metadata_json: str = typer.Option( + "", + "--metadata", + "-m", + help="Additional metadata as JSON string", + ), +): + """Upload documentation files to a Pinecone Assistant. + + NOTE: Only documentation files (markdown, text, PDF) are supported. + Code files are not recommended for Pinecone Assistant. + """ + + # Check for API key + api_key = os.environ.get("PINECONE_API_KEY") + if not api_key: + console.print("[red]Error: PINECONE_API_KEY environment variable not set[/red]") + console.print("\nGet your API key from: https://app.pinecone.io/?sessionType=signup") + raise typer.Exit(1) + + # Parse patterns and excludes + pattern_list = [p.strip() for p in patterns.split(",")] + exclude_list = [e.strip() for e in exclude.split(",")] + + # Parse additional metadata if provided + extra_metadata = {} + if metadata_json: + import json + try: + extra_metadata = json.loads(metadata_json) + except json.JSONDecodeError: + console.print("[red]Error: Invalid JSON in --metadata parameter[/red]") + raise typer.Exit(1) + + try: + # Initialize Pinecone client + pc = Pinecone(api_key=api_key, source_tag="claude_code_plugin:assistant") + asst = pc.assistant.Assistant(assistant_name=assistant) + + # Find files to upload + console.print(f"\n[bold]Scanning for documentation files in:[/bold] {source}") + console.print(f"[dim]Patterns: {', '.join(pattern_list)}[/dim]\n") + + files = find_files(source, pattern_list, exclude_list) + + if not files: + console.print("[yellow]No documentation files found matching the specified patterns[/yellow]") + console.print("\n[dim]Tip: Pinecone Assistant works with .md, .txt, and .pdf files[/dim]") + return + + console.print(f"[green]Found {len(files)} documentation file(s) to upload[/green]\n") + + # Upload files with progress bar + uploaded = 0 + failed = 0 + failed_files = [] + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(), + TaskProgressColumn(), + console=console, + ) as progress: + task = progress.add_task("[cyan]Uploading files...", total=len(files)) + + for file_path in files: + try: + # Build metadata + rel_path = os.path.relpath(str(file_path), source) + stat = file_path.stat() + metadata = { + "source": "upload_script", + "file_path": rel_path, + "file_type": file_path.suffix, + "content_type": "documentation", + "mtime": stat.st_mtime, + "size": stat.st_size, + "uploaded_at": datetime.now(timezone.utc).isoformat(), + **extra_metadata, + } + + # Upload file + asst.upload_file( + file_path=str(file_path), + metadata=metadata, + timeout=None, + ) + uploaded += 1 + progress.update(task, advance=1, description=f"[cyan]Uploaded: {rel_path}") + + except Exception as e: + failed += 1 + failed_files.append((str(file_path), str(e))) + progress.update(task, advance=1) + + # Summary table + console.print() + summary = Table(show_header=False, box=None) + summary.add_column("Status", style="bold") + summary.add_column("Count") + + summary.add_row("[green]✓ Uploaded[/green]", str(uploaded)) + if failed > 0: + summary.add_row("[red]✗ Failed[/red]", str(failed)) + + console.print(Panel(summary, title="Upload Summary", border_style="blue")) + + # Show failed files if any + if failed_files: + console.print("\n[bold red]Failed uploads:[/bold red]") + for file_path, error in failed_files: + console.print(f" • {file_path}: [red]{error}[/red]") + + # Next steps + if uploaded > 0: + next_steps = f"""[bold]Next steps:[/bold] +• Chat: [cyan]/pinecone:assistant-chat assistant {assistant} message [your question][/cyan] +• Context: [cyan]/pinecone:assistant-context assistant {assistant} query [search][/cyan] + +[dim]Note: Files are being processed and will be available shortly[/dim]""" + console.print(Panel(next_steps, title="What's Next?", border_style="green")) + + except Exception as e: + console.print(f"[red]Error: {e}[/red]") + raise typer.Exit(1) + + +if __name__ == "__main__": + app() diff --git a/skills/pinecone-cli/SKILL.md b/skills/pinecone-cli/SKILL.md new file mode 100644 index 0000000..161aaa1 --- /dev/null +++ b/skills/pinecone-cli/SKILL.md @@ -0,0 +1,156 @@ +--- +name: pinecone-cli +description: Guide for using the Pinecone CLI (pc) to manage Pinecone resources from the terminal. The CLI supports ALL index types (standard, integrated, sparse) and all vector operations — unlike the MCP which only supports integrated indexes. Use for batch operations, vector management, backups, namespaces, CI/CD automation, and full control over Pinecone resources. +argument-hint: install | auth | index [op] | vector [op] | backup | namespace +--- + +# Pinecone CLI (`pc`) + +Manage Pinecone from the terminal. The CLI is especially valuable for vector operations across **all index types** — something the MCP currently can't do. + +## CLI vs MCP + +| | CLI | MCP | +|---|---|---| +| Index types | All (standard, integrated, sparse) | Integrated only | +| Vector ops (upsert, query, fetch, update, delete) | ✅ | ❌ | +| Text search on integrated indexes | ✅ | ✅ | +| Backups, namespaces, org/project mgmt | ✅ | ❌ | +| CI/CD / scripting | ✅ | ❌ | + +--- + +## Setup + +### Install (macOS) +```bash +brew tap pinecone-io/tap +brew install pinecone-io/tap/pinecone +``` + +Other platforms (Linux, Windows) — download from [GitHub Releases](https://github.com/pinecone-io/cli/releases). + +### Authenticate + +```bash +# Interactive (recommended for local dev) +pc login +pc target -o "my-org" -p "my-project" + +# Service account (recommended for CI/CD) +pc auth configure --client-id "$PINECONE_CLIENT_ID" --client-secret "$PINECONE_CLIENT_SECRET" + +# API key (quick testing) +pc config set-api-key $PINECONE_API_KEY +``` + +Check status: `pc auth status` · `pc target --show` + +> **Note for agent sessions**: If you need to run `pc login` inside an agent loop, the browser auth link may not surface correctly. It's best to authenticate **before** starting an agent session. Run `pc login` in your terminal directly, then invoke the agent once you're authenticated. + +### Authenticating the CLI does not set `PINECONE_API_KEY` + +`pc login` authenticates the CLI tool itself — it does **not** set `PINECONE_API_KEY` in your environment. Python scripts, Node.js SDKs, and other tools that use the Pinecone SDK need `PINECONE_API_KEY` set separately. + +Use the CLI to create a key and export it in one step: + +```bash +KEY=$(pc api-key create --name agent-sdk-key --json | jq -r '.value') +export PINECONE_API_KEY="$KEY" +``` + +Without `jq`: run `pc api-key create --name agent-sdk-key --json` and copy the `"value"` field manually. + +--- + +## Common Commands + +| Task | Command | +|---|---| +| List indexes | `pc index list` | +| Create serverless index | `pc index create -n my-index -d 1536 -m cosine -c aws -r us-east-1` | +| Index stats | `pc index stats -n my-index` | +| Upload vectors from file | `pc index vector upsert -n my-index --file ./vectors.json` | +| Query by vector | `pc index vector query -n my-index --vector '[0.1, ...]' -k 10 --include-metadata` | +| Query by vector ID | `pc index vector query -n my-index --id "doc-123" -k 10` | +| Fetch vectors by ID | `pc index vector fetch -n my-index --ids '["vec1","vec2"]'` | +| List vector IDs | `pc index vector list -n my-index` | +| Delete vectors by filter | `pc index vector delete -n my-index --filter '{"genre":"classical"}'` | +| List namespaces | `pc index namespace list -n my-index` | +| Create backup | `pc backup create -i my-index -n "my-backup"` | +| JSON output (for scripting) | Add `-j` to any command | + +--- + +## Interesting Things You Can Do + +### Query with custom vectors (not just text) +Unlike the MCP, the CLI lets you query any index with raw vector values — useful when you generate embeddings externally (OpenAI, HuggingFace, etc.): +```bash +pc index vector query -n my-index \ + --vector '[0.1, 0.2, ..., 0.9]' \ + --filter '{"source":{"$eq":"docs"}}' \ + -k 20 --include-metadata +``` + +### Pipe embeddings directly into queries +```bash +jq -c '.embedding' doc.json | pc index vector query -n my-index --vector - -k 10 +``` + +### Bulk metadata update with preview +```bash +# Preview first +pc index vector update -n my-index \ + --filter '{"env":{"$eq":"staging"}}' \ + --metadata '{"env":"production"}' \ + --dry-run + +# Apply +pc index vector update -n my-index \ + --filter '{"env":{"$eq":"staging"}}' \ + --metadata '{"env":"production"}' +``` + +### Backup and restore +```bash +# Snapshot before a migration +pc backup create -i my-index -n "pre-migration" + +# Restore to a new index if something goes wrong +pc backup restore -i -n my-index-restored +``` + +### Automate in CI/CD +```bash +export PINECONE_CLIENT_ID="..." +export PINECONE_CLIENT_SECRET="..." +pc auth configure --client-id "$PINECONE_CLIENT_ID" --client-secret "$PINECONE_CLIENT_SECRET" +pc index vector upsert -n my-index --file ./vectors.jsonl --batch-size 1000 +``` + +### Script against JSON output +```bash +# Get all index names as a list +pc index list -j | jq -r '.[] | .name' + +# Check if an index exists before creating +if ! pc index describe -n my-index -j 2>/dev/null | jq -e '.name' > /dev/null; then + pc index create -n my-index -d 1536 -m cosine -c aws -r us-east-1 +fi +``` + +--- + +## Reference Files + +- [Full command reference](references/command-reference.md) — all commands with flags and examples +- [Troubleshooting & best practices](references/troubleshooting.md) + +## Documentation + +- [CLI Quickstart](https://docs.pinecone.io/reference/cli/quickstart) +- [Command Reference](https://docs.pinecone.io/reference/cli/command-reference) +- [Authentication](https://docs.pinecone.io/reference/cli/authentication) +- [Target Context](https://docs.pinecone.io/reference/cli/target-context) +- [GitHub Releases](https://github.com/pinecone-io/cli/releases) diff --git a/skills/pinecone-cli/references/command-reference.md b/skills/pinecone-cli/references/command-reference.md new file mode 100644 index 0000000..0091ad2 --- /dev/null +++ b/skills/pinecone-cli/references/command-reference.md @@ -0,0 +1,237 @@ +# Pinecone CLI — Full Command Reference + +## Index Management + +### Create Index +```bash +# Serverless index +pc index create -n my-index -d 1536 -m cosine -c aws -r us-east-1 + +# With integrated embedding model +pc index create -n my-index -m cosine -c aws -r us-east-1 \ + --model multilingual-e5-large \ + --field-map text=chunk_text + +# Sparse vector index +pc index create -n sparse-index -m dotproduct -c aws -r us-east-1 --vector-type sparse + +# With deletion protection +pc index create -n my-index -d 1536 -m cosine -c aws -r us-east-1 --deletion-protection enabled + +# From collection +pc index create -n my-index -d 1536 -m cosine -c aws -r us-east-1 --source-collection my-collection +``` + +### List / Describe / Stats +```bash +pc index list # Summary view +pc index list --wide # Additional columns (host, embed, tags) +pc index list -j # JSON output + +pc index describe -n my-index +pc index describe -n my-index -j + +pc index stats -n my-index +pc index stats -n my-index --filter '{"genre":{"$eq":"rock"}}' +``` + +### Configure / Delete +```bash +# Enable deletion protection +pc index configure -n my-index --deletion-protection enabled + +# Add tags +pc index configure -n my-index --tags environment=production,team=ml + +# Switch to dedicated read capacity +pc index configure -n my-index \ + --read-mode dedicated \ + --read-node-type b1 \ + --read-shards 2 \ + --read-replicas 2 + +pc index delete -n my-index +``` + +--- + +## Vector Operations + +### Upsert +```bash +# From JSON file (with "vectors" array) +pc index vector upsert -n my-index --file ./vectors.json + +# From JSONL file (one vector per line) +pc index vector upsert -n my-index --file ./vectors.jsonl + +# Inline JSON +pc index vector upsert -n my-index --file '{"vectors": [{"id": "vec1", "values": [0.1, 0.2, 0.3]}]}' + +# From stdin +cat vectors.json | pc index vector upsert -n my-index --file - + +# With namespace, custom batch size +pc index vector upsert -n my-index --namespace tenant-a --file ./vectors.json --batch-size 1000 +``` + +**File formats:** +```json +// JSON (vectors.json) +{"vectors": [{"id": "vec1", "values": [0.1, 0.2, 0.3], "metadata": {"genre": "comedy"}}]} + +// JSONL (vectors.jsonl) +{"id": "vec1", "values": [0.1, 0.2, 0.3], "metadata": {"genre": "comedy"}} +{"id": "vec2", "values": [0.4, 0.5, 0.6], "metadata": {"genre": "drama"}} +``` + +### Query +```bash +# By vector values +pc index vector query -n my-index --vector '[0.1, 0.2, 0.3]' -k 10 --include-metadata + +# By vector ID +pc index vector query -n my-index --id "doc-123" -k 10 --include-metadata + +# With metadata filter +pc index vector query -n my-index \ + --vector '[0.1, 0.2, 0.3]' \ + --filter '{"genre":{"$eq":"sci-fi"}}' \ + --include-metadata + +# Sparse vectors +pc index vector query -n my-index \ + --sparse-indices '[0, 5, 12]' \ + --sparse-values '[0.5, 0.3, 0.8]' \ + -k 15 + +# From stdin +jq -c '.embedding' doc.json | pc index vector query -n my-index --vector - -k 10 +``` + +### Fetch +```bash +pc index vector fetch -n my-index --ids '["vec1","vec2","vec3"]' +pc index vector fetch -n my-index --filter '{"genre":{"$eq":"rock"}}' +pc index vector fetch -n my-index --namespace tenant-a --ids '["doc-123"]' +pc index vector fetch -n my-index --filter '{"genre":{"$eq":"rock"}}' --limit 100 +``` + +### List / Update / Delete +```bash +# List vector IDs +pc index vector list -n my-index +pc index vector list -n my-index --namespace tenant-a --limit 50 + +# Update metadata or values +pc index vector update -n my-index --id "vec1" --metadata '{"category":"updated"}' +pc index vector update -n my-index --id "vec1" --values '[0.2, 0.3, 0.4]' + +# Bulk update with dry-run +pc index vector update -n my-index \ + --filter '{"genre":{"$eq":"sci-fi"}}' \ + --metadata '{"genre":"fantasy"}' \ + --dry-run + +# Delete by IDs or filter +pc index vector delete -n my-index --ids '["vec1","vec2"]' +pc index vector delete -n my-index --filter '{"genre":"classical"}' +pc index vector delete -n my-index --namespace old-data --all-vectors +``` + +--- + +## Namespace Management + +```bash +pc index namespace create -n my-index --name tenant-a +pc index namespace create -n my-index --name tenant-b --schema "category,brand" +pc index namespace list -n my-index +pc index namespace list -n my-index --prefix "tenant-" +pc index namespace describe -n my-index --name tenant-a +pc index namespace delete -n my-index --name tenant-a # WARNING: deletes all vectors +``` + +--- + +## Backup and Restore + +```bash +# Create / list / describe +pc backup create -i my-index -n "nightly-backup" -d "Backup before deployment" +pc backup list +pc backup list --index-name my-index +pc backup describe -i + +# Restore (creates a new index) +pc backup restore -i -n restored-index +pc backup restore -i -n restored-index --deletion-protection enabled + +# Check restore job status +pc backup restore list +pc backup restore describe -i rj-abc123 + +# Delete backup +pc backup delete -i +``` + +--- + +## Project Management + +```bash +pc project list +pc project create -n "demo-project" +pc project create -n "demo-project" --target +pc project describe -i proj-abc123 +pc project update -i proj-abc123 -n "new-name" +pc project delete -i proj-abc123 +``` + +--- + +## Organization Management + +```bash +pc organization list +pc organization describe -i org-abc123 +pc organization update -i org-abc123 -n "new-name" +pc organization delete -i org-abc123 # WARNING: highly destructive +``` + +--- + +## API Key Management + +```bash +pc api-key create -n "my-key" +pc api-key create -n "my-key" --store +pc api-key create -n "my-key" -i proj-abc123 +pc api-key list +pc api-key describe -i key-abc123 +pc api-key update -i key-abc123 --roles ProjectEditor +pc api-key delete -i key-abc123 +``` + +--- + +## Global Flags + +Available on all commands: +- `-h, --help` — Show help +- `-j, --json` — JSON output (great for scripting) +- `-q, --quiet` — Suppress output +- `--timeout` — Command timeout (default: 60s, 0 to disable) + +## Exit Codes + +- `0` — success +- `1` — error + +```bash +if pc index describe -n my-index 2>/dev/null; then + echo "Index exists" +else + pc index create -n my-index -d 1536 -m cosine -c aws -r us-east-1 +fi +``` diff --git a/skills/pinecone-cli/references/troubleshooting.md b/skills/pinecone-cli/references/troubleshooting.md new file mode 100644 index 0000000..6458bd8 --- /dev/null +++ b/skills/pinecone-cli/references/troubleshooting.md @@ -0,0 +1,136 @@ +# Pinecone CLI — Troubleshooting & Best Practices + +## Troubleshooting + +### Authentication Issues + +**"Not authenticated" or "Invalid credentials"** +```bash +pc auth status +pc logout +pc login +pc target -o "my-org" -p "my-project" +``` + +**Service account can't access resources** +```bash +pc target --show # Verify correct project is targeted +``` + +### API Key Issues + +**API key not working** +```bash +pc config get-api-key # Verify key is set +# API keys are scoped to org + project — get a new one if needed +pc api-key create -n "new-key" --store +``` + +### Target Context Issues + +**"Project not found" or "Organization not found"** +```bash +pc target --show +pc target --clear +pc target -o "my-org" -p "my-project" +``` + +### Index Issues + +**Index operations failing** +```bash +pc index describe -n my-index +# "Initializing" → wait and retry +# "Terminating" → recreate it +``` + +**Can't delete index** +```bash +# Check if deletion protection is on +pc index describe -n my-index +pc index configure -n my-index --deletion-protection disabled +pc index delete -n my-index +``` + +### Vector Upload Issues + +**Upsert fails with dimension mismatch** +```bash +pc index describe -n my-index # Check configured dimension +# Ensure all vectors have exactly that many values +``` + +**Large file upload is slow** +```bash +# Use max batch size +pc index vector upsert -n my-index --file ./large.json --batch-size 1000 + +# Or split JSONL and loop +split -l 10000 large.jsonl chunk- +for file in chunk-*; do + pc index vector upsert -n my-index --file "$file" +done +``` + +### Query Issues + +**Query returns no results** +```bash +pc index stats -n my-index # Check if data exists +pc index namespace list -n my-index # Verify namespace +# Filters use MongoDB query syntax — double-check filter format +``` + +### Backup Issues + +**Backup creation fails** +```bash +pc index describe -n my-index +# Backups are only supported for serverless indexes in "Ready" state +``` + +**Can't find backup ID** +```bash +pc backup list --index-name my-index +# Use the UUID (e.g. c84725e5-...) not the name for restore/delete +``` + +--- + +## Best Practices + +### Use the right auth method +- **Interactive dev**: `pc login` +- **CI/CD pipelines**: service accounts +- **Quick testing**: `pc api-key create -n "my-key" --store` + +### Check status before operating +```bash +pc auth status +pc target --show +pc index describe -n my-index +``` + +### Use JSON output for scripts +```bash +pc index list -j | jq -r '.[] | .name' +``` + +### Preview destructive operations +```bash +pc index vector update -n my-index \ + --filter '{"genre":{"$eq":"old"}}' \ + --metadata '{"genre":"new"}' \ + --dry-run +``` + +### Protect production indexes +```bash +pc index create -n prod-index -d 1536 -m cosine -c aws -r us-east-1 \ + --deletion-protection enabled +``` + +### Automate backups +```bash +pc backup create -i my-index -n "daily-backup-$(date +%Y%m%d)" +``` diff --git a/skills/pinecone-docs/SKILL.md b/skills/pinecone-docs/SKILL.md new file mode 100644 index 0000000..9a681c9 --- /dev/null +++ b/skills/pinecone-docs/SKILL.md @@ -0,0 +1,86 @@ +--- +name: pinecone-docs +description: Curated documentation reference for developers building with Pinecone. Contains links to official docs organized by topic and data format references. Use when writing Pinecone code, looking up API parameters, or needing the correct format for vectors or records. +--- + +# Pinecone Developer Reference + +A curated index of Pinecone documentation. Fetch the relevant page(s) for the task at hand rather than relying on training data. + +--- + +## NOTE TO AGENT +Please attempt to fetch the url listed when relevant. If you run into an error, please attempt to append ".md" to the url to retrieve the markdown version of the Docs page. + +In case you need it: A full reference to ALL relevant URLs can be found here: https://docs.pinecone.io/llms.txt + +Use this as a last resort if you cannot find the relevant page below. + +--- + +## Getting Started + +| Topic | URL | +|---|---| +| Quickstart for all languages and coding environments (Cursor, Claude Code, n8n, Python, JavaScript, Java, Go, C#) | https://docs.pinecone.io/guides/get-started/quickstart | +| Pinecone concepts — namespaces, terminology, and key database concepts | https://docs.pinecone.io/guides/get-started/concepts | +| Data modeling for text and vectors | https://docs.pinecone.io/guides/index-data/data-modeling | +| Architecture of Pinecone | https://docs.pinecone.io/guides/get-started/database-architecture | +| Pinecone Assistant overview | https://docs.pinecone.io/guides/assistant/overview | + +--- + +## Indexes + +| Topic | URL | +|---|---| +| Create an index | https://docs.pinecone.io/guides/index-data/create-an-index | +| Index types and conceptual overview | https://docs.pinecone.io/guides/index-data/indexing-overview | +| Integrated inference (built-in embedding models) | https://docs.pinecone.io/guides/index-data/indexing-overview#integrated-embedding | +| Dedicated read nodes — predictable low-latency performance at high query volumes | https://docs.pinecone.io/guides/index-data/dedicated-read-nodes | + +--- + +## Upsert & Data + +| Topic | URL | +|---|---| +| Upsert vectors and text | https://docs.pinecone.io/guides/index-data/upsert-data | +| Multitenancy with namespaces | https://docs.pinecone.io/guides/index-data/implement-multitenancy | + +--- + +## Search + +| Topic | URL | +|---|---| +| Semantic search | https://docs.pinecone.io/guides/search/semantic-search | +| Hybrid search | https://docs.pinecone.io/guides/search/hybrid-search | +| Lexical search | https://docs.pinecone.io/guides/search/lexical-search | +| Full-text search (preview) — document-schema FTS indexes with `text` / `query_string` / dense / sparse scoring | https://docs.pinecone.io/guides/search/full-text-search | +| Metadata filtering — narrow results and speed up searches | https://docs.pinecone.io/guides/search/filter-by-metadata | + +--- + +## API & SDK Reference + +| Topic | URL | +|---|---| +| Python SDK reference | https://docs.pinecone.io/reference/sdks/python/overview | +| Example Colab notebooks | https://docs.pinecone.io/examples/notebooks | + +--- + +## Production + +| Topic | URL | +|---|---| +| Production checklist — preparing your index for production | https://docs.pinecone.io/guides/production/production-checklist | +| Common errors and what they mean | https://docs.pinecone.io/guides/production/error-handling | +| Targeting indexes correctly — don't use index names in prod | https://docs.pinecone.io/guides/manage-data/target-an-index#target-by-index-host-recommended | + +--- + +## Data Formats + +See [references/data-formats.md](references/data-formats.md) for vector and record schemas. diff --git a/skills/pinecone-docs/references/data-formats.md b/skills/pinecone-docs/references/data-formats.md new file mode 100644 index 0000000..2a55f45 --- /dev/null +++ b/skills/pinecone-docs/references/data-formats.md @@ -0,0 +1,81 @@ +# Data Formats + +## Integrated Index Records + +Used with `upsert_records()` (Python SDK) or `upsert-records` (MCP). Records are automatically embedded using the index's configured model. + +**JSON** +```json +[ + { + "_id": "rec1", + "chunk_text": "Your text content here.", + "category": "example" + }, + { + "_id": "rec2", + "chunk_text": "Another piece of text.", + "category": "example" + } +] +``` + +- `_id` — unique record identifier (required) +- The text field name must match the index's `fieldMap` (e.g. `chunk_text` if `fieldMap: {text: "chunk_text"}`) +- All other fields are stored as metadata and can be used for filtering +- Do **not** nest extra fields under a `metadata` key — put them directly on the record + +--- + +## Standard Index Vectors + +Used with `upsert()` (Python SDK) or `pc index vector upsert` (CLI). + +**JSON (with `vectors` array)** +```json +{ + "vectors": [ + { + "id": "vec1", + "values": [0.1, 0.2, 0.3], + "metadata": { "genre": "comedy", "year": 2021 } + }, + { + "id": "vec2", + "values": [0.4, 0.5, 0.6], + "metadata": { "genre": "drama", "year": 2019 } + } + ] +} +``` + +**JSONL (one vector per line)** +```jsonl +{"id": "vec1", "values": [0.1, 0.2, 0.3], "metadata": {"genre": "comedy"}} +{"id": "vec2", "values": [0.4, 0.5, 0.6], "metadata": {"genre": "drama"}} +``` + +- `id` — unique vector identifier (required) +- `values` — dense vector as float array, length must match index dimension (required) +- `metadata` — arbitrary key-value pairs for filtering (optional) + +--- + +## Sparse Vectors + +Used for keyword or hybrid search with sparse indexes. + +```json +{ + "id": "vec1", + "values": [0.1, 0.2, 0.3], + "sparse_values": { + "indices": [10, 45, 316], + "values": [0.5, 0.3, 0.8] + }, + "metadata": { "genre": "comedy" } +} +``` + +- `sparse_values.indices` — non-zero dimension indices +- `sparse_values.values` — corresponding float values, same length as `indices` diff --git a/skills/pinecone-full-text-search/SKILL.md b/skills/pinecone-full-text-search/SKILL.md new file mode 100644 index 0000000..2d4b942 --- /dev/null +++ b/skills/pinecone-full-text-search/SKILL.md @@ -0,0 +1,354 @@ +--- +name: pinecone-full-text-search +description: Create, ingest into, and query a Pinecone full-text-search (FTS) index using the preview API (2026-01.alpha, public preview). Use when the user or agent asks to build a text search index on Pinecone, add dense or sparse vector fields, ingest documents, construct score_by clauses (text / query_string / dense_vector / sparse_vector), or compose with text-match filters ($match_phrase / $match_all / $match_any). Ships `scripts/ingest.py` for safe bulk ingestion (batch_upsert + error inspection + readiness polling); query construction is documented inline in this skill — write `documents.search(...)` calls directly, validated against `pc.preview.indexes.describe(...)` output. +--- + +# pinecone-full-text-search + +> **Requires `pinecone` Python SDK ≥ 9.0** (`pip install pinecone>=9.0`). The FTS document-schema API lives under `pinecone.preview` and is incomplete or absent in earlier SDK builds. The packaged helper scripts pin `pinecone==9.0.0` via PEP 723 inline metadata; if you're writing your own code against this skill, pin v9 explicitly. The wire API version is `2026-01.alpha`. + +> **Authoritative reference (last resort).** If you hit a question this skill and its `references/*.md` files don't answer, the official Pinecone FTS docs are at . Prefer this skill's content for anything covered here — the docs may describe surfaces (e.g. classic vector API) that don't apply to the document-schema FTS path. Consult the link only when you're genuinely stuck. + +> **Tell the user up front:** "This skill ships a helper at `scripts/ingest.py` that handles bulk ingestion safely (batched upsert, error inspection, readiness polling). When we get to the ingest step, I'll use it." Surface this at the start of the conversation so the user knows the helper exists. Query construction is hand-written `documents.search(...)` per the **Querying** section below — there is no query helper. + +A workflow skill for building a Pinecone full-text-search index with the preview API (`pinecone.preview`, API version `2026-01.alpha`, public preview as of April 2026). Covers schema design (text, dense vector, sparse vector, filterable metadata), ingestion (including async indexing and polling), and query construction (`text` / `query_string` / `dense_vector` / `sparse_vector` scoring; `$match_phrase` / `$match_all` / `$match_any` text-match filters; `$eq` / `$in` / `$gte` / `$exists` / `$and` / `$or` / `$not` metadata filters). + +## Scope — this skill is for the document-schema FTS API only + +This skill covers `pc.preview.indexes.create(..., schema=...)`, `pc.preview.index(name)`, `idx.documents.upsert(...)` / `idx.documents.batch_upsert(...)` / `idx.documents.search(...)`. If you find yourself reaching for any of the following, **stop** — those are different Pinecone APIs and this skill's guidance and helpers won't apply: + +- **Classic vector / records API**: `pc.Index(name)`, `index.upsert(vectors=[...])` / `index.upsert_records(...)`, `index.query(vector=..., sparse_vector=...)`, `index.search_records(...)`, `pc.create_index(...)` with `ServerlessSpec`, the legacy `pinecone_text.sparse.BM25Encoder` for sparse-dense hybrid. For indexes WITHOUT a schema (raw vectors). +- **Integrated-embedding indexes**: `pc.create_index_for_model(...)` with `embed={...}`. Pinecone vectorizes text server-side. Different upsert/search shapes. Cannot be combined with `full_text_search` fields in the same index. + +If the user already has a non-document-schema index, they can stand up a separate document-schema index alongside it — the two are independent — but you can't add FTS fields to a classic index after the fact. + +## Querying — construct `documents.search(...)` calls + +For any task that asks you to query an FTS index, you write a `documents.search(...)` call directly. The schema is authoritative — describe the index live before constructing the call so you know which fields are FTS-enabled, which are filterable, and which are vectors. + +**Workflow:** + +1. **Discover the schema.** Call `pc.preview.indexes.describe()` and read the `schema.fields` dict. Each field's class indicates its type (`PreviewStringField`, `PreviewIntegerField`, `PreviewDenseVectorField`, etc.); attributes tell you whether it's FTS-enabled (`full_text_search`), filterable, or carries a `dimension`. Skip this step only if you've already seen the schema in this conversation. +2. **Construct the call** matching the rules below — one scoring type per request, hard requirements in `filter`, ranking signals in `score_by`, `include_fields` explicit on every call. +3. **Execute** with `idx = pc.preview.index(name=); resp = idx.documents.search(...)` and read `resp.matches`. + +**Canonical shapes:** + +```python +# Pure BM25 keyword search +resp = idx.documents.search( + namespace="__default__", + top_k=10, + score_by=[{"type": "text", "field": "body", "query": "machine learning"}], + filter={"year": {"$gt": 2024}, "category": {"$eq": "ai"}}, # optional + include_fields=["*"], # always pass explicitly +) + +# Hybrid: dense ranking with a lexical filter (one type in score_by + filter narrows) +resp = idx.documents.search( + namespace="__default__", + top_k=10, + score_by=[{"type": "dense_vector", "field": "embedding", "values": query_embedding}], + filter={"body": {"$match_all": "TensorFlow"}, "year": {"$gt": 2024}}, + include_fields=["*"], +) +``` + +**Key rules** (the server enforces these; following them locally keeps the agent loop tight): + +- `score_by` is a list of clauses, but **exactly one scoring type per request** (server rejects mixed types). Multi-field BM25 is the one exception: multiple `text` clauses, or one `query_string` with `fields: [...]`. To combine BM25 + dense signals, restrict the dense search with a text-match filter (`$match_all` / `$match_phrase` / `$match_any`); do NOT mix scoring types in `score_by`. +- `filter` keys are field names (must exist in schema and be filterable) OR logical operators (`$and`, `$or`, `$not`). Field values are operator dicts (`{"$gt": 5}`, NOT bare values). +- `include_fields` is required on every call. Pass `["*"]` for all stored fields, `[]` for ids+score only, or a list of names. Some SDK builds 400/422 if it's omitted. + +**Clause shapes** (for `score_by`): + +| `type` | Required keys | When to pick this | +|---|---|---| +| `text` | `field` (string FTS), `query` | Open-ended keyword search; BM25 ranking on one field | +| `query_string` | `query` (Lucene), `fields` optional | Lucene boost (`^N`), proximity (`~N`), cross-field boolean, phrase prefix | +| `dense_vector` | `field` (dense_vector), `values` (list of floats) | Semantic / mood / topic ranking | +| `sparse_vector` | `field` (sparse_vector), `sparse_values` ({indices, values}) | Custom sparse-encoder ranking | + +`text` / `dense_vector` / `sparse_vector` use singular `field`. Only `query_string` accepts a `fields` array (and also accepts singular `field` as an alias). `sparse_vector` uses `sparse_values` (NOT `values`) — distinct from dense. + +**Filter operators by field type:** + +| Field type | Legal operators | +|---|---| +| `string` with FTS | `$match_phrase`, `$match_all`, `$match_any` | +| `string` filterable | `$eq`, `$ne`, `$in`, `$nin`, `$exists` | +| `string_list` filterable | `$in`, `$nin`, `$exists` | +| `float` filterable | `$eq`, `$ne`, `$gt`, `$gte`, `$lt`, `$lte`, `$exists` | +| `boolean` filterable | `$eq`, `$exists` | +| logical wrappers | `$and: [filters]`, `$or: [filters]`, `$not: filter` | + +**Match shape on response:** + +```python +for m in resp.matches: + m._id # document id + m._score # match score (NOT `score`); some older SDK builds may also surface `score` + m.to_dict() # full doc payload (when include_fields includes the field) +``` + +For deeper coverage — multi-field BM25, Lucene patterns, hybrid composition, RRF merges, common error symptoms — see `references/querying.md`. For schema field types and what they enable on the query side, see `references/schema-design.md`. + +## Ingesting — use the packaged helper + +For **any task that asks you to bulk-ingest a JSONL file into an existing FTS index**, the canonical path is to invoke the bundled helper, NOT to hand-write a Python script. **Do not read the script's source** — everything you need is in this section. + +The script does three things bare-LLM ingest code reliably skips, each of which corresponds to a silent production failure: + +1. **Bulk-upserts in batches.** No per-doc `upsert` loops. +2. **Inspects every batch result.** `batch_upsert` returns 202 even when individual documents fail; the failures live in `result.errors` / `result.has_errors`. Without inspection, "100 docs ingested" silently becomes "73 docs ingested + 27 lost." +3. **Polls until searchable.** After upsert, Pinecone is still building the inverted index. A `documents.search` call during that window returns empty. Without the poll, the user debugs their *query* code for an hour without finding the indexing race. + +You provide a prepared, schema-conformant JSONL file and the index name; the script does the rest. Schema validation is upstream concerns (your prep pipeline, or `prepare_documents.py` when it lands) — `ingest.py` trusts what you hand it. + +**Invocation:** + +```bash +uv run --script .claude/skills/pinecone-fts-index/scripts/ingest.py \ + --data processed.jsonl \ + --index \ + --sentinel-field +``` + +**Flags:** + +| Flag | Short | Required | Purpose | +|---|---|---|---| +| `--data` | `-d` | yes | Path to JSONL file with prepared documents (one per line) | +| `--index` | `-i` | yes | Pinecone index name (must already exist) | +| `--sentinel-field` | `-f` | yes | An FTS-enabled field on the index, used for the readiness-poll query. Pick the longest free-text field on your schema. | +| `--namespace` | `-n` | no | Default `__default__` | +| `--batch-size` | `-b` | no | Default 100. **Reduce for large dense vectors.** A 50-doc batch with 3072-dim float vectors lands ~5-10 MB and can be rejected; drop to `--batch-size 50` (or lower) at high dimensions. | +| `--poll-deadline` | — | no | Default 300 (seconds). Time to wait for documents to become searchable before giving up. | +| `--sentinel` | `-s` | no | Token used for the readiness-poll query. Default: first whitespace-separated token of `doc[0][sentinel-field]`. | + +**What the script prints:** + +``` +Loading processed.jsonl ... +Loaded 5000 document(s). +Sentinel: body='The' + +Upserting in batches of 100 ... + batch @ 0: 100 docs in 0.42s (total: 100/5000) + batch @ 100: 100 docs in 0.39s (total: 200/5000) + ... + +Upsert complete: 5000 doc(s) in 21.4s. + +Polling for searchability (deadline 300s) ... +Searchable after 12.3s (3 probe(s)). + +Done — total 33.7s. +``` + +If a batch fails, the script prints every error message and exits non-zero. If the poll deadline expires, the script prints a hint about why (sentinel field isn't FTS-enabled, deadline too tight, docs structurally upserted but rejected by the inverted-index builder) and exits non-zero. **Don't suppress these errors** — they're surfacing real problems with the data or the index. + +**When you should NOT use the script:** + +- The user is doing per-doc patch updates (single-doc `documents.upsert` calls with selective fields). The script is for bulk loads, not per-record operations. +- The user is ingesting from a non-JSONL source (CSV, Parquet, Postgres dump). Convert to JSONL first; the script doesn't parse other formats. +- The user explicitly asks you to write the ingestion code from scratch (teaching context). Honor the request and follow the canonical pattern: `documents.batch_upsert` + `result.has_errors` inspection + `documents.search` polling with sentinel and deadline. + +The script lives at `.claude/skills/pinecone-fts-index/scripts/ingest.py`. PEP 723 inline-metadata script — `uv run --script` installs `typer` and `pinecone` automatically on first invocation. No setup needed. + +## Use cases + +Three concrete shapes to model your task on. Match the user's request to the closest one and follow its steps; improvise if the task is genuinely a hybrid. + +### UC-1: Index a new corpus end-to-end + +**Trigger.** "Index this CSV / JSONL / folder for search," "build a search backend over [my articles / products / tickets / transcripts]," "make my [dataset] searchable." + +**For unprocessed / messy data, load the onboarding walkthrough first.** If the user is showing up with raw data (unclear field types, possibly long text fields exceeding FTS limits, comma-separated tag strings, dates as strings, possibly duplicate IDs, etc.) and they haven't given you an explicit schema, **read `references/onboarding-walkthrough.md` and follow it stage-by-stage.** It's a conversational guide — meet the data, surface the processing decisions to the user, propose a schema, confirm before creating, then process+ingest+verify together. The walkthrough exists because schemas are immutable and "onboarding a new corpus" is a high-stakes flow that benefits from explicit user buy-in at each decision point. + +If the user already gave you a clean JSONL + a schema spec, follow the abbreviated steps below. + +**Steps (when data is already prepared and the schema is decided):** +1. Inspect the corpus shape — text fields, structured metadata, do you also need a vector? Match it to one of the canonical shapes in `references/schema-design.md` (articles, products, tickets, image library, code). +2. Pick analyzer settings on each text field — `language`, `stemming`, `stop_words`. Stemming on for long prose, off for proper nouns / identifiers. +3. Assemble the schema with `SchemaBuilder` and **confirm it with the user before calling `indexes.create`** — schemas are immutable in `2026-01.alpha`, so a wrong call costs a re-ingest. +4. Create the index, poll `describe()` until `status.ready: true`. +5. **Run `scripts/ingest.py --data --index --sentinel-field `** — see the **Ingesting — use the packaged helper** section above. The script handles `batch_upsert` + per-batch error inspection + post-upsert readiness polling in one invocation. Don't hand-write the loop unless the user explicitly asks you to. +6. (The script polls automatically — by the time it exits cleanly, the index is searchable. If you skip the script and roll your own, you must poll `documents.search` with a sentinel query and a deadline; `batch_upsert` returning ≠ searchable.) +7. Validate with one or two probe queries against fields you know contain the sentinel content. + +**Result.** A working `documents.search` call against the user's data, returning ranked matches. + +### UC-2: Add a dense (or sparse) signal to a text-only corpus + +**Trigger.** "Add semantic search," "add embeddings," "make this hybrid," or any prompt that describes a query pattern text alone can't serve (visual similarity, mood, cross-modal "looks like"). + +**Steps.** +1. Confirm the new signal represents a **modality or signal text can't express** — image / audio / external score, *or* a different corpus than the existing FTS field. Re-encoding the same text into a dense field is an anti-pattern (`references/schema-design.md` → "When to add a dense field at all"). +2. Because schemas are immutable, **plan a new index, not a migration**. Get user confirmation before recreating. +3. Pick an embedding provider and pin its output dimension at schema time. Beware payload-size pitfalls at native dimensions — Gemini-3072 etc. need truncation (`references/ingestion.md` → "Dense-vector payload size"). +4. Schema → create → wait Ready → ingest with embeddings inline or pre-cached. +5. Validate with a **hybrid query**: `dense_vector` score_by + text-match filter (`$match_phrase` / `$match_all`). That's the supported single-call cross-modal shape. + +**Result.** One index, two retrieval shapes — pure text *and* dense+filter hybrid — both runnable without further setup. + +### UC-3: Build a `documents.search` call from a natural-language user prompt (agent mode) + +**Trigger.** Agent receives a user prompt like "find articles about machine learning that mention TensorFlow and were published after 2024" or "documents about climate policy ranked by similarity to this paragraph." The index already exists. + +**Steps.** +1. **(Optional) Discover the schema** by calling `pc.preview.indexes.describe()` and reading `schema.fields`. Skip if you already know the field types from earlier in the conversation. +2. **Decompose the user's prompt** into `score_by` / `filter` shapes using the agent-mode decomposition table below. (Hard requirements → `filter`. Ranking signals → `score_by`. Always include `include_fields` explicitly.) +3. **Construct the `documents.search(...)` call** following the rules in the Querying section above — one scoring type per request, operator/field-type matching, `include_fields` always set. +4. **Execute** the call. The response carries `resp.matches`; iterate to get `m._id`, `m._score`, and field values via `m.to_dict()`. Use the matches in whatever shape the user asked for. +5. If results come back empty or wrong, walk the failure tree in `Common gotchas`. + +**Result.** Live search results matching the user's intent. + +**The four common UC-3 mistakes** to actively avoid: +- Mixing scoring types in `score_by` (server rejects). Put hard requirements in `filter`; rank by one signal in `score_by`. +- Putting hard requirements in `score_by` as BM25 terms instead of in `filter` as `$match_all` / `$match_phrase` (returns ranked results that don't *guarantee* the term is present). +- Operator/field-type mismatches (e.g. `$match_all` on a float field, `$gt` on a string field). Consult the operator table in the Querying section. +- Omitting `include_fields` (some SDK builds 400/422). Always pass it explicitly. + +## Agent-mode query decomposition + +Map user prompt cues to API shapes. Read top-down — identify the cue, copy the corresponding shape. + +| User prompt cue | API shape | +|---|---| +| Open-ended keywords ("articles about machine learning", search-bar query) | `score_by=[{"type": "text", "field": "", "query": ""}]` — BM25 token-OR | +| Exact phrase, drives ranking ("rank by 'beautifully written'") | `score_by=[{"type": "query_string", "query": ':("phrase here")'}]` | +| Exact phrase, hard requirement ("must contain 'machine learning'") | `filter={"": {"$match_phrase": "machine learning"}}` | +| Required tokens, any order ("must mention TensorFlow", "must be about Illinois") | `filter={"": {"$match_all": "tokens space-separated"}}` — preferred over `query_string` `+token` because it's a true hard filter, doesn't contribute to score | +| At least one of these tokens ("contains AI or ML or robotics") | `filter={"": {"$match_any": "AI ML robotics"}}` | +| Excluded tokens ("not about deprecated", "no opinion pieces") | `filter={"$not": {"": {"$match_any": "deprecated opinion"}}}` — or `-token` inside `query_string` | +| Boolean / boost / slop / phrase-prefix ("weight 'eagle' 3x", "within N words") | `score_by=[{"type": "query_string", "query": ''}]` — only Lucene supports these | +| Cross-field boolean ("title or body contains X") | `score_by=[{"type": "query_string", "query": 'title:(X) OR body:(X)'}]` | +| Numeric / date / range / boolean metadata ("after 2024", "rating > 4", "in stock") | `filter={"": {"$gt": ..., "$gte": ..., "$eq": ..., "$exists": true}}` | +| Category / tag / list membership ("category = fiction", "tagged X") | `filter={"": {"$in": [...]}}` (works on `string` and `string_list` filterable fields) | +| Semantic similarity / mood / topic ("articles about ML", "documents that feel sombre") | `score_by=[{"type": "dense_vector", "field": "", "values": embed()}]` — requires a `dense_vector` field | +| Visual appearance / cross-modal text query against an image corpus | Same dense_vector shape, with the embedding model that produced the stored image vectors. Multimodal embedders (Gemini-2 etc.) map a text query into the image space. | +| Hybrid: lexical requirement + semantic ranking ("articles about ML that mention TensorFlow") | Lexical → `filter` (`$match_all` / `$match_phrase`); semantic → `score_by` (`dense_vector`). Single call. | + +**Two structural rules the agent must enforce, no exceptions:** + +- **One scoring type per request.** `score_by` accepts `text` / `query_string` / `dense_vector` / `sparse_vector`, but a request ranks by *one*. Don't mix dense + text in `score_by` — the server rejects it. Multi-field BM25 is the only "list" pattern that's allowed (multiple `text` clauses, or one cross-field `query_string`). +- **Hybrid = filter + score_by, not two `score_by` clauses.** When a prompt has both a lexical requirement and a semantic ranking signal, lexical goes in `filter` (via `$match_*` operators) and semantic goes in `score_by`. If both signals genuinely need to drive *ranking*, run two searches and merge IDs client-side. + +## Workflow at a glance + +Three phases. Each has its own reference file — consult it before writing code for that phase. + +1. **Design the schema.** Decide which string fields are full-text-searchable, which are filterable metadata, whether you need a `dense_vector` field (and whether it earns its place), whether you also need a `sparse_vector` field, and which numeric / boolean / array filters to declare. Schemas are **fixed at index creation** in `2026-01.alpha` — plan carefully. → `references/schema-design.md` +2. **Ingest documents.** For bulk loads from a prepared JSONL, run the bundled `scripts/ingest.py` helper (it does `batch_upsert` + error inspection + readiness polling correctly by construction — see the **Ingesting — use the packaged helper** section above). For per-doc patch updates, hand-call `documents.upsert`. Either way, documents are indexed asynchronously after the HTTP call returns; `batch_upsert` returning 202 ≠ searchable. → `references/ingestion.md` for the canonical pattern in detail. +3. **Query the index.** A single search request ranks by **one** scoring type — pass exactly one of `text`, `query_string`, `dense_vector`, or `sparse_vector` in `score_by` (multi-field BM25 is supported via multiple `text` clauses or a cross-field `query_string`). Layer `filter={...}` for text-match (`$match_phrase` / `$match_all` / `$match_any`) and metadata filters (`$eq` / `$in` / `$gte` / `$exists` / `$and` / `$or` / `$not`). Control the response payload with `include_fields`. → `references/querying.md` + +## Quick template + +End-to-end skeleton for a minimal text + filterable-metadata index. Copy it and edit every spot marked `# TODO:`. The template deliberately omits external embedding calls so it stays generic; see `references/ingestion.md` for dense / sparse field patterns and embedding-provider integration, and `references/querying.md` for the four scoring shapes plus text-match and metadata filters. + +```python +import time +from pinecone import Pinecone +from pinecone.preview import SchemaBuilder + +INDEX_NAME = "my-fts-index" # TODO: name your index (lowercase alphanumeric + hyphens, ≤45 chars) +NAMESPACE = "__default__" # TODO: pick a namespace; auto-created on first upsert + +pc = Pinecone() # reads PINECONE_API_KEY +# TODO: preprod backends require an x-environment header on the client: +# pc = Pinecone(additional_headers={"x-environment": "preprod-aws-0"}) + +# 1. Schema — one FTS string field, one filterable string, one filterable float. +# Field names must NOT start with `_` (reserved for `_id` / `_score`) or `$` +# (reserved for filter operators), and are limited to 64 bytes. +schema = ( + SchemaBuilder() + .add_string_field("body", full_text_search={"language": "en"}) # TODO: rename for your content + .add_string_field("category", filterable=True) # TODO: any exact-match metadata + .add_integer_field("year", filterable=True) # TODO: any numeric filter — emits `"type": "float"` on the wire + .build() +) + +# 2. Create the index. read_capacity defaults to {"mode": "OnDemand"}; pass +# {"mode": "Dedicated", ...} only if you specifically want provisioned reads. +if not pc.preview.indexes.exists(INDEX_NAME): + pc.preview.indexes.create(name=INDEX_NAME, schema=schema) + +# 3. Wait for the index itself to become Ready. +while not pc.preview.indexes.describe(INDEX_NAME).status.ready: + time.sleep(5) + +idx = pc.preview.index(name=INDEX_NAME) + +# 4. Upsert a single document. `_id` is required, every other field is optional. +# upsert REPLACES the document on conflict — there is no per-field merge in 2026-01.alpha. +idx.documents.upsert( + namespace=NAMESPACE, + documents=[{ + "_id": "doc-1", + "body": "Full-text search is great for keyword queries.", + "category": "intro", + "year": 2025.0, + }], +) + +# 5. Poll until the FTS side is searchable (upsert returns BEFORE docs are indexed). +deadline = time.time() + 300 +while time.time() < deadline: + resp = idx.documents.search( + namespace=NAMESPACE, top_k=1, + score_by=[{"type": "text", "field": "body", "query": "search"}], # TODO: sentinel query likely to hit + include_fields=[], # required on every search; [] = lightest payload (ids + _score only) + ) + if resp.matches: + break + time.sleep(5) + +# 6. Search — text scoring composed with metadata filter. +resp = idx.documents.search( + namespace=NAMESPACE, + top_k=5, + score_by=[{"type": "text", "field": "body", "query": "keyword queries"}], + filter={"year": {"$gte": 2024}}, # TODO: adjust filter or drop it + include_fields=["*"], # "*" = all stored fields; [] = `_id` + `_score` only +) +for m in resp.matches: + print(m._id, getattr(m, "_score", getattr(m, "score", None)), m.to_dict()) +``` + +## Common gotchas + +- **One scoring type per search request.** `score_by` accepts `text`, `query_string`, `dense_vector`, or `sparse_vector` — but a request ranks by *one* type. Multi-field BM25 is fine (pass several `text` clauses, or a single cross-field `query_string`). To combine BM25 ranking with a `dense_vector` (or `sparse_vector`) signal, restrict the dense search with a text-match `filter` operator (`$match_phrase` / `$match_all` / `$match_any`) on the lexical field, *not* by mixing types in `score_by`. The "blend a dense vector and a text clause in `score_by`" pattern is rejected by the server. +- **Text-match filter operators are the cross-modal hinge.** `$match_phrase` (exact phrase), `$match_all` (every token, any order), `$match_any` (at least one token) are filter-side operators on `full_text_search` fields. Each takes a single string (max 128 tokens). They reuse the field's tokenizer / stemmer, compose under `$and` / `$or` / `$not`, and are the supported way to compose lexical pre-filtering with dense or sparse ranking. **Phrase slop (`"…"~N`), term boost (`^N`), and phrase prefix (`"… word"*`) are scoring-only — they live in `query_string`, not in `filter`.** +- **Preprod backends need `additional_headers={"x-environment": "..."}` on the `Pinecone()` client.** Missing the header lands you on prod and you'll see "index not found" / empty-result symptoms that look like code bugs but aren't. +- **`include_fields` is required on every `documents.search(...)` call.** When omitted, defaults to `[]` (`_id` + `_score` only). Pass `["*"]` for all stored fields or a list of names to project. Omitting it on some SDK builds yields `400` / `422` instead of the documented default; always pass it explicitly to avoid surprises. +- **Match score is `_score`; doc id is `_id`.** Public-preview docs return the system match score on the `_score` field so a user metadata field literally named `score` can coexist. Always prefer `_score` on read; some older SDK builds may still surface plain `score`, so for defensive code use `getattr(m, "_score", getattr(m, "score", None))`. +- **Reserved field names: leading `_` and `$`, max 64 bytes.** `_` is for system fields (`_id`, `_score`); `$` is for filter operators. Schema validation rejects names that violate either rule. Length cap is bytes, not characters — be careful with non-ASCII names. +- **Vector-field cardinality: at most one `dense_vector` and at most one `sparse_vector` per index** in `2026-01.alpha`. Multiple text fields are fine. +- **`batch_upsert` failures are silent by default.** The return value carries `has_errors`, `failed_batch_count`, and a list of `BatchError` objects with `error_message`. If you don't inspect them, you'll see "Uploaded 0 / N" and an indefinite "not yet indexed" poll — with the real cause (payload-too-large, schema mismatch, reserved field name) hidden. Always print `result.errors[*].error_message` before downstream steps. +- **Dense-vector payload size matters at batch time.** A 50-doc batch with 3072-dim float vectors lands around 5–10 MB and can be rejected by the preview backend. If every batch fails, try reducing the embedding dimension via your provider's truncation knob (e.g. Gemini's `output_dimensionality=768`) before debugging schema. +- **Async indexing: `batch_upsert` returning ≠ searchable.** The server builds inverted indexes in the background after the HTTP call returns. If you query immediately you'll see empty result sets. Always poll `documents.search` with a sentinel query and a deadline (pattern in `references/ingestion.md`). +- **String FTS field shape is `full_text_search={...}` (dict).** Pass `{}` to enable with all server defaults. **User-settable sub-fields:** `language`, `stemming`, `stop_words`. **Server-applied** (visible in `describe()` responses but NOT settable at index creation): `lowercase` (default `true`) and `max_token_length` (default `40`). Stemming is opt-in (default `false`); `stop_words` is opt-in (default `false`, opposite of pre-public-preview docs). The earlier SDK shape `full_text_searchable=True, language="en"` is legacy and should be avoided. +- **Schemas are fixed at index creation in `2026-01.alpha`.** Adding, removing, or retyping fields after creation is not supported. Changing dimension or metric on an existing vector field requires a new index. Plan the schema once. +- **No partial / per-field updates.** `documents.upsert` always replaces the entire document for a given `_id`. To update one field, fetch the doc, modify in client code, and upsert the full doc back under the same `_id`. +- **Document operations: search supports `filter`, fetch and delete do not.** Fetch is **ID-only** (`POST /documents/fetch` with `ids: [...]`); delete accepts only `ids` or `delete_all: true`. To act on a metadata expression, search first to collect IDs, then fetch or delete those IDs. +- **Namespaces auto-create on first upsert.** Pass any namespace string to `documents.upsert` / `batch_upsert` and the namespace is created on the fly; documents from different namespaces are fully isolated. Use `"__default__"` if you don't need partitioning. **Caveat:** the namespace management endpoints (`POST /namespaces`, `GET /namespaces`, `DELETE /namespaces/{namespace}`) and `describe_index_stats` are NOT yet supported on indexes with document schemas — you can write to a namespace, you just can't list / delete them via the API yet. +- **Document and request size limits** (preview): per-document max **2 MB**; per-request max **2 MB and 1000 documents**; per FTS-enabled `string` field max **100 KB and 10,000 tokens** (tokens > 256 bytes are truncated by the analyzer); per-document filterable metadata (everything *not* in an FTS field) max **40 KB**. A schema can declare up to **100 FTS string fields**. For long-prose corpora, chunk before ingest — see `references/ingestion.md`. +- **`score_by` clause shape — singular `field` is canonical for `text`/`dense_vector`/`sparse_vector`; only `query_string` takes a `fields` array.** + - `text`: `{"type":"text", "field":"", "query":""}`. + - `query_string`: `{"type":"query_string", "query":"", "fields":["",""]}` (the optional `fields` array; `query_string` also accepts a bare `"fields":"body"` string and the legacy `"field":"body"` as an alias). + - `dense_vector`: `{"type":"dense_vector", "field":"", "values":[/*floats*/]}`. + - `sparse_vector`: `{"type":"sparse_vector", "field":"", "sparse_values":{"indices":[...],"values":[...]}}` — note `sparse_values` (NOT `values`) for sparse clauses. +- **Single-term prefix wildcards aren't supported.** `auto*` doesn't work in `query_string`; use phrase prefix (`"machine lea"*` — phrase must contain at least two terms, last term is matched as prefix). +- **Indexes can't be created in CMEK-enabled projects, no backup/restore, no fuzzy or regex search, no S3 bulk import** for document-shaped indexes in `2026-01.alpha`. If any of these are hard requirements, the public-preview FTS surface isn't yet ready. + +## Extension points + +Currently shipped under `scripts/`: + +- `scripts/ingest.py` — bulk-ingest a prepared JSONL into an existing FTS index. Handles `batch_upsert` in safe-sized chunks, inspects every batch's `result.errors` and aborts loudly on failure, then polls `documents.search` with a sentinel + deadline until docs are searchable. Schema-agnostic: takes only `--data`, `--index`, `--sentinel-field`. Usage in **Ingesting — use the packaged helper** section above. + +Query construction does NOT have a packaged helper — write `documents.search(...)` calls directly per the **Querying** section above. + diff --git a/skills/pinecone-full-text-search/references/ingestion.md b/skills/pinecone-full-text-search/references/ingestion.md new file mode 100644 index 0000000..56e3502 --- /dev/null +++ b/skills/pinecone-full-text-search/references/ingestion.md @@ -0,0 +1,263 @@ +# Ingestion + +Writing documents into a Pinecone preview document index uses two methods. Pick based on volume, then handle the *async indexing* gotcha on the other side. + +## `documents.upsert` — small writes / patches + +```python +idx = pc.preview.index(name=INDEX_NAME) + +upsert_resp = idx.documents.upsert( + namespace=NAMESPACE, + documents=[ + { + "_id": "doc-1", + "title": "A landmark work that every reader should experience.", + "body": "Lorem ipsum...", + "category": "fiction", + "year": 2024.0, + }, + # ... up to ~1000 documents per call (per public-preview docs) + ], +) +print(upsert_resp.upserted_count) +``` + +Use `upsert` when: + +- You're writing a single document (e.g. a sentinel doc to verify end-to-end before a bulk load). +- You're "patching" a doc after a correction. *Note*: `2026-01.alpha` has **no per-field merge** — every upsert replaces the entire document on conflicting `_id`. To update a single field, fetch the doc, modify in client code, and upsert the full doc back under the same `_id`. +- You're streaming writes from user actions and each request fits in a single batch. + +Each document is a dict keyed by field name. `_id` is required and must be a non-empty unique string within the namespace. Values must match the declared schema types (FTS strings → `str`, filterable `float` → `int|float`, dense vectors → `list[float]`, sparse → `{"indices": [...], "values": [...]}`). Field names that start with `_` or `$` are rejected; field names are limited to 64 bytes. + +The endpoint returns `202 Accepted` (async) and the body's `upserted_count` is the number of items accepted, not the number that have finished indexing. + +## `documents.batch_upsert` — bulk loads + +```python +result = idx.documents.batch_upsert( + namespace=NAMESPACE, + documents=documents, # list of dicts, any length + batch_size=50, + max_workers=2, + show_progress=True, +) +print(f"{result.successful_item_count:,} / {result.total_item_count:,} succeeded") +if result.has_errors: + print(f"Failed batches: {result.failed_batch_count}") + # Always surface the actual reason — silent failures mask payload-size + # caps, schema mismatches, and reserved-field-name violations. + for err in result.errors[:3]: + sample = err.items[0].get("_id") if err.items else "?" + print(f" batch #{err.batch_index} ({len(err.items)} items, " + f"first _id={sample!r}): {err.error_message}") +``` + +The SDK splits `documents` into `batch_size`-sized chunks and uploads them over `max_workers` parallel HTTP connections. `show_progress=True` prints a tqdm-style bar. + +### Tuning `batch_size` and `max_workers` + +- **`batch_size=50`** is the sweet spot — comfortably below the per-request cap and small enough that transient failures cost less to redo. +- **`max_workers=2`** is a safe default. Bump to `4` for large (thousands-of-docs) loads where you're not simultaneously embedding. Ramp cautiously above 4 — you'll hit Pinecone or upstream embedding-provider rate limits first. +- If you're embedding on the fly (computing vectors inside the upsert loop), keep `max_workers` low so embedding latency dominates rather than index write latency. + +### Document and request size caps + +**Hard limits in `2026-01.alpha`:** + +- **Per document**: max **2 MB** (serialized JSON, all stored fields combined). +- **Per `full_text_search` string field**: max **100 KB** AND max **10,000 tokens**. Tokens longer than 256 bytes are silently truncated by the analyzer. +- **Per upsert request**: max **2 MB total** AND max **1,000 documents**. +- **Per document filterable metadata** (everything *not* in an FTS field): max **40 KB** combined. +- **Schema-level**: up to **100 FTS string fields** per index. + +If any one of these is exceeded, the batch fails as a whole. The most common limit to hit on long-prose corpora is the per-FTS-field 100 KB / 10,000-token cap on a single body field — chunking is the standard fix (see below). + +### Dense-vector payload size + +A high-dimensional dense field can silently turn a 50-doc batch into a 5–10 MB request, which the preview backend will reject wholesale. If every batch fails and the error message is opaque, the first thing to try is dropping the embedding dimension before debugging schema: + +- **Gemini**: pass `config=types.EmbedContentConfig(output_dimensionality=768)`. The model uses Matryoshka representations, so smaller dimensions are valid truncations of the native output. 768 is usually a 4× payload reduction vs. the native 3072 and costs very little quality. +- **OpenAI `text-embedding-3-*`**: pass `dimensions=768` (or similar) to `embeddings.create`. +- **Pinecone hosted / fixed-dim models**: dimension is fixed; the only levers are `batch_size` (halve it to 25) and per-document body size. + +## The async-indexing footgun + +After `batch_upsert` returns, **your documents are written but not yet searchable.** The server builds inverted indexes for FTS fields and ANN graphs for vector fields in the background. A search query issued immediately will return empty matches. Schemas with multiple indexed fields (e.g. text + dense + sparse) may take slightly longer. + +**Always poll with a deadline** before trusting the index: + +```python +import time + +deadline = time.time() + 300 # up to 5 minutes +while time.time() < deadline: + resp = idx.documents.search( + namespace=NAMESPACE, top_k=1, + score_by=[{"type": "text", "field": "", "query": ""}], + include_fields=[], # required on every search; [] = ids + _score only + ) + if resp.matches: + print("Data is searchable.") + break + time.sleep(5) + print("Not yet indexed, retrying...") +else: + print("WARNING: Documents may not be fully indexed after 5 minutes.") +``` + +Pick a sentinel query likely to hit at least one document. For a typical corpus, a single common token works (e.g. `"book"` for a book-reviews corpus). For a small corpus, use a term you *know* appears in at least one document. + +## Chunking oversized text + +Per the public-preview docs (above), the per-FTS-field hard limits are 100 KB and 10,000 tokens. In practice, plan for the *token* limit kicking in first on natural prose (~5,000 English words at ~2 tokens each is the rough ceiling). Probe before ingesting at scale — chunk anything that approaches either bound, with safety margin. + +**Strategy: probe first, then chunk if needed.** + +1. Find the longest document in your corpus: `max(len(doc["body"]) for doc in docs)`. +2. Try upserting it as-is. If the upsert errors, chunk. + +**Chunking pattern:** + +```python +def chunk_text(text, max_chars=32_000): + # Simple paragraph-aware chunking. Adjust the boundary for your corpus. + paras = text.split("\n\n") + chunks, cur = [], [] + cur_len = 0 + for p in paras: + if cur_len + len(p) > max_chars and cur: + chunks.append("\n\n".join(cur)) + cur, cur_len = [p], len(p) + else: + cur.append(p) + cur_len += len(p) + if cur: + chunks.append("\n\n".join(cur)) + return chunks + +docs = [] +for doc_id, text, title in source: + chunks = chunk_text(text) + for i, chunk in enumerate(chunks): + chunk_id = doc_id if i == 0 else f"{doc_id}#p{i + 1}" + docs.append({ + "_id": chunk_id, + "parent_doc_id": doc_id, # duplicate identifying metadata across chunks + "title": title, # so title matches hit every chunk + "body": chunk, + }) +``` + +Conventions: + +- **Shared key prefix.** First chunk keeps the original `_id`; subsequent chunks append `#p2`, `#p3`. Easy to parse client-side. +- **Duplicate identifying metadata.** Fields like `title`, `parent_doc_id`, `url`, or whatever identifies the logical document should be present on every chunk so queries that filter or score against those fields work uniformly. +- **Deduplicate at query time.** After `documents.search`, group matches by `parent_doc_id` (or strip the `#p*` suffix from `_id`) and keep the highest-scoring chunk per parent. This preserves relevance ranking while collapsing duplicates in the UI. + +## Updating documents + +There is **no per-field update or merge** in `2026-01.alpha`. `documents.upsert` always replaces the entire document for a given `_id`. To update one field: + +```python +fetched = idx.documents.fetch( + namespace=NAMESPACE, + ids=["doc-42"], + include_fields=["*"], # need the full doc to round-trip it +) +doc = fetched.documents["doc-42"].to_dict() +doc["category"] = "biography" # patch in client code + +idx.documents.upsert(namespace=NAMESPACE, documents=[doc]) +``` + +If the document includes a dense vector, you re-upsert that vector verbatim. If it changes, embed the new content first. + +## Deletes + +`documents.delete` accepts either `ids: [...]` (1–1000 items) or `delete_all: true`. There is **no delete-by-filter** — to delete documents matching a metadata expression, search first to collect IDs, then pass them in: + +```python +ids_to_kill = [ + m._id for m in idx.documents.search( + namespace=NAMESPACE, top_k=1000, + score_by=[{"type": "text", "field": "body", "query": "deprecated"}], + filter={"category": {"$eq": "archive"}}, + include_fields=[], + ).matches +] +idx.documents.delete(namespace=NAMESPACE, ids=ids_to_kill) +``` + +`delete_all=True` wipes the entire namespace. Use carefully. + +## Integrating embedding providers + +If your index has a dense or sparse vector field, you need embeddings. Three common paths: + +### Pinecone hosted inference + +Cleanest integration — no extra API keys, same client as the index. + +```python +# Indexing side: use input_type="passage" for stored content +resp = pc.inference.embed( + model="multilingual-e5-large", + inputs=[doc["body"] for doc in batch], + parameters={"input_type": "passage", "truncate": "END"}, +) +embeddings = [e.values for e in resp.data] + +# Query side: use input_type="query" for query strings +q_resp = pc.inference.embed( + model="multilingual-e5-large", + inputs=[user_query], + parameters={"input_type": "query"}, +) +q_emb = q_resp.data[0].values +``` + +The distinction between `input_type="passage"` (stored content) and `input_type="query"` (runtime queries) matters for models that encode them asymmetrically (`multilingual-e5-large` is one). For sparse learned embeddings like `pinecone-sparse-english-v0`, the same convention applies, and each embedding has `.sparse_indices` / `.sparse_values` rather than `.values`. + +Batch size: ~96 inputs per `embed` call is the typical server limit. Loop in chunks: + +```python +EMBED_BATCH = 96 +embeddings = [] +for i in range(0, len(docs), EMBED_BATCH): + chunk = docs[i : i + EMBED_BATCH] + resp = pc.inference.embed( + model="multilingual-e5-large", + inputs=[d["body"] for d in chunk], + parameters={"input_type": "passage", "truncate": "END"}, + ) + embeddings.extend(e.values for e in resp.data) +``` + +### Generic pattern — any third-party provider + +Wrap the provider-specific call in a thin adapter so ingestion logic doesn't know which provider is in use: + +```python +def embed(content) -> list[float]: + """Return a single dense embedding for a piece of content. + + `content` may be a string or a PIL.Image, depending on the provider. + Swap the implementation to change providers without touching callers. + """ + resp = provider.embed(content) + return resp.values # or resp.data[0].embedding, etc. + +docs = [{"_id": d["id"], "body": d["text"], "embedding": embed(d["text"])} for d in source] +``` + +This adapter also gives you a single chokepoint for retries, rate-limit backoff, and caching — add them once in `embed()` rather than at every call site. + +## Limits to be aware of + +- **No bulk import (S3 import job)** for document-shaped indexes in `2026-01.alpha`. Load through `documents.upsert` / `documents.batch_upsert`. +- **No backup/restore.** If you need recoverability, snapshot your source data, not the index. +- **No CMEK projects** — indexes can't be created in CMEK-enabled projects. +- **Indexing latency**: documents become searchable in ≲1 minute typically; multi-field schemas can take slightly longer. + diff --git a/skills/pinecone-full-text-search/references/onboarding-walkthrough.md b/skills/pinecone-full-text-search/references/onboarding-walkthrough.md new file mode 100644 index 0000000..65407c4 --- /dev/null +++ b/skills/pinecone-full-text-search/references/onboarding-walkthrough.md @@ -0,0 +1,159 @@ +# Onboarding walkthrough — taking unprocessed data to a working FTS index + +Load this file when the user shows up with **unprocessed data** and asks "make this searchable in Pinecone." The walkthrough is conversational on purpose — the goal is to help the user understand what decisions are being made, not to surprise them with a finished schema. **At each `ASK` beat, stop and wait for the user's response before proceeding.** Schemas are immutable; ingest is async; getting any of this wrong costs a re-ingest, so it's worth the chat round-trips. + +## When this walkthrough applies + +- The user has data sitting in a file (CSV, JSONL, JSON, Parquet, Postgres dump, etc.) +- The data hasn't been pre-cleaned for Pinecone — types may be off, fields may be messy, long bodies may exceed FTS limits, duplicates may exist +- The user said something like "make this searchable" / "build me a search index" / "put this in Pinecone" +- The user did NOT provide an explicit schema or describe one in detail + +If the user already gave you a schema spec or a JSONL of clean records, skip to UC-1's standard steps in SKILL.md. + +If the user has an integrated-embedding records index already and wants to add FTS to it, see the Scope section in SKILL.md — that's a different surface and you can't add FTS fields after the fact. + +## Stage 1 — Meet the data + +**Goal:** Both you and the user need to see the actual shape of the data before you can decide what to do with it. + +**Action:** +1. Read the first 3-5 records of the file. Don't skim — look at field names, types, lengths, anything that looks off. +2. Summarize what you see in chat. Be concrete: + - "Your file has N records." + - "Each record has fields: A (string, ~X chars), B (string, up to Y chars), C (number), D (looks like a list/array)." + - "I noticed: " + +**ASK the user** (one chat turn, numbered): +1. "Is this what you expected, or are there fields I should ignore / rename?" +2. "Are there any records that aren't representative? (e.g. test rows, debug entries)" +3. "Any fields that should be unique IDs? (Pinecone needs a `_id` per document; if your data already has a unique key like `slug` or `uuid`, we'll map it.)" + +**Wait for the response.** If the user adjusts your understanding, re-read what you need and re-summarize. Don't move on until they've confirmed you understand the data. + +## Stage 2 — Surface processing decisions + +**Goal:** Most "unprocessed data" needs *some* transformation before it fits the FTS API. Surface the decisions so the user knows what you're going to do. + +For each issue you saw in Stage 1, say what's happening and ask the user how to handle it. **Don't decide silently.** Common cases: + +### Long FTS text fields + +If any text field exceeds **100 KB** (or roughly **10,000 tokens** ≈ ~5,000 English words): +- Tell the user: "Field `body` has records up to N KB. The Pinecone FTS limit is 100 KB / 10,000 tokens per field. We'll need to chunk longer ones." +- **ASK**: "Want to chunk by paragraph (the standard for prose), by fixed character count, or by semantic units I can detect? When I chunk, the first sub-document keeps the original `_id` and subsequent ones get a suffix like `doc-42#p2`, `doc-42#p3` (the convention used by `scripts/ingest.py`). Sound OK?" + +### Type mismatches (CSV/Excel/Postgres common cases) + +If types don't match what an FTS schema would want: +- **Numbers stored as strings**: "Your `year` field is a string like `"2024"`. The schema needs a number. I'll coerce — but if any value can't be parsed, I'll abort and show you the offending row." +- **Booleans as strings**: same. "Convert `"true"` / `"false"` to booleans?" +- **Comma-separated tags**: "`tags` is a string `'classic,american'`. The schema would index tags as a list. I'll split on `,` — speak up if your data uses a different separator." +- **Dates / timestamps**: "Pinecone has no date type. We'll either store as ISO-8601 strings (filterable for exact match), or convert to epoch milliseconds (filterable as numeric). Which do you want?" + +### Missing fields + +If some records are missing fields others have: +- "About M of N records are missing `intro`. Do you want to: skip those records / store them with the field omitted / set a default?" + +### Duplicates + +If `_id` (or your nominated unique key) repeats: +- "I found K duplicate IDs. Default behavior is last-wins (the later record overwrites). Or I can: abort if duplicates exist / merge fields across duplicates / first-wins. Which?" + +### Reserved field names + +If any field starts with `_` or `$`: +- "Your data has a field called `_started_at` — Pinecone reserves leading-underscore names for system fields. I'll rename to `started_at` unless you want a different name." + +**ASK** (one chat turn): present each decision as a numbered question. Wait for answers. Don't proceed until every decision has a confirmed answer. + +## Stage 3 — Propose the schema + +**Goal:** Translate the user's data + their answers from Stage 2 into a concrete Pinecone schema. Show it in chat as readable Python, with a one-line rationale per field. + +**Action:** Write a chat message like: + +> Based on what we've discussed, here's the schema I'd create: +> +> ```python +> SchemaBuilder() +> # Searchable text — long prose, stemming on so "running" matches "ran" +> .add_string_field("body", full_text_search={"language":"en", "stemming":True}) +> # Searchable text — short titles, stemming off (proper nouns shouldn't over-match) +> .add_string_field("title", full_text_search={}) +> # Filter only — exact-match category like "fiction" +> .add_string_field("category", filterable=True) +> # Numeric range filter (e.g. year > 2024) +> .add_integer_field("year", filterable=True) +> # Tag filter — list membership ($in) +> .add_string_list_field("tags", filterable=True) +> .build() +> ``` +> +> A few notes: +> - **No dense_vector field** — you said you don't have embeddings yet. We can add one later, but it requires creating a *new* index because schemas are immutable. Want to add a placeholder now and keep the door open? +> - **`year` uses `add_integer_field`** but Pinecone stores it as `float` on the wire. The naming is confusing but normal — there's no separate integer type. +> - **`tags`** will become `["a","b","c"]` after the comma-split we discussed. +> +> Schemas are immutable in `2026-01.alpha` — once we create this, changing it means re-creating the index and re-ingesting all the data. + +**ASK** (one question this time): "Look right? Want to adjust anything before I create the index?" + +**Wait.** Do not proceed until the user explicitly approves (`yes`, `looks good`, `go ahead`, `ship it`, etc.). If they ask for changes, revise the schema in chat, re-show, ask again. + +## Stage 4 — Create the index + +Once approved: +1. Write the Python (`create.py` or inline) using the approved schema. +2. Run it. Poll until `pc.preview.indexes.describe(name).status.ready: True`. +3. Tell the user when it's ready: "Index `` created and ready. Now ingesting your data." + +If creation fails for a reason you didn't anticipate (e.g. name conflict, region mismatch, CMEK restriction), tell the user the specific error and how to fix — don't auto-retry under a different name without asking. + +## Stage 5 — Process and ingest + +**Goal:** Actually transform the data per Stage 2 decisions and bulk-load. + +**Action:** +1. Write a small processing script that applies the agreed transformations: type coercion, chunking, dedup, list splitting, etc. Save the result to `processed.jsonl`. +2. Show the user a summary: "Wrote N processed records (was M raw; X chunked / Y deduped / Z dropped). Sample record: ``." +3. **ASK** (only if any record was dropped or substantially changed): "Look right?" If they confirm, proceed. +4. Invoke `scripts/ingest.py --data processed.jsonl --index --sentinel-field `. The script handles `batch_upsert` + error inspection + readiness polling. Don't reimplement that loop. +5. Watch its output. If it fails, tell the user *what* it complained about (field type mismatch, payload size, etc.) — don't just say "ingest failed." + +## Stage 6 — Verify together + +**Goal:** Confirm the data is actually searchable. Don't trust polling alone — run a real query. + +**Action:** +1. Pick a sentinel query you know should match a known record. E.g. for a corpus of book reviews, the first word of the first record's `body`; for a corpus with named entities, a known title or proper noun against the relevant FTS field. +2. Run a `documents.search(...)` call against the index — `score_by=[{"type":"text", "field":"", "query":""}]`, `include_fields=["*"]`, `top_k=3`. See the **Querying** section in SKILL.md for the canonical shape. +3. Show the user the results: "Search returned K matches. Top match was `` with score ``." +4. If it didn't match what you expected, debug *with* the user — don't silently rerun. The async-indexing window may not have closed yet, OR your processing dropped a field, OR the user's expectation was off. + +## Stage 7 — Hand off + +**Goal:** Make sure the user can use the index without you. + +Tell them: +1. **The index name and schema** in one line. +2. **How to query** — give them a copy-pasteable `idx.documents.search(...)` snippet shaped to their schema (one `score_by` clause + the `include_fields` they care about). Refer them to the **Querying** section in SKILL.md or `references/querying.md` for variations. +3. **How to ingest more** — `scripts/ingest.py` with the same `--sentinel-field` they should use. +4. **How to delete** — `pc.preview.indexes.delete("")` when done. +5. **What's in the way of changing the schema** — recreate + re-ingest, no schema migration. + +Optionally save a small `README.md` in the working directory with the same info, so they have it when they come back. + +## Anti-patterns — don't do these + +- **Don't decide silently.** Every decision in Stage 2 should be surfaced. If you assume a separator, a coercion, a dedup policy, you'll be wrong sometimes and the user won't know to push back. +- **Don't call `indexes.create()` without explicit approval** — schemas are immutable. +- **Don't write a giant pre-flight script** that does Stages 1-2 in code without ever showing the user. The point is the conversation, not the automation. +- **Don't skip Stage 6.** Polling says the index is "ready"; only a real query confirms the documents are there in the shape you expected. +- **Don't add a dense_vector field "just in case."** It commits the user to a specific embedding dimension forever — and if they don't have embeddings to ingest, the field is useless. +- **Don't promise reversibility.** Whenever you say "we can change this later," follow up with: "...by creating a new index and re-ingesting. There's no schema migration." + +## Non-interactive runtimes + +If you're running in a non-interactive harness (CI bot, batch processor, eval sandbox where the user can't respond), make best-effort decisions, document each one in chat or comments, and proceed. Document the assumptions clearly so a human reviewing later can spot them: "I picked stemming=on for `body` because it's long prose. I assumed comma-separated `tags`. I dropped duplicate `_id`s with last-wins." When in doubt, prefer the conservative choice (don't chunk if you're unsure how, don't coerce if the values look ambiguous). diff --git a/skills/pinecone-full-text-search/references/querying.md b/skills/pinecone-full-text-search/references/querying.md new file mode 100644 index 0000000..86da8b5 --- /dev/null +++ b/skills/pinecone-full-text-search/references/querying.md @@ -0,0 +1,341 @@ +# Querying + +All reads on a Pinecone preview document index go through `idx.documents.search(...)` (ranked) or `idx.documents.fetch(...)` (direct, ID-only). The interesting shape is the **single** `score_by` clause and the `filter={...}` predicate — everything else is plumbing. + +## The one-scoring-type rule + +A single `documents.search` request ranks by **one** scoring type. `score_by` accepts a list, but every entry must share a type: + +- Multiple `text` clauses (one per field) — that's how multi-field BM25 works. +- A single `query_string` clause (which can target multiple fields via `fields: [...]` or inline `field:term` syntax inside the Lucene expression). +- `dense_vector` clauses must appear alone. +- `sparse_vector` clauses must appear alone. +- You **cannot** blend types — no `text` + `query_string`, no `text` + `dense_vector`, no cross-type mix. The server rejects it. + +To compose lexical and dense / sparse signals, put the lexical signal in `filter` via the text-match operators (`$match_phrase` / `$match_all` / `$match_any`) and let the vector clause in `score_by` do the ranking. That's the supported hybrid pattern in `2026-01.alpha`. + +## `score_by` signal types + +### 1. `text` — BM25 token-OR on a single text field + +```python +resp = idx.documents.search( + namespace=NAMESPACE, + top_k=5, + score_by=[{"type": "text", "field": "body", "query": "beautifully written"}], + include_fields=["*"], +) +``` + +Tokenizes `query` with the field's analyzer, scores each matching document with a BM25 ranker over the inverted index, returns the top `top_k`. Multiple terms use **OR semantics** — documents matching any token participate, those matching more / rarer tokens score higher. Phrase constraints (adjacent words in order) are **not** supported here — use `query_string` with quotes, or a `$match_phrase` filter, for phrase semantics. + +`field` is a **single string** (singular) naming an FTS-enabled `string` field — `text` clauses are scoped to one field at a time. For multi-field BM25, pass several `text` clauses (one per field) or use a `query_string` clause with a `fields` array (see Multi-field BM25 below). + +### 2. `query_string` — Lucene syntax (boolean / phrase / boost / slop / prefix / cross-field) + +```python +resp = idx.documents.search( + namespace=NAMESPACE, + top_k=5, + score_by=[{ + "type": "query_string", + "query": 'body:(classic AND ("masterpiece" OR timeless)) NOT body:boring', + }], + include_fields=["*"], +) +``` + +Supported operators (full table in the public-preview docs, summarized here): + +| Operator | Syntax | Example | +|----------------|---------------------|-----------------------------------| +| Term | `field:(word)` | `body:(computers)` | +| Multiple terms | `field:(a b)` | `body:(machine learning)` (OR) | +| Exact phrase | `field:("words")` | `body:("machine learning")` | +| AND / OR / NOT | `AND` / `OR` / `NOT`| `body:(a AND (b OR c)) NOT d` | +| Required | `+term` | `body:(+database search)` | +| Excluded | `-term` | `body:(database -deprecated)` | +| Phrase slop | `"…"~N` | `body:("fast search"~2)` | +| Boost | `term^N` | `body:(machine^3 learning)` | +| Phrase prefix | `"… word"*` | `body:("james w"*)` | +| Cross-field | `f1:(…) OR f2:(…)` | `title:(quantum) OR body:(quantum machine)` | + +**Cross-field clauses** are unique to `query_string` — they let one expression target multiple text-searchable fields with their own sub-clauses. Optionally pass a top-level `fields` array on the clause to restrict scope; omitted, the query runs against every text-searchable field in the schema. + +```python +score_by=[{ + "type": "query_string", + "fields": ["title", "body"], # optional; restricts the query + "query": 'title:(quantum)^2 OR body:("machine learning")', +}] +``` + +Single-term prefix wildcards (`auto*`) are **not** supported. Use phrase prefix instead: `"machine lea"*` (phrase must contain at least two terms; only the last is matched as prefix). + +### 3. `dense_vector` — score against a stored dense vector + +```python +resp = idx.documents.search( + namespace=NAMESPACE, + top_k=5, + score_by=[{"type": "dense_vector", "field": "embedding", "values": query_vector}], + include_fields=["title", "body"], +) +``` + +`field` is a single string (singular) naming a `dense_vector` field. `values` is a `list[float]` matching the field's declared dimension. Typically produced by embedding the user's query through the same (or a compatible) model at runtime. For text embedders with a passage/query distinction (e.g. `multilingual-e5-large`), use `input_type="query"` on the query side. Must appear alone in `score_by`. + +### 4. `sparse_vector` — score against a stored sparse vector + +```python +resp = idx.documents.search( + namespace=NAMESPACE, + top_k=5, + score_by=[{ + "type": "sparse_vector", + "field": "sparse_embedding", + "sparse_values": {"indices": q.sparse_indices, "values": q.sparse_values}, + }], + include_fields=["title", "body"], +) +``` + +Stored and queried as `{"indices": [...], "values": [...]}`. Hosted sparse models (e.g. `pinecone-sparse-english-v0`) return embeddings with `.sparse_indices` and `.sparse_values` ready to drop in. Must appear alone in `score_by`. + +## Multi-field BM25 + +Two equivalent ways to score across multiple text fields in one request: + +**Option A — multiple `text` clauses (one per field):** + +```python +score_by=[ + {"type": "text", "field": "title", "query": q}, + {"type": "text", "field": "intro", "query": q}, + {"type": "text", "field": "body", "query": q}, +] +``` + +**Option B — one `query_string` cross-field expression:** + +```python +score_by=[{ + "type": "query_string", + "query": f'title:({q}) OR intro:({q}) OR body:({q})', +}] +``` + +Both reward documents that match in multiple fields. **`2026-01.alpha` weights every contributing field equally** — there is no per-clause weight parameter. To approximate weighting, use Option B with `^N` term boosts inside the query string (`title:({q})^3 OR body:({q})`). + +## Filtering + +Filters run **before** scoring — they shrink the candidate set, then the chosen `score_by` ranks survivors. Two families of operators. + +### Text-match filters (on text-searchable fields) + +These operate on `full_text_search`-enabled fields and reuse the field's tokenizer / stemmer. Each value is a single string (max 128 tokens). Not available inside `query_string` — they live in `filter`. + +| Operator | Semantics | +|------------------|--------------------------------------------------------------| +| `$match_phrase` | Exact phrase match — tokens must be contiguous and in order. | +| `$match_all` | All tokens present, in any order. | +| `$match_any` | At least one token present. | + +```python +filter={"body": {"$match_phrase": "machine learning"}} # exact phrase +filter={"body": {"$match_all": "machine learning"}} # both tokens, any order +filter={"body": {"$match_any": "AI robotics"}} # either token +``` + +These are the supported way to compose lexical pre-filtering with `dense_vector` (or `sparse_vector`) scoring — see "Cross-modal hybrid" below. + +> **Scoring-only operators don't go in `filter`.** Phrase slop (`"…"~N`), term boost (`^N`), and phrase prefix (`"… word"*`) influence ranking, so they're available in `query_string` `score_by` but not in `filter`. + +### Metadata filters (on `filterable: true` fields) + +Standard comparison and membership operators — work on `string`, `string_list`, `float`, and `boolean` filterable fields. + +| Operator | Example | Semantics | +|----------|-------------------------------------------------------------|--------------------------------------| +| `$eq` | `{"category": {"$eq": "tech"}}` | Equals | +| `$ne` | `{"category": {"$ne": "archive"}}` | Not equals | +| `$gt` | `{"year": {"$gt": 2023}}` | Greater than | +| `$gte` | `{"year": {"$gte": 2023}}` | Greater than or equal | +| `$lt` | `{"year": {"$lt": 2025}}` | Less than | +| `$lte` | `{"year": {"$lte": 2025}}` | Less than or equal | +| `$in` | `{"category": {"$in": ["a", "b"]}}` | In list (works on `string_list` too) | +| `$nin` | `{"category": {"$nin": ["a", "b"]}}` | Not in list | +| `$exists`| `{"category": {"$exists": true}}` | Field has a value (`true`) or absent | + +### Composing with `$and` / `$or` / `$not` + +Multiple keys at the top level of a filter object are implicitly AND-ed. Use `$and`, `$or`, `$not` for explicit / nested composition. Text-match and metadata filters compose freely: + +```python +filter={ + "$and": [ + {"body": {"$match_all": "federal reserve"}}, # text-match operator + {"category": {"$eq": "finance"}}, # metadata operator + {"year": {"$gte": 2024}}, + {"$not": {"tags": {"$in": ["opinion"]}}}, + ], +} +``` + +## Cross-modal hybrid: dense ranking + text-match filter + +The supported way to compose lexical and dense signals in one request: dense (or sparse) `score_by`, plus a text-match `filter` that hard-restricts the candidate set to documents whose lexical field contains the right tokens / phrase. + +```python +results = idx.documents.search( + namespace=NAMESPACE, + top_k=10, + filter={"body": {"$match_phrase": "beautifully written"}}, + score_by=[{ + "type": "dense_vector", + "field": "review_embedding", + "values": embed("a moving family epic"), + }], + include_fields=["*"], +) +``` + +Read it top-down: only docs whose `body` contains the exact phrase `"beautifully written"`, ranked by dense-vector similarity to the embedding of "a moving family epic." One round trip, server-side hard filter, dense rerank. + +When to use which text-match operator inside a hybrid query: + +| Use `$match_phrase` when… | Use `$match_all` when… | Use `$match_any` when… | +|---------------------------|----------------------------------------------|----------------------------------------| +| Adjacency matters (named events, idioms, multi-word concepts where order is the signal). | All tokens are required but order is not (geography + topic, e.g. `"illinois cardinal"`). | At least one token is enough (broader recall — useful as a soft filter). | + +## `include_fields` modes + +`include_fields` controls what each match object carries back in the response. + +| Value | Behaviour | +|------------------------------|----------------------------------------------------------------| +| *(omitted, or `null`)* | Defaults to `[]` — `_id` and `_score` only. | +| `[]` | `_id` and `_score` only (lightest payload). | +| `["*"]` | All stored fields (including fields not declared in the schema).| +| `["field1", "field2"]` | Only the listed fields (projection). | + +**Always pass `include_fields` explicitly** on `documents.search`. Some SDK builds default to `[]`; some return `400` / `422` if it's missing. Being explicit avoids surprises and makes the call's intent obvious. + +User metadata fields literally named `score` are returned alongside the system-owned `_score` match score — the leading underscore prevents collisions. + +## Reading match objects + +Match objects carry: + +- `_id` (string) — document ID. +- `_score` (float) — system match score; **higher is better**. +- The fields requested via `include_fields`. + +The `score` field name is reserved for **user metadata**; the system match score is always `_score`. Older SDK / backend builds may still emit unprefixed `score`; reading via `getattr(match, "_score", getattr(match, "score", None))` covers both. + +## `documents.fetch` — direct retrieval, ID-only + +Fetch is **ID-only** in `2026-01.alpha`. It does **not** accept a `filter`. To retrieve documents matching a metadata expression, search first to get IDs, then fetch: + +```python +fetched = idx.documents.fetch( + namespace=NAMESPACE, + ids=["doc-1", "doc-2", "does-not-exist"], + include_fields=["*"], +) +for doc_id, doc in fetched.documents.items(): + print(doc_id, doc.to_dict()) +``` + +Missing IDs are silently omitted from the response (no error). `ids` accepts 1–1000 entries per call. + +## `documents.delete` — by ID or `delete_all` + +```python +# By IDs (1–1000 per call). Non-existent IDs are silently ignored. +idx.documents.delete( + namespace=NAMESPACE, + ids=["doc-1", "doc-2"], +) + +# Wipe the entire namespace. +idx.documents.delete( + namespace=NAMESPACE, + delete_all=True, +) +``` + +Delete does **not** accept a `filter`. To delete documents matching a metadata expression, search first to collect IDs, then pass them to `delete`. Deletes are permanent within the namespace. + +## Worked cross-modal example — "pick your signal" pattern + +One index with two FTS text fields and one multimodal dense vector field. The dense field holds an embedding that lives in a shared text/image space (e.g. a Gemini multimodal embedding of each document's representative image). Because text and image share the space, a typed description can be embedded as text and scored against the stored image vectors. + +### Schema + +```python +schema = ( + SchemaBuilder() + .add_string_field("title", full_text_search={"language": "en"}) + .add_string_field("body", full_text_search={"language": "en", "stemming": True}) + .add_dense_vector_field("image_embedding", dimension=DIM, metric="cosine") + .build() +) +``` + +### Three query modes against the same index + +**1. Pure text — multi-field BM25 token-OR.** + +```python +resp = idx.documents.search( + namespace=NS, + top_k=10, + score_by=[ + {"type": "text", "field": "title", "query": "transformer architecture"}, + {"type": "text", "field": "body", "query": "transformer architecture"}, + ], + include_fields=["title", "body"], +) +``` + +**2. Exact phrase via `query_string`.** + +```python +resp = idx.documents.search( + namespace=NS, + top_k=10, + score_by=[{ + "type": "query_string", + "query": 'body:("attention is all you need")', + }], + include_fields=["title", "body"], +) +``` + +**3. Pure dense — semantic query against stored vectors.** + +```python +q_emb = embed("a paper introducing self-attention for sequence modeling") +resp = idx.documents.search( + namespace=NS, + top_k=10, + score_by=[{"type": "dense_vector", "field": "image_embedding", "values": q_emb}], + include_fields=["title", "body"], +) +``` + +**4. Hybrid — `$match_all` filter narrows; dense ranks.** + +```python +q_emb = embed("self-attention for sequence modeling") +resp = idx.documents.search( + namespace=NS, + top_k=10, + filter={"body": {"$match_all": "transformer"}}, + score_by=[{"type": "dense_vector", "field": "image_embedding", "values": q_emb}], + include_fields=["title", "body"], +) +``` + +The same index supports all four modes; which one you want depends on whether the user's intent is keyword-driven (Mode 1), phrase-driven (Mode 2), appearance-driven (Mode 3), or "constrain by keyword, rank by appearance" (Mode 4). That's the pick-your-signal pattern — build the index once, vary the query shape per user intent. diff --git a/skills/pinecone-full-text-search/references/schema-design.md b/skills/pinecone-full-text-search/references/schema-design.md new file mode 100644 index 0000000..7bf5659 --- /dev/null +++ b/skills/pinecone-full-text-search/references/schema-design.md @@ -0,0 +1,171 @@ +# Schema design + +Everything a Pinecone preview document index needs is declared up-front via `SchemaBuilder`. The schema pins which fields are searchable, which are filterable metadata, which hold vectors, and what their dimensions / metrics are. **Schemas are fixed at index creation in `2026-01.alpha`** — adding, removing, or retyping fields afterwards is not supported. Plan carefully. + +## `SchemaBuilder` overview + +```python +from pinecone.preview import SchemaBuilder + +schema = ( + SchemaBuilder() + .add_string_field("title", full_text_search={"language": "en"}) + .add_string_field("body", full_text_search={"language": "en", "stemming": True}) + .add_string_field("category", filterable=True) + .add_integer_field("year", filterable=True) # emits `"type": "float"` on the wire + .add_dense_vector_field("embedding", dimension=1024, metric="cosine") + .add_sparse_vector_field("sparse_embedding", metric="dotproduct") + .build() # terminal: returns the schema object you pass to indexes.create +) +``` + +`.build()` is the terminal call — every chain ends with it. The resulting schema is passed to `pc.preview.indexes.create(name=..., schema=schema, read_capacity=...)`. `read_capacity` defaults to `{"mode": "OnDemand"}` (auto-scaled shared reads); pass `{"mode": "Dedicated", "dedicated": {...}}` only if you specifically want provisioned read nodes. + +## Field types at a glance + +| Type | Purpose | Required options | How it's queried | +|-----------------|--------------------------------------------|-------------------------------------------|-------------------------------------------| +| `string` (text) | Full-text search (BM25 / Lucene) | `full_text_search: {...}` (dict, may be `{}`) | `score_by` `text` or `query_string`; filter via `$match_phrase` / `$match_all` / `$match_any` | +| `string` (metadata) | Exact-match metadata filtering | `filterable: true` | `filter` with `$eq` / `$in` / `$ne` / `$nin` / `$exists` | +| `string_list` | Array-valued metadata filtering | `filterable: true` | `filter` with `$in` / `$nin` (membership) | +| `float` | Numeric metadata filtering | `filterable: true` | `filter` with `$eq` / `$gt` / `$gte` / `$lt` / `$lte` / `$in` / `$nin` | +| `boolean` | Boolean metadata filtering | `filterable: true` | `filter` with `$eq` / `$exists` | +| `dense_vector` | ANN similarity search | `dimension`, `metric` (`cosine` / `dotproduct` / `euclidean`) | `score_by` `dense_vector` | +| `sparse_vector` | Sparse-vector lexical / hybrid scoring | `metric` (typically `dotproduct`) | `score_by` `sparse_vector` | + +Every field can also include an optional `description` string — surfaced by `DescribeIndex` and useful for agentic workflows where an LLM inspects the schema to decide how to query. + +## Reserved field names + +Field names must be unique, non-empty strings. Two hard rules: + +- **Must not start with `_`** — reserved for system-managed fields (`_id`, `_score`). +- **Must not start with `$`** — reserved for filter operators. +- **Limited to 64 bytes** (bytes, not characters — non-ASCII names take extra space). + +`_id` is required on every document. `_score` is the system match-score field name returned by `documents.search`. A user metadata field literally named `score` is allowed and won't collide with `_score`. + +## String fields — text vs. metadata + +A single `string` field is *either* full-text-search (BM25 / Lucene scoring + text-match filters) **or** filterable metadata (exact-match), never both. If you need both surfaces for the same logical content, duplicate it into two differently-configured fields. + +### Full-text-searchable string + +```python +.add_string_field("body", full_text_search={"language": "en", "stemming": True}) +``` + +`full_text_search` takes a dict. Pass `{}` for all server defaults; populate it with any of: + +- `language` (string, default `"en"`) — selects the analyzer (tokenizer + stemmer + stopword set). Supported short codes: `ar`, `da`, `de`, `el`, `en`, `es`, `fi`, `fr`, `hu`, `it`, `nl`, `no`, `pt`, `ro`, `ru`, `sv`, `ta`, `tr`. Full names are also accepted (e.g. `"english"`, `"french"`, `"arabic"`). Stop-word lists are available for most languages but a few are tokenize/stem only (no stop_word filtering even when `stop_words: true` is set) — `ar`, `da`, `de` are notable cases; `en`, `es`, `fr` etc. have full stop-word support. +- `stemming` (boolean, default `false`) — if `true`, applies the language's stemmer so `running` matches `runs`. +- `stop_words` (boolean, default `false`) — if `true`, the analyzer's stopword set is filtered out at index and query time. +- `lowercase` (boolean, default `true`, server-applied) — case-insensitive matching. +- `max_token_length` (int, default `40`, server-applied) — discards excessively long tokens. + +Heuristic on stemming: turn it on for long prose fields where morphological variants of a root should match (`running` ~ `runs` ~ `ran`); leave off for short / identifier fields like titles, tags, or proper nouns where stemming would over-match (a book titled `Running` probably shouldn't also match the query `ran`). Typical pattern: stemming on for `body`, off for `title` / proper-noun fields. + +Enables, on `field_name`: +- BM25 token scoring with `score_by=[{"type": "text", "field": "field_name", "query": "..."}]`. +- Lucene scoring with `score_by=[{"type": "query_string", "query": "field_name:(a AND (b OR c)) NOT field_name:d"}]`. +- Phrase / token filters: `filter={"field_name": {"$match_phrase": "..."}}`, `{"$match_all": "..."}`, `{"$match_any": "..."}`. + +### Filterable-only string + +```python +.add_string_field("category", filterable=True) +``` + +Stored verbatim, not tokenized, not text-scored. Enables exact-match filtering: `{"category": {"$eq": "fiction"}}`, `{"category": {"$in": ["fiction", "biography"]}}`, `{"category": {"$exists": true}}`. + +## Numeric, boolean, and array metadata + +```python +.add_integer_field("year", filterable=True) # wire type: "float" +.add_custom_field("featured", {"type": "boolean", "filterable": True}) # no add_boolean_field helper in v9 +.add_string_list_field("tags", filterable=True) +``` + +- **`float`** is the only numeric wire type — there is no separate integer type. The SchemaBuilder helper is misleadingly named `add_integer_field` but emits `{"type": "float", "filterable": ...}`. Supports `$eq`, `$ne`, `$gt`, `$gte`, `$lt`, `$lte`, `$in`, `$nin`, `$exists`. +- **`boolean`** has no dedicated builder helper in pinecone v9 — declare via `add_custom_field("name", {"type": "boolean", "filterable": True})`. Supports `$eq` and `$exists`. +- **`string_list`** supports `$in` / `$nin` membership semantics — handy for tag-style metadata. + +All filter operators compose under `$and`, `$or`, `$not`. Multiple keys at the top level of `filter` are combined with implicit AND. + +> **SchemaBuilder helper-name pitfall** (pinecone v9): `add_integer_field()` produces `{"type": "float"}`, not a separate integer type. The class name in `describe()` responses is also `PreviewIntegerField`, but the wire/server type is `"float"`. Use `add_integer_field` for any numeric metadata; use `add_custom_field` with an explicit `{"type": "boolean", ...}` dict for booleans. + +> **Forward-looking note.** In the public preview, metadata fields you send at upsert time are auto-indexed for filtering even if not declared in the schema. In a future release, only schema-declared fields with `filterable: true` will be indexed. Declare your metadata fields in the schema today to be future-proof. + +> **Metadata size limit.** Filterable metadata on a single document is capped at **40 KB** combined (everything that's not in an FTS-enabled `string` field). FTS-enabled `string` fields don't count toward this — they have their own per-field limit (100 KB / 10,000 tokens, see `references/ingestion.md`). + +## Dense vector fields + +```python +.add_dense_vector_field("embedding", dimension=1024, metric="cosine") +``` + +- `dimension` must match whatever embedding model you'll store. If the model is chosen at runtime, query its default dimension first (e.g. `pc.inference.get_model(model="multilingual-e5-large").default_dimension`) and pass that in. +- `metric` is one of `"cosine"`, `"dotproduct"`, `"euclidean"`. Pick the metric the embedding provider recommends — most text embedders use cosine. +- Scored at query time with `score_by=[{"type": "dense_vector", "field": "embedding", "values": [...]}]`. + +**At most one `dense_vector` field per index** in `2026-01.alpha`. If you need two semantically distinct dense signals, you need two indexes. + +## Sparse vector fields + +```python +.add_sparse_vector_field("sparse_embedding", metric="dotproduct") +``` + +- No `dimension` — sparse vectors are variable-length. +- `metric="dotproduct"` is the standard choice for learned sparse embeddings (e.g. `pinecone-sparse-english-v0`). +- Stored and queried as `{"indices": [...], "values": [...]}`; query side: `score_by=[{"type": "sparse_vector", "field": "sparse_embedding", "sparse_values": {"indices": [...], "values": [...]}}]`. + +**At most one `sparse_vector` field per index** in `2026-01.alpha`. + +## When to add a dense field at all + +This is the key design question when the index already has FTS fields. **Only add a dense vector field when it represents a modality or signal that FTS cannot express.** Examples of justified dense fields: + +- An **image embedding** over pictures associated with each document — visual appearance is not text. +- An **audio embedding** over voice clips or music — timbre and melody are not text. +- An **external ranking-model score** pre-computed and stored as a 1-D "vector" for sort purposes. +- A **semantic text embedding over a different corpus** than the one in the FTS field — e.g. the FTS field holds the product description, the dense field holds an embedding of the seller's support-ticket history for that product. Different data, different signal. + +Anti-pattern: **re-encoding text that already lives in an FTS field on the same index.** Indexing the `body` string as FTS *and* embedding that same `body` into a dense text vector on the same index is redundant modeling, not an additive signal. FTS already gives you lexical retrieval; adding a dense re-encoding only pays off when the lexical signal is demonstrably insufficient (typically: very large corpus, very semantic queries, and you've measured the gap). + +## Multi-field text design heuristics + +When a document has a natural hierarchy (title → intro → body, or summary → transcript, or headline → lede → article), splitting across FTS fields enables two things you can't get from one blob: + +1. **Per-field scoring.** A match on `title` is almost always a stronger signal than a match on `body`. With separate fields you can search just the title, just the body, or blend them at query time by listing each as its own `score_by` entry (see `references/querying.md` — multi-field BM25). +2. **Multi-field blended relevance.** Passing `score_by=[{text, title, q}, {text, intro, q}, {text, body, q}]` rewards documents that match in multiple fields. (`2026-01.alpha` weights every contributing field equally — no per-clause weight parameter.) + +Keep it a single field when: + +- The content has no natural subdivision (a tweet, a log line, a chat message). +- You will never want per-field weighting at query time. +- Your documents are short enough that inter-field distinctions are noise. + +## Schemas are fixed at creation + +`2026-01.alpha` does **not** support schema migration. You cannot: + +- Add a new field after creation. +- Remove an existing field. +- Change a field's type or sub-config (e.g. flip a filterable string to FTS, toggle stemming, change dense vector dimension). + +The supported workaround is to create a new index with the desired schema and reindex documents (the document set is small enough at preview-launch scale that this is usually painless). Existing pre-public-preview indexes from earlier API versions cannot be backfilled with a 2026-01.alpha schema. + +## `description` for agentic / LLM-driven workflows + +Each field accepts an optional `description` string: + +```python +.add_string_field( + "body", + full_text_search={"language": "en", "stemming": True}, + description="Full article text. Use for keyword searches, narrative phrases, and topical queries.", +) +``` + +Returned by `DescribeIndex`. Useful when an LLM is choosing how to query: it can read the descriptions and pick the right field + operator without hard-coded prompt engineering. diff --git a/skills/pinecone-full-text-search/scripts/ingest.py b/skills/pinecone-full-text-search/scripts/ingest.py new file mode 100755 index 0000000..4c26148 --- /dev/null +++ b/skills/pinecone-full-text-search/scripts/ingest.py @@ -0,0 +1,281 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "typer>=0.12", +# "pinecone==9.0.0", +# ] +# /// +"""Ingest a JSONL file into a Pinecone FTS index — safely. + +A bare-LLM ingest path skips three things and breaks in three different ways: + + 1. Per-doc upsert in a Python loop instead of `batch_upsert`. Slow. + 2. Discards the upsert response. Silent failures look like success. + 3. Doesn't poll. The HTTP call returns 202 before async indexing finishes, + so the next search call comes back empty and looks like a query bug. + +This script does all three correctly: + + 1. Bulk-upserts in batches. + 2. Inspects every batch result; aborts loudly on any error. + 3. Polls `documents.search` with a sentinel query until matches appear. + +You provide prepared, schema-conformant JSONL + the index name. Schema +validation belongs upstream; +this script trusts the input and focuses on getting it indexed safely. + +Usage: + + uv run --script ingest.py \\ + --data processed.jsonl \\ + --index articles \\ + --sentinel-field body + +Run `--help` for the full flag list. +""" + +from __future__ import annotations + +import json +import os +import time +from pathlib import Path + +import typer +from pinecone import Pinecone + + +# --------------------------------------------------------------------------- +# Helpers — small functions, each does one thing. +# --------------------------------------------------------------------------- + +def load_jsonl(path: Path) -> list[dict]: + """Read a JSONL file into a list of dicts. Fail loudly on parse errors.""" + docs: list[dict] = [] + for lineno, line in enumerate(path.read_text().splitlines(), start=1): + line = line.strip() + if not line: + continue + try: + docs.append(json.loads(line)) + except json.JSONDecodeError as e: + raise typer.BadParameter(f"{path}:{lineno}: invalid JSON ({e.msg})") + if not docs: + raise typer.BadParameter(f"{path}: file is empty") + return docs + + +def pick_sentinel_token(docs: list[dict], field: str) -> str: + """Pick a token from `docs[*][field]` to use as the readiness-poll query. + + A sentinel just needs to match *something* in the freshly-ingested data. + Scan from the first doc onward and return the first whitespace-split token + we find — first-doc-is-special datasets (cover pages, header rows, test + records with empty bodies) won't make us abort. + """ + for doc in docs: + val = doc.get(field) + if isinstance(val, str) and val.strip(): + return val.strip().split()[0] + sample = ", ".join(sorted(docs[0].keys())) or "(none)" + raise typer.BadParameter( + f"can't auto-pick sentinel: no document has a non-empty string in {field!r} " + f"(scanned all {len(docs)} record(s)). Available fields in doc[0]: {sample}. " + f"Either fix --sentinel-field, or pass --sentinel TEXT explicitly." + ) + + +def upsert_batches( + idx, + namespace: str, + docs: list[dict], + batch_size: int, +) -> int: + """Bulk-upsert in batches; abort on the first failed batch. + + Why we inspect the result every time: + `batch_upsert` returns 202 even when individual documents fail — the + failures are reported in `result.errors` / `result.has_errors`. + """ + upserted = 0 + for start in range(0, len(docs), batch_size): + batch = docs[start:start + batch_size] + t0 = time.time() + result = idx.documents.batch_upsert(namespace=namespace, documents=batch) + elapsed = time.time() - t0 + + has_errors = getattr(result, "has_errors", False) or getattr(result, "failed_batch_count", 0) + if has_errors: + for err in getattr(result, "errors", []) or []: + msg = getattr(err, "error_message", None) or str(err) + typer.secho(f" batch error: {msg}", fg=typer.colors.RED, err=True) + raise typer.Exit(code=1) + + upserted += len(batch) + typer.echo( + f" batch @{start:>6}: {len(batch):>4} docs in {elapsed:>5.2f}s" + f" (total: {upserted}/{len(docs)})" + ) + return upserted + + +def poll_until_searchable( + idx, + namespace: str, + sentinel_field: str, + sentinel_token: str, + deadline_s: int, +) -> tuple[float, int]: + """Poll `documents.search` until the sentinel query returns matches. + + Why this exists: + After `batch_upsert` returns, Pinecone is still building the inverted + index. A search call that arrives during that window comes back empty. + Without this poll, the user sees an empty `documents.search` and + debugs their *query*, never noticing it was an indexing race. + + Returns: + (seconds_elapsed, number_of_probes) + """ + start = time.time() + deadline = start + deadline_s + probes = 0 + while time.time() < deadline: + probes += 1 + resp = idx.documents.search( + namespace=namespace, + top_k=1, + score_by=[{"type": "text", "field": sentinel_field, "query": sentinel_token}], + include_fields=[], # required on every search; [] = lightest payload + ) + if resp.matches: + return time.time() - start, probes + time.sleep(5) + + raise typer.Exit(code=1) + + +def resolve_index_with_retry(pc, name: str, *, deadline_s: int = 60): + """Resolve `pc.preview.index(name=...)`, retrying briefly during data-plane warmup. + + """ + deadline = time.time() + deadline_s + delay = 2.0 + last_exc = None + while time.time() < deadline: + try: + return pc.preview.index(name=name) + except Exception as exc: + last_exc = exc + time.sleep(delay) + delay = min(delay * 1.5, 8.0) + raise typer.Exit( + f"Could not resolve index '{name}' within {deadline_s}s " + f"(last error: {type(last_exc).__name__}: {last_exc}). " + f"Check the index exists, the API key has access to it, and that the " + f"data-plane host has finished provisioning (control-plane `status.ready: True` " + f"can lag the data plane by a few seconds)." + ) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +app = typer.Typer( + add_completion=False, + help="Ingest a JSONL file into a Pinecone FTS index, safely.", + rich_markup_mode="rich", +) + + +@app.command() +def main( + data: Path = typer.Option( + ..., "--data", "-d", + exists=True, dir_okay=False, readable=True, + help="Path to JSONL of prepared, schema-conformant documents (one per line).", + ), + index: str = typer.Option( + ..., "--index", "-i", + help="Pinecone index name.", + ), + sentinel_field: str = typer.Option( + ..., "--sentinel-field", "-f", + help="An FTS-enabled field on the index. Used for the readiness-poll query. " + "If you don't know which to use, pick the longest free-text field on your schema.", + ), + namespace: str = typer.Option( + "__default__", "--namespace", "-n", + help="Index namespace.", + ), + batch_size: int = typer.Option( + 100, "--batch-size", "-b", min=1, max=200, + help="Documents per batch_upsert call. Reduce if your dense vectors are large " + "(e.g. 50 for dim=3072) and you hit payload-size errors.", + ), + poll_deadline: int = typer.Option( + 300, "--poll-deadline", min=10, max=3600, + help="Seconds to wait for docs to become searchable before giving up.", + ), + sentinel: str | None = typer.Option( + None, "--sentinel", "-s", + help="Token used for the readiness-poll query. " + "Default: first word of doc[0][sentinel-field].", + ), +): + """Bulk-ingest prepared documents into a Pinecone FTS index. + + [bold]Pipeline[/bold] + + 1. Load JSONL. + 2. `batch_upsert` in batches; abort on any batch error. + 3. Poll `documents.search` with a sentinel query until matches appear. + 4. Report timings. + + [bold]Required[/bold]: PINECONE_API_KEY in the environment, an existing + index named [bold]--index[/bold], and prepared JSONL at [bold]--data[/bold]. + """ + if not os.environ.get("PINECONE_API_KEY"): + raise typer.Exit("PINECONE_API_KEY not set in environment.") + + typer.echo(f"Loading {data} ...") + docs = load_jsonl(data) + typer.echo(f"Loaded {len(docs)} document(s).") + + if sentinel is None: + sentinel = pick_sentinel_token(docs, sentinel_field) + typer.echo(f"Sentinel: {sentinel_field}={sentinel!r}") + + pc = Pinecone(source_tag="pinecone_skills:full_text_search_ingest") # reads PINECONE_API_KEY + idx = resolve_index_with_retry(pc, index) + + typer.echo(f"\nUpserting in batches of {batch_size} ...") + t_upsert_start = time.time() + upserted = upsert_batches(idx, namespace, docs, batch_size) + upsert_seconds = time.time() - t_upsert_start + typer.echo(f"\nUpsert complete: {upserted} doc(s) in {upsert_seconds:.1f}s.") + + typer.echo(f"\nPolling for searchability (deadline {poll_deadline}s) ...") + try: + poll_seconds, probes = poll_until_searchable( + idx, namespace, sentinel_field, sentinel, poll_deadline, + ) + except typer.Exit: + typer.secho( + f"\nDocs not searchable within {poll_deadline}s. " + f"Sentinel: {sentinel_field}={sentinel!r}. " + f"Possible causes: sentinel field isn't FTS-enabled on this index; " + f"the upserts succeeded structurally but the documents themselves were " + f"rejected by the inverted-index builder; the deadline is too tight.", + fg=typer.colors.RED, err=True, + ) + raise + + typer.echo(f"Searchable after {poll_seconds:.1f}s ({probes} probe(s)).") + typer.echo(f"\nDone — total {upsert_seconds + poll_seconds:.1f}s.") + + +if __name__ == "__main__": + app() \ No newline at end of file diff --git a/skills/pinecone-help/SKILL.md b/skills/pinecone-help/SKILL.md new file mode 100644 index 0000000..38234bf --- /dev/null +++ b/skills/pinecone-help/SKILL.md @@ -0,0 +1,64 @@ +--- +name: pinecone-help +description: Overview of all available Pinecone skills and what a user needs to get started. Invoke when a user asks what skills are available, how to get started with Pinecone, or what they need to set up before using any Pinecone skill. +--- + +# Pinecone Skills — Help & Overview + +Pinecone is the leading vector database for building accurate and performant AI applications at scale in production. It's useful for building semantic search, retrieval augmented generation, recommendation systems, and agentic applications. + +Here's everything you need to get started and a summary of all available skills. + +--- + +## What You Need + +### Required +- **Pinecone account** — free to create at https://app.pinecone.io/?sessionType=signup +- **API key** — create one in the Pinecone console after signing up, then either export it in your terminal: + ```bash + export PINECONE_API_KEY="your-key" + ``` + Or add it to a `.env` file if your IDE doesn't inherit shell variables: `PINECONE_API_KEY=your-key` + +### Optional (unlock more capabilities) + +| Tool | What it enables | Install | +|---|---|---| +| **Pinecone MCP server** | Use Pinecone directly inside your AI agent/IDE without writing code | [Setup guide](https://docs.pinecone.io/guides/operations/mcp-server#tools) | +| **Pinecone CLI (`pc`)** | Manage all index types from the terminal, batch operations, backups, CI/CD | `brew tap pinecone-io/tap && brew install pinecone-io/tap/pinecone` | +| **uv** | Run the packaged Python scripts included in these skills | [Install uv](https://docs.astral.sh/uv/getting-started/installation/) | + +--- + +## Available Skills + +| Skill | What it does | +|---|---| +| `pinecone-quickstart` | Step-by-step onboarding — create an index, upload data, and run your first search | +| `pinecone-query` | Search integrated indexes using natural language text via the Pinecone MCP | +| `pinecone-cli` | Use the Pinecone CLI (`pc`) for terminal-based index and vector management | +| `pinecone-assistant` | Create, manage, and chat with Pinecone Assistants for document Q&A with citations | +| `pinecone-mcp` | Reference for all Pinecone MCP server tools and their parameters | +| `pinecone-full-text-search` | Build a full-text-search index — schema design, safe bulk ingestion, and query construction (`text` / `query_string` / dense / sparse scoring with text-match and metadata filters). **Preview API (`2026-01.alpha`); requires `pinecone` Python SDK ≥ 9.0.** | +| `pinecone-docs` | Curated links to official Pinecone documentation, organized by topic | + +--- + +## Which skill should I use? + +**Just getting started?** → `pinecone-quickstart` + +**Want to search an index you already have?** +- Integrated index (built-in embedding model) → `pinecone-query` (uses MCP) +- Any other index type → `pinecone-cli` + +**Working with documents and Q&A?** → `pinecone-assistant` + +**Building a full-text search index (BM25-style keyword/phrase matching, optionally combined with dense or sparse vectors)?** → `pinecone-full-text-search` (preview API, needs `pinecone` Python SDK ≥ 9.0) + +**Need to manage indexes, bulk upload vectors, or automate workflows?** → `pinecone-cli` + +**Looking up API parameters or SDK usage?** → `pinecone-docs` + +**Need to understand what MCP tools are available?** → `pinecone-mcp` diff --git a/skills/pinecone-mcp/SKILL.md b/skills/pinecone-mcp/SKILL.md new file mode 100644 index 0000000..5fbd7ae --- /dev/null +++ b/skills/pinecone-mcp/SKILL.md @@ -0,0 +1,106 @@ +--- +name: pinecone-mcp +description: Reference for the Pinecone MCP server tools. Documents all available tools - list-indexes, describe-index, describe-index-stats, create-index-for-model, upsert-records, search-records, cascading-search, and rerank-documents. Use when an agent needs to understand what Pinecone MCP tools are available, how to use them, or what parameters they accept. +--- + +# Pinecone MCP Tools Reference + +The Pinecone MCP server exposes the following tools to AI agents and IDEs. For setup and installation instructions, see the [MCP server guide](https://docs.pinecone.io/guides/operations/mcp-server#tools). + +> **Key Limitation:** The Pinecone MCP only supports **integrated indexes** — indexes created with a built-in Pinecone embedding model. It does not work with standard indexes using external embedding models. For those, use the Pinecone CLI. + +--- + +## `list-indexes` + +List all indexes in the current Pinecone project. + +--- + +## `describe-index` + +Get configuration details for a specific index — cloud, region, dimension, metric, embedding model, field map, and status. + +**Parameters:** +- `name` (required) — Index name + +--- + +## `describe-index-stats` + +Get statistics for an index including total record count and per-namespace breakdown. + +**Parameters:** +- `name` (required) — Index name + +--- + +## `create-index-for-model` + +Create a new serverless index with an integrated embedding model. Pinecone handles embedding automatically — no external model needed. + +**Parameters:** +- `name` (required) — Index name +- `cloud` (required) — `aws`, `gcp`, or `azure` +- `region` (required) — Cloud region (e.g. `us-east-1`) +- `embed.model` (required) — Embedding model: `llama-text-embed-v2`, `multilingual-e5-large`, or `pinecone-sparse-english-v0` +- `embed.fieldMap.text` (required) — The record field that contains text to embed (e.g. `chunk_text`) + +--- + +## `upsert-records` + +Insert or update records in an integrated index. Records are automatically embedded using the index's configured model. + +**Parameters:** +- `name` (required) — Index name +- `namespace` (required) — Namespace to upsert into +- `records` (required) — Array of records. Each record must have an `id` or `_id` field and contain the text field specified in the index's `fieldMap`. Do not nest fields under `metadata` — put them directly on the record. + +**Example record:** +```json +{ "_id": "rec1", "chunk_text": "The Eiffel Tower was built in 1889.", "category": "architecture" } +``` + +--- + +## `search-records` + +Semantic text search against an integrated index. Pass plain text — the MCP embeds the query automatically using the index's model. + +**Parameters:** +- `name` (required) — Index name +- `namespace` (required) — Namespace to search +- `query.inputs.text` (required) — The text query +- `query.topK` (required) — Number of results to return +- `query.filter` (optional) — Metadata filter using MongoDB-style operators (`$eq`, `$ne`, `$in`, `$gt`, `$gte`, `$lt`, `$lte`) +- `rerank.model` (optional) — Reranking model: `bge-reranker-v2-m3`, `cohere-rerank-3.5`, or `pinecone-rerank-v0` +- `rerank.rankFields` (optional) — Fields to rerank on (e.g. `["chunk_text"]`) +- `rerank.topN` (optional) — Number of results to return after reranking + +--- + +## `cascading-search` + +Search across multiple indexes simultaneously, then deduplicate and rerank results into a single ranked list. + +**Parameters:** +- `indexes` (required) — Array of `{ name, namespace }` objects to search across +- `query.inputs.text` (required) — The text query +- `query.topK` (required) — Number of results to retrieve per index before reranking +- `rerank.model` (required) — Reranking model: `bge-reranker-v2-m3`, `cohere-rerank-3.5`, or `pinecone-rerank-v0` +- `rerank.rankFields` (required) — Fields to rerank on +- `rerank.topN` (optional) — Final number of results to return after reranking + +--- + +## `rerank-documents` + +Rerank a set of documents or records against a query without performing a vector search first. + +**Parameters:** +- `model` (required) — `bge-reranker-v2-m3`, `cohere-rerank-3.5`, or `pinecone-rerank-v0` +- `query` (required) — The query to rerank against +- `documents` (required) — Array of strings or records to rerank +- `options.topN` (required) — Number of results to return +- `options.rankFields` (optional) — If documents are records, the field(s) to rerank on diff --git a/skills/pinecone-query/SKILL.md b/skills/pinecone-query/SKILL.md new file mode 100644 index 0000000..f73abff --- /dev/null +++ b/skills/pinecone-query/SKILL.md @@ -0,0 +1,84 @@ +--- +name: pinecone-query +description: Query integrated indexes using text with Pinecone MCP. IMPORTANT - This skill ONLY works with integrated indexes (indexes with built-in Pinecone embedding models like multilingual-e5-large). For standard indexes or advanced vector operations, use the CLI skill instead. Requires PINECONE_API_KEY environment variable and Pinecone MCP server to be configured. +argument-hint: query [q] index [indexName] namespace [ns] topK [k] reranker [rerankModel] +--- + +# Pinecone Query Skill + +Search for records in Pinecone integrated indexes using natural language text queries via the Pinecone MCP server. + +## What is this skill for? + +This skill provides a simple way to query **integrated indexes** (indexes with built-in Pinecone embedding models) using text queries. The MCP server automatically converts your text into embeddings and searches the index. + +### Prerequisites + +**Required:** +1. ✅ **Pinecone MCP server must be configured** - Check if MCP tools are available +2. ✅ **PINECONE_API_KEY environment variable must be set** - Get a free API key at https://app.pinecone.io/?sessionType=signup +3. ✅ **Index must be an integrated index** - Uses Pinecone embedding models (e.g., multilingual-e5-large, llama-text-embed-v2, pinecone-sparse-english-v0) + +### When NOT to use this skill + +**Use the CLI skill instead if:** +- ❌ Your index is a standard index (no integrated embedding model) +- ❌ You need to query with custom vector values (not text) +- ❌ You need advanced vector operations (fetch by ID, list vectors, bulk operations) +- ❌ Your index uses third-party embedding models (OpenAI, HuggingFace, Cohere) + +**MCP Limitation**: The Pinecone MCP currently only supports integrated indexes. For all other use cases, use the Pinecone CLI skill. + +## How it works + +Utilize Pinecone MCP's `search-records` tool to search for records within a specified Pinecone integrated index using a text query. + +## Workflow + +**IMPORTANT: Before proceeding, verify the Pinecone MCP tools are available.** If MCP tools are not accessible: +- Inform the user that the Pinecone MCP server needs to be configured +- Check if `PINECONE_API_KEY` environment variable is set +- Direct them to the MCP setup documentation or the `pinecone-help` skill + +1. Parse the user's input for: + - `query` (required): The text to search for. + - `index` (required): The name of the Pinecone index to search. + - `namespace` (optional): The namespace within the index. + - `reranker` (optional): The reranking model to use for improved relevance. + +2. If the user omits required arguments: + - If only the index name is provided, use the `describe-index` tool to retrieve available namespaces and ask the user to choose. + - If only a query is provided, use `list-indexes` to get available indexes, ask the user to pick one, then use `describe-index` for namespaces if needed. + +3. Call the `search-records` tool with the gathered arguments to perform the search. + +4. Format and display the returned results in a clear, readable table including field highlights (such as ID, score, and relevant metadata). + +--- + +## Troubleshooting + +**`PINECONE_API_KEY` is required.** Get a free key at https://app.pinecone.io/?sessionType=signup + +If you get an access error, the key is likely missing. Ask the user to set it and restart their IDE or agent session: +- Terminal: `export PINECONE_API_KEY="your-key"` +- IDE without shell inheritance: add `PINECONE_API_KEY=your-key` to a `.env` file + +**IMPORTANT** At the moment, the /query command can only be used with integrated indexes, which use hosted Pinecone embedding models to embed and search for data. +If a user attempts to query an index that uses a third party API model such as OpenAI, or HuggingFace embedding models, remind them that this capability is not available yet +with the Pinecone MCP server. + +- If required arguments are missing, prompt the user to supply them, using Pinecone MCP tools as needed (e.g., `list-indexes`, `describe-index`). +- Guide the user interactively through argument selection until the search can be completed. +- If an invalid value is provided for any argument (e.g., nonexistent index or namespace), surface the error and suggest valid options. + +## Tools Reference + +- `search-records`: Search records in a given index with optional metadata filtering and reranking. +- `list-indexes`: List all available Pinecone indexes. +- `describe-index`: Get index configuration and namespaces. +- `describe-index-stats`: Get stats including record counts and namespaces. +- `rerank-documents`: Rerank returned documents using a specified reranking model. +- Ask the user interactively to clarify missing information when needed. + +--- diff --git a/skills/pinecone-quickstart/SKILL.md b/skills/pinecone-quickstart/SKILL.md new file mode 100644 index 0000000..139cb4c --- /dev/null +++ b/skills/pinecone-quickstart/SKILL.md @@ -0,0 +1,230 @@ +--- +name: pinecone-quickstart +description: Interactive Pinecone quickstart for new developers. Choose between two paths - Database (create an integrated index, upsert data, and query using Pinecone MCP + Python) or Assistant (create a Pinecone Assistant for document Q&A). Use when a user wants to get started with Pinecone for the first time or wants a guided tour of Pinecone's tools. +--- + +# Pinecone Quickstart + +Welcome! This skill walks you through your first Pinecone experience using the tools available to you. In this quickstart, +you will learn how to do a simple form of semantic search over some example data. + +## Prerequisites + +Before starting either path, verify the API key works by calling `list-indexes` via the Pinecone MCP. If it succeeds, proceed. If it fails, ask the user to set their key: + +- Terminal: `export PINECONE_API_KEY="your-key"` +- Or create a `.env` file in the project root: `PINECONE_API_KEY=your-key` + +Then retry `list-indexes` to confirm. + +## Step 0: Choose Your Path + +Ask the user which path they want: + +- **Database** – Build a vector search index. Best for developers who want to store and search embeddings. Uses the Pinecone MCP + a Python upsert script. +- **Assistant** – Build a document Q&A assistant. Best for users who want to upload files and ask questions with cited answers. No code required. + +--- + +## Path A: Database Quickstart + +For each step, explain to the user what will happen. An overview is here: + +1. Check if MCP is set +2. Create an integrated index with MCP +3. Upsert sample data using the bundled script (9 sentences across productivity, health, and nature themes) +4. Run a semantic search query and explore further queries +5. Optionally try reranking +6. Offer the complete standalone script + +### Step 1 – Verify MCP is Available + +The prerequisite check already called `list-indexes`. If it succeeded, the MCP is working — proceed to Step 2. + +If it failed because MCP tools were unavailable (not an auth error): +- Tell the user the MCP server needs to be configured +- Point them to: https://docs.pinecone.io/reference/tools/mcp + +### Step 2 – Create an Integrated Index + +Use the MCP `create-index-for-model` tool to create a serverless index with integrated embeddings: + +``` +name: quickstart-skills +cloud: aws +region: us-east-1 +embed: + model: llama-text-embed-v2 + fieldMap: + text: chunk_text +``` + +**Explain to the user what's happening:** +- An *integrated index* uses a built-in Pinecone embedding model (`llama-text-embed-v2`) +- This means you send plain text and Pinecone handles the embedding automatically +- The `field_map` tells Pinecone which field in your records contains the text to embed + +Wait for the index to become ready before proceeding. Waiting a few seconds is sufficient. + +### Step 3 – Upsert Sample Data + +Run the bundled upsert script to seed the index with sample records. + +If `PINECONE_API_KEY` is set in the environment: +```bash +uv run scripts/upsert.py --index quickstart-skills +``` + +If using a `.env` file: +```bash +uv run --env-file .env scripts/upsert.py --index quickstart-skills +``` + +**Explain to the user what's happening:** +- The script uploads 9 sample records across three themes: **productivity** (getting work done), **health** (feeling unwell), and **nature** (outdoors/wildlife) +- The dataset is intentionally varied so semantic search can show its value — the queries below use completely different words than the records, but the right ones still surface +- Each record has an `_id`, a `chunk_text` field (the text that gets embedded), and a `category` field +- This is the same structure you'd use for your own data — just replace the records + +### Step 4 – Query with the MCP + +Use the MCP `search-records` tool to run the first semantic search: + +``` +index: quickstart-skills +namespace: example-namespace +query: + topK: 3 + inputs: + text: "getting things done efficiently" +``` + +Display the results in a clean table: ID, score, and `chunk_text`. + +**Explain to the user what's happening:** +- Notice the query shares no keywords with the records — but it surfaces the productivity sentences +- That's semantic search: it finds meaning, not just matching words +- You sent plain text — Pinecone embedded the query using the same model as the index + +**Offer to explore further:** Ask the user if they'd like to try another query to see the effect more clearly: +- Option A: `"feeling under the weather"` — should surface the health records +- Option B: `"wildlife spotting outside"` — should surface the nature records +- Option C: No thanks, move on + +Run whichever query they choose and display the results the same way. If they want to try both, do both. After each result, point out which theme surfaced and why. + +If they decline or are done exploring, proceed to Step 5 or offer to skip ahead to the complete script. + +### Step 5 – Try Reranking (Optional) + +Ask the user if they want to try reranking. + +If yes, use `search-records` again with reranking enabled: + +``` +rerank: + model: bge-reranker-v2-m3 + rankFields: [chunk_text] + topN: 3 +``` + +**Explain**: Reranking runs a second-pass model over the results to improve relevance ordering. + +### Step 6 – Wrap Up + +Congratulate the user on completing the quickstart. Ask if they'd like a standalone Python script that does everything in one go — create index, upsert, query, and rerank. + +If yes, copy it to their working directory: + +```bash +cp scripts/quickstart_complete.py ./pinecone_quickstart.py +``` + +Tell the user: +- The script is at `./pinecone_quickstart.py` +- Run it with: `uv run pinecone_quickstart.py` +- It uses `uv` inline dependencies — no separate install needed +- They can swap in their own `records` list to build something real + +--- + +## Path B: Assistant Quickstart + +Guide the user through the Pinecone Assistant workflow using the existing assistant skills: + +### Step 1 – Check for Documents + +Before anything else, ask the user if they have files to upload. Pinecone Assistant accepts `.pdf`, `.md`, `.txt`, and `.docx` files — a single file or a folder of files both work. + +**If they have files:** ask for the path and proceed to Step 2. + +**If they don't have files:** offer two options: +- **Generate sample docs** — create a few short markdown files in `./sample-docs/` so they can complete the quickstart right now. Ask what topics they'd like (or default to: a product FAQ, a short how-to guide, and a brief company overview). Write 3 files, each 150–250 words. +- **Come back later** — let them know they can return once they have documents and pick up from Step 2. + +### Step 2 – Create an Assistant + +Invoke `pinecone-assistant` or run (add `--env-file .env` if using a `.env` file): +```bash +uv run ../pinecone-assistant/scripts/create.py --name my-assistant +``` + +Explain: The assistant is a fully managed RAG service — upload documents, ask questions, get cited answers. + +### Step 3 – Upload Documents + +Invoke `pinecone-assistant` or run (add `--env-file .env` if using a `.env` file): +```bash +uv run ../pinecone-assistant/scripts/upload.py --assistant my-assistant --source ./your-docs +``` + +Explain: Pinecone handles chunking, embedding, and indexing automatically — no configuration needed. + +### Step 4 – Chat with the Assistant + +Invoke `pinecone-assistant` or run (add `--env-file .env` if using a `.env` file): +```bash +uv run ../pinecone-assistant/scripts/chat.py --assistant my-assistant --message "What are the main topics in these documents?" +``` + +Explain: Responses include citations with source file and page number. + +### Next Steps for Assistant + +- Invoke `pinecone-assistant` to keep the assistant up to date as documents change +- Use the assistant skill to retrieve raw context snippets for custom workflows +- Every assistant is also an MCP server — see https://docs.pinecone.io/guides/assistant/mcp-server + +--- + +## Troubleshooting + +**`PINECONE_API_KEY` not set** + +Terminal environments: +```bash +export PINECONE_API_KEY="your-key" +``` +IDEs that don't inherit shell variables: create a `.env` file in the project root: +``` +PINECONE_API_KEY=your-key +``` +Then use `uv run --env-file .env` when running scripts. Restart your IDE/agent session after setting. + +**MCP tools not available** +- Verify the Pinecone MCP server is configured in your IDE's MCP settings +- Check that `PINECONE_API_KEY` is set before the MCP server starts + +**Index already exists** +- The upsert script is safe to re-run — it will upsert over existing records +- Or delete and recreate: use `pc index delete -n quickstart-skills` via the CLI + +**`uv` not installed** +See the [uv installation guide](https://docs.astral.sh/uv/getting-started/installation/). + +## Further Reading + +- Quickstart docs: https://docs.pinecone.io/guides/get-started/quickstart +- Integrated indexes: https://docs.pinecone.io/guides/index-data/create-an-index +- Python SDK: https://docs.pinecone.io/guides/get-started/python-sdk +- MCP server: https://docs.pinecone.io/reference/tools/mcp diff --git a/skills/pinecone-quickstart/scripts/quickstart_complete.py b/skills/pinecone-quickstart/scripts/quickstart_complete.py new file mode 100644 index 0000000..c07643a --- /dev/null +++ b/skills/pinecone-quickstart/scripts/quickstart_complete.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# /// script +# dependencies = [ +# "pinecone>=8.0.0", +# ] +# /// + +import os +from pinecone import Pinecone + +api_key = os.environ.get("PINECONE_API_KEY") +if not api_key: + raise ValueError("PINECONE_API_KEY environment variable not set") + +pc = Pinecone(api_key=api_key, source_tag="pinecone_skills:index_quickstart") + +# 1. Create a serverless index with an integrated embedding model +index_name = "quickstart" + +if not pc.has_index(index_name): + pc.create_index_for_model( + name=index_name, + cloud="aws", + region="us-east-1", + embed={ + "model": "llama-text-embed-v2", + "field_map": {"text": "chunk_text"} + } + ) + +# 2. Upsert records +# Three distinct themes — notice the queries below use different words than the records. +# That's semantic search: finding meaning, not just matching keywords. +records = [ + # Health / feeling unwell + {"_id": "rec1", "chunk_text": "I've been sneezing all day and my nose won't stop running.", "category": "health"}, + {"_id": "rec2", "chunk_text": "She stayed home with a pounding headache and a low-grade fever.", "category": "health"}, + {"_id": "rec3", "chunk_text": "He felt completely drained after waking up with a sore throat and chills.", "category": "health"}, + # Productivity / work + {"_id": "rec4", "chunk_text": "She blocked off two hours in the morning to focus without interruptions.", "category": "productivity"}, + {"_id": "rec5", "chunk_text": "He finished all his tasks ahead of schedule by prioritizing the hardest ones first.", "category": "productivity"}, + {"_id": "rec6", "chunk_text": "Turning off notifications helped her get into a deep flow state.", "category": "productivity"}, + # Outdoors / nature + {"_id": "rec7", "chunk_text": "A red fox darted across the trail and disappeared into the underbrush.", "category": "nature"}, + {"_id": "rec8", "chunk_text": "The hikers paused to watch a bald eagle circle lazily over the valley.", "category": "nature"}, + {"_id": "rec9", "chunk_text": "Fireflies lit up the meadow as the sun dipped below the treeline.", "category": "nature"}, +] + +dense_index = pc.Index(index_name) +dense_index.upsert_records("example-namespace", records) + +# 3. Search records +# The query uses different words than the records — semantic search finds meaning, not keywords. +query = "feeling ill and run down" + +results = dense_index.search( + namespace="example-namespace", + query={"top_k": 3, "inputs": {"text": query}} +) + +print("Search results:") +for hit in results["result"]["hits"]: + print(f" id: {hit['_id']} | score: {round(hit['_score'], 2)} | text: {hit['fields']['chunk_text']}") + +# 4. Search with reranking +reranked_results = dense_index.search( + namespace="example-namespace", + query={"top_k": 3, "inputs": {"text": query}}, + rerank={"model": "bge-reranker-v2-m3", "top_n": 3, "rank_fields": ["chunk_text"]} +) + +print("\nReranked results:") +for hit in reranked_results["result"]["hits"]: + print(f" id: {hit['_id']} | score: {round(hit['_score'], 2)} | text: {hit['fields']['chunk_text']}") diff --git a/skills/pinecone-quickstart/scripts/upsert.py b/skills/pinecone-quickstart/scripts/upsert.py new file mode 100644 index 0000000..1642bef --- /dev/null +++ b/skills/pinecone-quickstart/scripts/upsert.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# /// script +# dependencies = [ +# "pinecone>=8.0.0", +# "typer>=0.15.0", +# ] +# /// + +import os +import typer +from pinecone import Pinecone + +app = typer.Typer() + +@app.command() +def main( + index: str = typer.Option(..., "--index", help="Name of the Pinecone index to upsert into"), + namespace: str = typer.Option("example-namespace", "--namespace", help="Namespace to upsert into"), +): + api_key = os.environ.get("PINECONE_API_KEY") + if not api_key: + typer.echo("Error: PINECONE_API_KEY environment variable not set", err=True) + raise typer.Exit(1) + + pc = Pinecone(api_key=api_key, source_tag="pinecone_skills:upsert") + + records = [ + # Health / feeling unwell + {"_id": "rec1", "chunk_text": "I've been sneezing all day and my nose won't stop running.", "category": "health"}, + {"_id": "rec2", "chunk_text": "She stayed home with a pounding headache and a low-grade fever.", "category": "health"}, + {"_id": "rec3", "chunk_text": "He felt completely drained after waking up with a sore throat and chills.", "category": "health"}, + # Productivity / work + {"_id": "rec4", "chunk_text": "She blocked off two hours in the morning to focus without interruptions.", "category": "productivity"}, + {"_id": "rec5", "chunk_text": "He finished all his tasks ahead of schedule by prioritizing the hardest ones first.", "category": "productivity"}, + {"_id": "rec6", "chunk_text": "Turning off notifications helped her get into a deep flow state.", "category": "productivity"}, + # Outdoors / nature + {"_id": "rec7", "chunk_text": "A red fox darted across the trail and disappeared into the underbrush.", "category": "nature"}, + {"_id": "rec8", "chunk_text": "The hikers paused to watch a bald eagle circle lazily over the valley.", "category": "nature"}, + {"_id": "rec9", "chunk_text": "Fireflies lit up the meadow as the sun dipped below the treeline.", "category": "nature"}, + ] + + idx = pc.Index(index) + idx.upsert_records(namespace, records) + typer.echo(f"Upserted {len(records)} records into '{index}' (namespace: '{namespace}')") + +if __name__ == "__main__": + app() From d6db8d5402d2581dbdaee056e6cf441c419fc5cd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 13 May 2026 21:04:51 +0000 Subject: [PATCH 2/2] contextualize: adapt synced skills for Cursor plugin Co-authored-by: Cursor --- skills/assistant/.gitkeep | 0 skills/assistant/SKILL.md | 4 +- skills/assistant/references/.gitkeep | 0 skills/assistant/scripts/.gitkeep | 0 skills/assistant/scripts/chat.py | 2 +- skills/assistant/scripts/context.py | 2 +- skills/assistant/scripts/create.py | 4 +- skills/assistant/scripts/list.py | 2 +- skills/assistant/scripts/sync.py | 2 +- skills/assistant/scripts/upload.py | 2 +- skills/cli/.gitkeep | 0 skills/cli/SKILL.md | 2 + skills/cli/references/.gitkeep | 0 skills/docs/.gitkeep | 0 skills/docs/SKILL.md | 3 + skills/docs/references/.gitkeep | 0 .../SKILL.md | 10 +- .../references/ingestion.md | 0 .../references/onboarding-walkthrough.md | 0 .../references/querying.md | 0 .../references/schema-design.md | 0 .../scripts/ingest.py | 0 skills/help/.gitkeep | 0 skills/help/SKILL.md | 9 +- skills/mcp/.gitkeep | 0 skills/mcp/SKILL.md | 2 + skills/pinecone-assistant/SKILL.md | 78 ---- skills/pinecone-assistant/references/chat.md | 33 -- .../pinecone-assistant/references/context.md | 39 -- .../pinecone-assistant/references/create.md | 43 --- skills/pinecone-assistant/references/list.md | 31 -- skills/pinecone-assistant/references/sync.md | 51 --- .../pinecone-assistant/references/upload.md | 49 --- skills/pinecone-assistant/scripts/chat.py | 141 ------- skills/pinecone-assistant/scripts/context.py | 146 -------- skills/pinecone-assistant/scripts/create.py | 126 ------- skills/pinecone-assistant/scripts/list.py | 200 ---------- skills/pinecone-assistant/scripts/sync.py | 354 ------------------ skills/pinecone-assistant/scripts/upload.py | 222 ----------- skills/pinecone-cli/SKILL.md | 156 -------- .../references/command-reference.md | 237 ------------ .../references/troubleshooting.md | 136 ------- skills/pinecone-docs/SKILL.md | 86 ----- .../pinecone-docs/references/data-formats.md | 81 ---- skills/pinecone-help/SKILL.md | 64 ---- skills/pinecone-mcp/SKILL.md | 106 ------ skills/pinecone-query/SKILL.md | 84 ----- skills/pinecone-quickstart/SKILL.md | 230 ------------ .../scripts/quickstart_complete.py | 74 ---- skills/pinecone-quickstart/scripts/upsert.py | 47 --- skills/query/.gitkeep | 0 skills/query/SKILL.md | 8 +- skills/quickstart/.gitkeep | 0 skills/quickstart/SKILL.md | 14 +- skills/quickstart/scripts/.gitkeep | 0 .../quickstart/scripts/quickstart_complete.py | 2 +- skills/quickstart/scripts/upsert.py | 2 +- 57 files changed, 47 insertions(+), 2837 deletions(-) delete mode 100644 skills/assistant/.gitkeep delete mode 100644 skills/assistant/references/.gitkeep delete mode 100644 skills/assistant/scripts/.gitkeep delete mode 100644 skills/cli/.gitkeep delete mode 100644 skills/cli/references/.gitkeep delete mode 100644 skills/docs/.gitkeep delete mode 100644 skills/docs/references/.gitkeep rename skills/{pinecone-full-text-search => full-text-search}/SKILL.md (98%) rename skills/{pinecone-full-text-search => full-text-search}/references/ingestion.md (100%) rename skills/{pinecone-full-text-search => full-text-search}/references/onboarding-walkthrough.md (100%) rename skills/{pinecone-full-text-search => full-text-search}/references/querying.md (100%) rename skills/{pinecone-full-text-search => full-text-search}/references/schema-design.md (100%) rename skills/{pinecone-full-text-search => full-text-search}/scripts/ingest.py (100%) delete mode 100644 skills/help/.gitkeep delete mode 100644 skills/mcp/.gitkeep delete mode 100644 skills/pinecone-assistant/SKILL.md delete mode 100644 skills/pinecone-assistant/references/chat.md delete mode 100644 skills/pinecone-assistant/references/context.md delete mode 100644 skills/pinecone-assistant/references/create.md delete mode 100644 skills/pinecone-assistant/references/list.md delete mode 100644 skills/pinecone-assistant/references/sync.md delete mode 100644 skills/pinecone-assistant/references/upload.md delete mode 100755 skills/pinecone-assistant/scripts/chat.py delete mode 100755 skills/pinecone-assistant/scripts/context.py delete mode 100755 skills/pinecone-assistant/scripts/create.py delete mode 100755 skills/pinecone-assistant/scripts/list.py delete mode 100644 skills/pinecone-assistant/scripts/sync.py delete mode 100755 skills/pinecone-assistant/scripts/upload.py delete mode 100644 skills/pinecone-cli/SKILL.md delete mode 100644 skills/pinecone-cli/references/command-reference.md delete mode 100644 skills/pinecone-cli/references/troubleshooting.md delete mode 100644 skills/pinecone-docs/SKILL.md delete mode 100644 skills/pinecone-docs/references/data-formats.md delete mode 100644 skills/pinecone-help/SKILL.md delete mode 100644 skills/pinecone-mcp/SKILL.md delete mode 100644 skills/pinecone-query/SKILL.md delete mode 100644 skills/pinecone-quickstart/SKILL.md delete mode 100644 skills/pinecone-quickstart/scripts/quickstart_complete.py delete mode 100644 skills/pinecone-quickstart/scripts/upsert.py delete mode 100644 skills/query/.gitkeep delete mode 100644 skills/quickstart/.gitkeep delete mode 100644 skills/quickstart/scripts/.gitkeep diff --git a/skills/assistant/.gitkeep b/skills/assistant/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/skills/assistant/SKILL.md b/skills/assistant/SKILL.md index 7db3a6f..aa9721d 100644 --- a/skills/assistant/SKILL.md +++ b/skills/assistant/SKILL.md @@ -3,6 +3,8 @@ name: assistant description: Create, manage, and chat with Pinecone Assistants for document Q&A with citations. Handles all assistant operations - create, upload, sync, chat, context retrieval, and list. Recognizes natural language like "create an assistant from my docs", "ask my assistant about X", or "upload my docs to Pinecone". --- +> Packaged as a Cursor plugin skill ([pinecone-io/pinecone-cursor-plugin](https://github.com/pinecone-io/pinecone-cursor-plugin)). + # Pinecone Assistant Pinecone Assistant is a fully managed RAG service. Upload documents, ask questions, get cited answers. No embedding pipelines or infrastructure required. @@ -73,6 +75,6 @@ Handle chained requests naturally. Example: ## Prerequisites -- `PINECONE_API_KEY` must be available — add it to a `.env` file at your workspace root (the bundled MCP config loads it via Cursor's `envFile` field) and run scripts with `uv run --env-file .env scripts/...`. For terminal-only scripts, `export PINECONE_API_KEY="your-key"` also works. +- `PINECONE_API_KEY` must be available — add `PINECONE_API_KEY=your-key` to a `.env` file at your workspace root (the bundled MCP configuration for this Cursor plugin loads it via Cursor's `envFile` setting), and run scripts with `uv run --env-file .env scripts/...`. For terminal-only workflows, `export PINECONE_API_KEY="your-key"` also works - `uv` must be installed — [install uv](https://docs.astral.sh/uv/getting-started/installation/) - Get a free API key at: https://app.pinecone.io/?sessionType=signup diff --git a/skills/assistant/references/.gitkeep b/skills/assistant/references/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/skills/assistant/scripts/.gitkeep b/skills/assistant/scripts/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/skills/assistant/scripts/chat.py b/skills/assistant/scripts/chat.py index 637f858..843d3db 100755 --- a/skills/assistant/scripts/chat.py +++ b/skills/assistant/scripts/chat.py @@ -47,7 +47,7 @@ def main( try: # Initialize Pinecone client - pc = Pinecone(api_key=api_key,source_tag="cursor_plugin:assistant") + pc = Pinecone(api_key=api_key,source_tag="claude_code_plugin:assistant") asst = pc.assistant.Assistant(assistant_name=assistant) # Create message diff --git a/skills/assistant/scripts/context.py b/skills/assistant/scripts/context.py index 438c13f..81ae65c 100755 --- a/skills/assistant/scripts/context.py +++ b/skills/assistant/scripts/context.py @@ -51,7 +51,7 @@ def main( try: # Initialize Pinecone client - pc = Pinecone(api_key=api_key, source_tag="cursor_plugin:assistant") + pc = Pinecone(api_key=api_key, source_tag="claude_code_plugin:assistant") asst = pc.assistant.Assistant(assistant_name=assistant) # Display query diff --git a/skills/assistant/scripts/create.py b/skills/assistant/scripts/create.py index 724aa1c..c135b56 100755 --- a/skills/assistant/scripts/create.py +++ b/skills/assistant/scripts/create.py @@ -69,7 +69,7 @@ def main( try: # Initialize Pinecone client with console.status(f"[bold blue]Creating assistant '{name}'...[/bold blue]"): - pc = Pinecone(api_key=api_key, source_tag="cursor_plugin:assistant") + pc = Pinecone(api_key=api_key, source_tag="claude_code_plugin:assistant") # Create assistant assistant = pc.assistant.create_assistant( @@ -77,7 +77,7 @@ def main( instructions=instructions if instructions else None, region=region, timeout=timeout, - metadata={"agentic-ide-source":"cursor-plugin"} + metadata={"agentic-ide-source":"claude-code-plugin"} ) # Success message diff --git a/skills/assistant/scripts/list.py b/skills/assistant/scripts/list.py index c48c06b..bd8a373 100755 --- a/skills/assistant/scripts/list.py +++ b/skills/assistant/scripts/list.py @@ -49,7 +49,7 @@ def main( try: # Initialize Pinecone client - pc = Pinecone(api_key=api_key, source_tag="cursor_plugin:assistant") + pc = Pinecone(api_key=api_key, source_tag="claude_code_plugin:assistant") # List assistants assistants = pc.assistant.list_assistants() diff --git a/skills/assistant/scripts/sync.py b/skills/assistant/scripts/sync.py index 94c884f..13c98a5 100644 --- a/skills/assistant/scripts/sync.py +++ b/skills/assistant/scripts/sync.py @@ -118,7 +118,7 @@ def main( try: # Initialize Pinecone client - pc = Pinecone(api_key=api_key, source_tag="cursor_plugin:assistant") + pc = Pinecone(api_key=api_key, source_tag="claude_code_plugin:assistant") asst = pc.assistant.Assistant(assistant_name=assistant) console.print(Panel( diff --git a/skills/assistant/scripts/upload.py b/skills/assistant/scripts/upload.py index 57ec966..4484e75 100755 --- a/skills/assistant/scripts/upload.py +++ b/skills/assistant/scripts/upload.py @@ -126,7 +126,7 @@ def main( try: # Initialize Pinecone client - pc = Pinecone(api_key=api_key, source_tag="cursor_plugin:assistant") + pc = Pinecone(api_key=api_key, source_tag="claude_code_plugin:assistant") asst = pc.assistant.Assistant(assistant_name=assistant) # Find files to upload diff --git a/skills/cli/.gitkeep b/skills/cli/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/skills/cli/SKILL.md b/skills/cli/SKILL.md index c7c30e7..f7de1b1 100644 --- a/skills/cli/SKILL.md +++ b/skills/cli/SKILL.md @@ -4,6 +4,8 @@ description: Guide for using the Pinecone CLI (pc) to manage Pinecone resources argument-hint: install | auth | index [op] | vector [op] | backup | namespace --- +> Packaged as a Cursor plugin skill ([pinecone-io/pinecone-cursor-plugin](https://github.com/pinecone-io/pinecone-cursor-plugin)). + # Pinecone CLI (`pc`) Manage Pinecone from the terminal. The CLI is especially valuable for vector operations across **all index types** — something the MCP currently can't do. diff --git a/skills/cli/references/.gitkeep b/skills/cli/references/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/skills/docs/.gitkeep b/skills/docs/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/skills/docs/SKILL.md b/skills/docs/SKILL.md index 63d65f5..e68a381 100644 --- a/skills/docs/SKILL.md +++ b/skills/docs/SKILL.md @@ -3,6 +3,8 @@ name: docs description: Curated documentation reference for developers building with Pinecone. Contains links to official docs organized by topic and data format references. Use when writing Pinecone code, looking up API parameters, or needing the correct format for vectors or records. --- +> Packaged as a Cursor plugin skill ([pinecone-io/pinecone-cursor-plugin](https://github.com/pinecone-io/pinecone-cursor-plugin)). + # Pinecone Developer Reference A curated index of Pinecone documentation. Fetch the relevant page(s) for the task at hand rather than relying on training data. @@ -57,6 +59,7 @@ Use this as a last resort if you cannot find the relevant page below. | Semantic search | https://docs.pinecone.io/guides/search/semantic-search | | Hybrid search | https://docs.pinecone.io/guides/search/hybrid-search | | Lexical search | https://docs.pinecone.io/guides/search/lexical-search | +| Full-text search (preview) — document-schema FTS indexes with `text` / `query_string` / dense / sparse scoring | https://docs.pinecone.io/guides/search/full-text-search | | Metadata filtering — narrow results and speed up searches | https://docs.pinecone.io/guides/search/filter-by-metadata | --- diff --git a/skills/docs/references/.gitkeep b/skills/docs/references/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/skills/pinecone-full-text-search/SKILL.md b/skills/full-text-search/SKILL.md similarity index 98% rename from skills/pinecone-full-text-search/SKILL.md rename to skills/full-text-search/SKILL.md index 2d4b942..e1edd79 100644 --- a/skills/pinecone-full-text-search/SKILL.md +++ b/skills/full-text-search/SKILL.md @@ -1,9 +1,11 @@ --- -name: pinecone-full-text-search +name: full-text-search description: Create, ingest into, and query a Pinecone full-text-search (FTS) index using the preview API (2026-01.alpha, public preview). Use when the user or agent asks to build a text search index on Pinecone, add dense or sparse vector fields, ingest documents, construct score_by clauses (text / query_string / dense_vector / sparse_vector), or compose with text-match filters ($match_phrase / $match_all / $match_any). Ships `scripts/ingest.py` for safe bulk ingestion (batch_upsert + error inspection + readiness polling); query construction is documented inline in this skill — write `documents.search(...)` calls directly, validated against `pc.preview.indexes.describe(...)` output. --- -# pinecone-full-text-search +> Packaged as a Cursor plugin skill ([pinecone-io/pinecone-cursor-plugin](https://github.com/pinecone-io/pinecone-cursor-plugin)). + +# Pinecone full-text search > **Requires `pinecone` Python SDK ≥ 9.0** (`pip install pinecone>=9.0`). The FTS document-schema API lives under `pinecone.preview` and is incomplete or absent in earlier SDK builds. The packaged helper scripts pin `pinecone==9.0.0` via PEP 723 inline metadata; if you're writing your own code against this skill, pin v9 explicitly. The wire API version is `2026-01.alpha`. @@ -108,7 +110,7 @@ You provide a prepared, schema-conformant JSONL file and the index name; the scr **Invocation:** ```bash -uv run --script .claude/skills/pinecone-fts-index/scripts/ingest.py \ +uv run --script scripts/ingest.py \ --data processed.jsonl \ --index \ --sentinel-field @@ -154,7 +156,7 @@ If a batch fails, the script prints every error message and exits non-zero. If t - The user is ingesting from a non-JSONL source (CSV, Parquet, Postgres dump). Convert to JSONL first; the script doesn't parse other formats. - The user explicitly asks you to write the ingestion code from scratch (teaching context). Honor the request and follow the canonical pattern: `documents.batch_upsert` + `result.has_errors` inspection + `documents.search` polling with sentinel and deadline. -The script lives at `.claude/skills/pinecone-fts-index/scripts/ingest.py`. PEP 723 inline-metadata script — `uv run --script` installs `typer` and `pinecone` automatically on first invocation. No setup needed. +The script lives at `scripts/ingest.py` in this skill directory (run the command from this directory). PEP 723 inline-metadata script — `uv run --script` installs `typer` and `pinecone` automatically on first invocation. No setup needed. ## Use cases diff --git a/skills/pinecone-full-text-search/references/ingestion.md b/skills/full-text-search/references/ingestion.md similarity index 100% rename from skills/pinecone-full-text-search/references/ingestion.md rename to skills/full-text-search/references/ingestion.md diff --git a/skills/pinecone-full-text-search/references/onboarding-walkthrough.md b/skills/full-text-search/references/onboarding-walkthrough.md similarity index 100% rename from skills/pinecone-full-text-search/references/onboarding-walkthrough.md rename to skills/full-text-search/references/onboarding-walkthrough.md diff --git a/skills/pinecone-full-text-search/references/querying.md b/skills/full-text-search/references/querying.md similarity index 100% rename from skills/pinecone-full-text-search/references/querying.md rename to skills/full-text-search/references/querying.md diff --git a/skills/pinecone-full-text-search/references/schema-design.md b/skills/full-text-search/references/schema-design.md similarity index 100% rename from skills/pinecone-full-text-search/references/schema-design.md rename to skills/full-text-search/references/schema-design.md diff --git a/skills/pinecone-full-text-search/scripts/ingest.py b/skills/full-text-search/scripts/ingest.py similarity index 100% rename from skills/pinecone-full-text-search/scripts/ingest.py rename to skills/full-text-search/scripts/ingest.py diff --git a/skills/help/.gitkeep b/skills/help/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/skills/help/SKILL.md b/skills/help/SKILL.md index f726b63..3c62f8b 100644 --- a/skills/help/SKILL.md +++ b/skills/help/SKILL.md @@ -3,6 +3,8 @@ name: help description: Overview of all available Pinecone skills and what a user needs to get started. Invoke when a user asks what skills are available, how to get started with Pinecone, or what they need to set up before using any Pinecone skill. --- +> Packaged as a Cursor plugin skill ([pinecone-io/pinecone-cursor-plugin](https://github.com/pinecone-io/pinecone-cursor-plugin)). + # Pinecone Skills — Help & Overview Pinecone is the leading vector database for building accurate and performant AI applications at scale in production. It's useful for building semantic search, retrieval augmented generation, recommendation systems, and agentic applications. @@ -15,7 +17,7 @@ Here's everything you need to get started and a summary of all available skills. ### Required - **Pinecone account** — free to create at https://app.pinecone.io/?sessionType=signup -- **API key** — create one in the Pinecone console after signing up, then add it to a `.env` file at your workspace root (the bundled MCP config loads it via Cursor's `envFile` field): +- **API key** — create one in the Pinecone console after signing up, then add it to a `.env` file at your workspace root (the bundled MCP configuration for this Cursor plugin loads it via Cursor's `envFile` setting): ``` PINECONE_API_KEY=your-key ``` @@ -33,6 +35,8 @@ Here's everything you need to get started and a summary of all available skills. ## Available Skills +In Cursor Agent chat, invoke a skill with `/` plus the skill name (for example `/quickstart`, `/assistant`, `/full-text-search`). + | Skill | What it does | |---|---| | `quickstart` | Step-by-step onboarding — create an index, upload data, and run your first search | @@ -40,6 +44,7 @@ Here's everything you need to get started and a summary of all available skills. | `cli` | Use the Pinecone CLI (`pc`) for terminal-based index and vector management | | `assistant` | Create, manage, and chat with Pinecone Assistants for document Q&A with citations | | `mcp` | Reference for all Pinecone MCP server tools and their parameters | +| `full-text-search` | Build a full-text-search index — schema design, safe bulk ingestion, and query construction (`text` / `query_string` / dense / sparse scoring with text-match and metadata filters). **Preview API (`2026-01.alpha`); requires `pinecone` Python SDK ≥ 9.0.** | | `docs` | Curated links to official Pinecone documentation, organized by topic | --- @@ -54,6 +59,8 @@ Here's everything you need to get started and a summary of all available skills. **Working with documents and Q&A?** → `assistant` +**Building a full-text search index (BM25-style keyword/phrase matching, optionally combined with dense or sparse vectors)?** → `full-text-search` (preview API, needs `pinecone` Python SDK ≥ 9.0) + **Need to manage indexes, bulk upload vectors, or automate workflows?** → `cli` **Looking up API parameters or SDK usage?** → `docs` diff --git a/skills/mcp/.gitkeep b/skills/mcp/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/skills/mcp/SKILL.md b/skills/mcp/SKILL.md index 354fcd9..b016e7c 100644 --- a/skills/mcp/SKILL.md +++ b/skills/mcp/SKILL.md @@ -3,6 +3,8 @@ name: mcp description: Reference for the Pinecone MCP server tools. Documents all available tools - list-indexes, describe-index, describe-index-stats, create-index-for-model, upsert-records, search-records, cascading-search, and rerank-documents. Use when an agent needs to understand what Pinecone MCP tools are available, how to use them, or what parameters they accept. --- +> Packaged as a Cursor plugin skill ([pinecone-io/pinecone-cursor-plugin](https://github.com/pinecone-io/pinecone-cursor-plugin)). + # Pinecone MCP Tools Reference The Pinecone MCP server exposes the following tools to AI agents and IDEs. For setup and installation instructions, see the [MCP server guide](https://docs.pinecone.io/guides/operations/mcp-server#tools). diff --git a/skills/pinecone-assistant/SKILL.md b/skills/pinecone-assistant/SKILL.md deleted file mode 100644 index eeb3eb8..0000000 --- a/skills/pinecone-assistant/SKILL.md +++ /dev/null @@ -1,78 +0,0 @@ ---- -name: pinecone-assistant -description: Create, manage, and chat with Pinecone Assistants for document Q&A with citations. Handles all assistant operations - create, upload, sync, chat, context retrieval, and list. Recognizes natural language like "create an assistant from my docs", "ask my assistant about X", or "upload my docs to Pinecone". ---- - -# Pinecone Assistant - -Pinecone Assistant is a fully managed RAG service. Upload documents, ask questions, get cited answers. No embedding pipelines or infrastructure required. - -> All scripts are in `scripts/` relative to this skill directory. -> Run with: `uv run scripts/script_name.py [arguments]` - -## Operations - -| What to do | Script | Key args | -|---|---|---| -| Create an assistant | `scripts/create.py` | `--name` `--instructions` `--region` | -| Upload files | `scripts/upload.py` | `--assistant` `--source` `--patterns` | -| Sync files (incremental) | `scripts/sync.py` | `--assistant` `--source` `--delete-missing` `--dry-run` | -| Chat / ask a question | `scripts/chat.py` | `--assistant` `--message` | -| Get context snippets | `scripts/context.py` | `--assistant` `--query` `--top-k` | -| List assistants | `scripts/list.py` | `--files` `--json` | - -For full workflow details on any operation, read the relevant file in `references/`. - ---- - -## Natural Language Recognition - -Proactively handle these patterns without requiring explicit commands: - -**Create:** "create an assistant", "make an assistant called X", "set up an assistant for my docs" -→ See [references/create.md](references/create.md) - -**Upload:** "upload my docs", "add files to my assistant", "index my documentation" -→ See [references/upload.md](references/upload.md) - -**Sync:** "sync my docs", "update my assistant", "keep assistant in sync", "refresh from ./docs" -→ See [references/sync.md](references/sync.md) - -**Chat:** "ask my assistant about X", "what does my assistant know about X", "chat with X" -→ See [references/chat.md](references/chat.md) - -**Context:** "search my assistant for X", "find context about X" -→ See [references/context.md](references/context.md) - -**List:** "show my assistants", "what assistants do I have" -→ Run `uv run scripts/list.py` - ---- - -## Conversation Memory - -Track the last assistant used within the conversation: -- When a user creates or first uses an assistant, remember its name -- If user says "my assistant", "it", or "the assistant" → use the last one -- Briefly confirm which assistant you're using: "Asking docs-bot..." -- If ambiguous and multiple exist → ask the user to clarify - ---- - -## Multi-Step Requests - -Handle chained requests naturally. Example: - -> "Create an assistant called docs-bot, upload my ./docs folder, and ask what the main features are" - -1. `uv run scripts/create.py --name docs-bot` -2. `uv run scripts/upload.py --assistant docs-bot --source ./docs` -3. `uv run scripts/chat.py --assistant docs-bot --message "what are the main features?"` - ---- - -## Prerequisites - -- `PINECONE_API_KEY` must be available — terminal: `export PINECONE_API_KEY="your-key"`, or add to a `.env` file and run scripts with `uv run --env-file .env scripts/...` -- `uv` must be installed — [install uv](https://docs.astral.sh/uv/getting-started/installation/) -- Get a free API key at: https://app.pinecone.io/?sessionType=signup diff --git a/skills/pinecone-assistant/references/chat.md b/skills/pinecone-assistant/references/chat.md deleted file mode 100644 index d4a984f..0000000 --- a/skills/pinecone-assistant/references/chat.md +++ /dev/null @@ -1,33 +0,0 @@ -# Chat with Assistant - -Send a message to an assistant and receive a cited response. - -## Arguments - -- `--assistant` (required): Assistant name -- `--message` (required): The question or message -- `--stream` (optional flag): Enable streaming for faster perceived response - -## Workflow - -1. Parse arguments. If assistant missing, run `uv run scripts/list.py --json` and ask the user to select. -2. If message missing, prompt user for their question. -3. Execute: - ```bash - uv run scripts/chat.py \ - --assistant "assistant-name" \ - --message "user's question" - ``` -4. Display: - - Assistant's response - - Citations table: citation number, source file, page numbers, position - - Token usage statistics - -**Note:** File URLs in citations are temporary signed links (~1 hour). They are not displayed in output. - -## Troubleshooting - -**Assistant not found** — run list command, check for typos. -**No response or timeout** — verify assistant has files uploaded and status is "ready" (not "indexing"). -**Empty or poor responses** — assistant may lack relevant documents; suggest upload. -**PINECONE_API_KEY not set** — export the variable or add to a `.env` file, then restart your IDE/agent session. diff --git a/skills/pinecone-assistant/references/context.md b/skills/pinecone-assistant/references/context.md deleted file mode 100644 index 5208922..0000000 --- a/skills/pinecone-assistant/references/context.md +++ /dev/null @@ -1,39 +0,0 @@ -# Retrieve Context Snippets - -Get raw context snippets from an assistant's knowledge base without generating a full chat response. Useful for debugging, custom RAG workflows, or quick lookups. - -## Arguments - -- `--assistant` (required): Assistant name -- `--query` (required): Search query text -- `--top-k` (optional): Number of snippets — default `5`, max `16` -- `--snippet-size` (optional): Max tokens per snippet — default `2048` -- `--json` (optional flag): JSON output - -## Workflow - -1. Parse arguments. If missing, list assistants and prompt for selection. -2. Execute: - ```bash - uv run scripts/context.py \ - --assistant "assistant-name" \ - --query "search text" \ - --top-k 5 - ``` -3. Display snippets: file name, page numbers, relevance score, content. - -## Context vs Chat - -**Use context when:** you want raw snippets, are debugging knowledge, need source material, or are building custom workflows. -**Use chat when:** you want synthesized answers, citations in a conversational response, or multi-turn Q&A. - -## Interpreting Results - -- **Score:** Higher (closer to 1.0) = more relevant -- **Low scores (<0.5):** Weak match, assistant may need more relevant documents, or query is too broad/specific - -## Troubleshooting - -**No results** — try broader search terms; suggest uploading more documents. -**context method not available** — update SDK: `pip install --upgrade pinecone` (requires v8.0.0+). -**Assistant not found** — check name for typos, run list command. diff --git a/skills/pinecone-assistant/references/create.md b/skills/pinecone-assistant/references/create.md deleted file mode 100644 index 16d8c57..0000000 --- a/skills/pinecone-assistant/references/create.md +++ /dev/null @@ -1,43 +0,0 @@ -# Create Assistant - -Create a new Pinecone Assistant with custom configuration. - -## Arguments - -- `--name` (required): Unique name for the assistant -- `--instructions` (optional): Behavior directive (tone, format, language) -- `--region` (optional): `us` or `eu` — default `us` -- `--timeout` (optional): Seconds to wait for ready status — default `30` - -## Workflow - -1. Parse arguments. If name is missing, prompt the user. -2. Ask the user about region preference — US or EU. -3. Ask if user wants custom instructions. Offer examples: - - "Use professional technical tone and cite sources" - - "Respond in Spanish with formal language" -4. Execute: - ```bash - uv run scripts/create.py \ - --name "assistant-name" \ - --instructions "instructions" \ - --region "us" - ``` -5. Show assistant name, status, and host URL. -6. Offer to run upload next. - -## Naming Conventions - -Suggest: `{purpose}-{type}` — e.g. `docs-qa`, `support-bot`, `api-helper` -Avoid: `test`, `assistant1`, `my-assistant` - -## Post-Creation - -- Save the assistant host URL shown in output (needed for MCP config) -- View and manage at: https://app.pinecone.io/organizations/-/projects/-/assistant/ - -## Troubleshooting - -**Assistant name already exists** — list assistants and suggest a different name or delete the existing one. -**Timeout** — increase `--timeout 60`, check network connectivity. -**PINECONE_API_KEY not set** — export the variable or add to a `.env` file, then restart your IDE/agent session. diff --git a/skills/pinecone-assistant/references/list.md b/skills/pinecone-assistant/references/list.md deleted file mode 100644 index 078b387..0000000 --- a/skills/pinecone-assistant/references/list.md +++ /dev/null @@ -1,31 +0,0 @@ -# List Assistants - -List all Pinecone Assistants in the account with optional file details. - -## Arguments - -- `--files` (optional flag): Show file details for each assistant -- `--json` (optional flag): JSON output - -## Usage - -```bash -# Basic listing -uv run scripts/list.py - -# With file details -uv run scripts/list.py --files - -# JSON output -uv run scripts/list.py --json - -# JSON with files (useful for scripting) -uv run scripts/list.py --files --json -``` - -## Output - -**Without `--files`:** Table with name, region, status, host. -**With `--files`:** Adds file count column, plus detailed file tables per assistant showing file name, status, and ID. - -File status is color-coded: green = available, yellow = processing. diff --git a/skills/pinecone-assistant/references/sync.md b/skills/pinecone-assistant/references/sync.md deleted file mode 100644 index 504f257..0000000 --- a/skills/pinecone-assistant/references/sync.md +++ /dev/null @@ -1,51 +0,0 @@ -# Sync Files - -Incrementally sync local files to an assistant — only uploads new or changed files. Uses mtime and size to detect changes. - -## Arguments - -- `--assistant` (required): Assistant name -- `--source` (required): Local file or directory path -- `--delete-missing` (optional flag): Delete files from assistant that no longer exist locally -- `--dry-run` (optional flag): Preview changes without executing -- `--yes` / `-y` (optional flag): Skip confirmation prompt - -## Workflow - -1. Parse arguments. If missing, list assistants and prompt for selection. -2. Execute: - ```bash - uv run scripts/sync.py \ - --assistant "assistant-name" \ - --source "./docs" \ - [--delete-missing] \ - [--dry-run] \ - [--yes] - ``` -3. Script compares local files against stored metadata, shows summary, asks for confirmation (unless `--yes`). - -## Flags - -- **`--delete-missing`** — removes files from the assistant that no longer exist locally. Use when cleaning up removed content. -- **`--dry-run`** — shows exactly what would change with no side effects. Always recommend this first. -- **`--yes`** — skips confirmation. Useful for automation; combine with `--dry-run` to verify first. - -## Common Workflow - -```bash -# Preview first -uv run scripts/sync.py --assistant my-docs --source ./docs --dry-run - -# Then apply -uv run scripts/sync.py --assistant my-docs --source ./docs - -# Keep in sync after git pull -git pull -uv run scripts/sync.py --assistant my-docs --source ./docs --delete-missing -``` - -## Troubleshooting - -**Files showing as changed but content unchanged** — mtime updates on save even without content changes; harmless, file will be re-uploaded. -**Sync is slow** — each update = delete + re-upload (2 operations); use `--dry-run` first to check scope. -**No supported files found** — check source contains `.md`, `.txt`, `.pdf`, `.docx`, or `.json` files not in excluded directories. diff --git a/skills/pinecone-assistant/references/upload.md b/skills/pinecone-assistant/references/upload.md deleted file mode 100644 index 0c24292..0000000 --- a/skills/pinecone-assistant/references/upload.md +++ /dev/null @@ -1,49 +0,0 @@ -# Upload Files - -Upload files or directory contents to a Pinecone Assistant. - -**Supported formats:** `.md`, `.txt`, `.pdf`, `.docx`, `.json` -**Not supported:** Source code (`.py`, `.js`, `.ts`, etc.) — Assistant is optimized for natural language documents. - -## Arguments - -- `--assistant` (required): Assistant name -- `--source` (required): File path or directory to upload -- `--patterns` (optional): Comma-separated glob patterns — default: `*.md,*.txt,*.pdf,*.docx,*.json` -- `--exclude` (optional): Directories to exclude — default: `node_modules,.venv,.git,build,dist` -- `--metadata` (optional): JSON string of additional metadata - -## Workflow - -1. Parse arguments. If missing, list assistants and prompt for selection. -2. Use Glob to preview files. Show count and types. -3. **If code files detected:** Warn user and automatically filter them out: - ``` - ⚠️ Found 50 Python files. Assistant works with documents only — I'll skip the code files. - Found 25 Markdown and 8 PDF files to upload instead. - ``` -4. Confirm with the user before proceeding. -5. Execute: - ```bash - uv run scripts/upload.py \ - --assistant "assistant-name" \ - --source "./docs" \ - --patterns "*.md,*.pdf" - ``` -6. Show progress and results. Remind user files are being indexed. - -## Default Exclusions - -`node_modules`, `.venv`, `venv`, `.git`, `build`, `dist`, `__pycache__`, `.next`, `.cache` - -## Metadata Best Practices - -```bash ---metadata '{"source":"github","repo":"owner/repo","branch":"main"}' -``` - -## Troubleshooting - -**No files found** — check patterns match file types in directory; verify path exists. -**Upload failures** — check file format is supported; try smaller batches. -**>100 files** — ask user if they want to be more selective; suggest `./docs` subdirectory. diff --git a/skills/pinecone-assistant/scripts/chat.py b/skills/pinecone-assistant/scripts/chat.py deleted file mode 100755 index 843d3db..0000000 --- a/skills/pinecone-assistant/scripts/chat.py +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env python3 -# /// script -# dependencies = [ -# "pinecone>=8.0.0", -# "typer>=0.15.0", -# "rich>=13.0.0", -# ] -# /// -""" -Chat with a Pinecone Assistant and receive cited responses. - -Usage: - uv run chat.py --assistant NAME --message "Your question" [--stream] - -Environment Variables: - PINECONE_API_KEY: Required Pinecone API key - -Output: - Assistant's response with citations to source documents -""" - -import os -import typer -from rich.console import Console -from rich.panel import Panel -from rich.table import Table -from pinecone import Pinecone -from pinecone_plugins.assistant.models.chat import Message - -app = typer.Typer() -console = Console() - - -@app.command() -def main( - assistant: str = typer.Option(..., "--assistant", "-a", help="Name of the assistant to chat with"), - message: str = typer.Option(..., "--message", "-m", help="Your question or message"), -): - """Chat with a Pinecone Assistant and receive answers with source citations.""" - - # Check for API key - api_key = os.environ.get("PINECONE_API_KEY") - if not api_key: - console.print("[red]Error: PINECONE_API_KEY environment variable not set[/red]") - console.print("\nGet your API key from: https://app.pinecone.io/?sessionType=signup") - raise typer.Exit(1) - - try: - # Initialize Pinecone client - pc = Pinecone(api_key=api_key,source_tag="claude_code_plugin:assistant") - asst = pc.assistant.Assistant(assistant_name=assistant) - - # Create message - user_msg = Message(role="user", content=message) - - # Display user question - console.print(Panel(f"[bold cyan]Question:[/bold cyan] {message}", border_style="cyan")) - - # Get response - with console.status("[bold blue]Thinking...[/bold blue]"): - response = asst.chat(messages=[user_msg], stream=False) - - answer_content = response.message.content - citations = response.citations if hasattr(response, 'citations') else [] - usage = response.usage if hasattr(response, 'usage') else None - - # Display assistant's response (same for both modes) - console.print("\n[bold green]Answer:[/bold green]\n") - - if answer_content: - console.print(Panel(answer_content, border_style="green", title="Assistant Response")) - else: - console.print("[yellow]No response content received[/yellow]") - - # Display citations if available - if citations and len(citations) > 0: - - console.print("\n[bold yellow]Citations:[/bold yellow]\n") - - citations_table = Table(show_header=True, header_style="bold yellow") - citations_table.add_column("#", style="dim", width=4) - citations_table.add_column("File", style="cyan", width=40) - citations_table.add_column("Pages", style="blue", width=15) - citations_table.add_column("Position", style="green", width=10) - - citation_num = 0 - for citation in citations: - # Each citation has a list of references - if hasattr(citation, 'references') and citation.references: - for reference in citation.references: - citation_num += 1 - - # Get file name - file_name = "Unknown" - if hasattr(reference, 'file') and hasattr(reference.file, 'name'): - file_name = reference.file.name - - # Get pages - pages = [] - if hasattr(reference, 'pages') and reference.pages: - pages = reference.pages - - # Format pages - if pages: - pages_str = ", ".join(str(p) for p in pages) - else: - pages_str = "N/A" - - # Get position from citation - position = getattr(citation, 'position', 'N/A') - - citations_table.add_row( - str(citation_num), - file_name, - pages_str, - str(position) - ) - - console.print(citations_table) - - # Optionally show download links - console.print("\n[dim]Tip: File URLs are temporary signed links valid for ~1 hour[/dim]") - - # Display token usage - if usage: - usage_info = f"""[dim]Tokens used:[/dim] -• Prompt: {getattr(usage, 'prompt_tokens', 'N/A')} -• Completion: {getattr(usage, 'completion_tokens', 'N/A')} -• Total: {getattr(usage, 'total_tokens', 'N/A')}""" - console.print(Panel(usage_info, border_style="dim", title="Usage Stats")) - - # Follow-up suggestion - console.print(f"\n[dim]Continue the conversation with another message using the same command[/dim]") - - except Exception as e: - console.print(f"[red]Error: {e}[/red]") - raise typer.Exit(1) - - -if __name__ == "__main__": - app() diff --git a/skills/pinecone-assistant/scripts/context.py b/skills/pinecone-assistant/scripts/context.py deleted file mode 100755 index 81ae65c..0000000 --- a/skills/pinecone-assistant/scripts/context.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python3 -# /// script -# dependencies = [ -# "pinecone>=8.0.0", -# "typer>=0.15.0", -# "rich>=13.0.0", -# ] -# /// -""" -Retrieve context snippets from a Pinecone Assistant's knowledge base. - -Usage: - uv run context.py --assistant NAME --query "search text" [--top-k 5] [--json] - -Environment Variables: - PINECONE_API_KEY: Required Pinecone API key - -Output: - Relevant context snippets with file sources, page numbers, and relevance scores -""" - -import os -import json as json_module -import typer -from rich.console import Console -from rich.panel import Panel -from rich.table import Table -from rich.text import Text -from pinecone import Pinecone - -app = typer.Typer() -console = Console() - - -@app.command() -def main( - assistant: str = typer.Option(..., "--assistant", "-a", help="Name of the assistant"), - query: str = typer.Option(..., "--query", "-q", help="Search query text"), - top_k: int = typer.Option(5, "--top-k", "-k", help="Number of results to return (max 16)"), - snippet_size: int = typer.Option(1024, "--snippet-size", "-s", help="Maximum tokens per snippet"), - json: bool = typer.Option(False, "--json", help="Output in JSON format"), -): - """Retrieve relevant context snippets from an assistant's knowledge base.""" - - # Check for API key - api_key = os.environ.get("PINECONE_API_KEY") - if not api_key: - console.print("[red]Error: PINECONE_API_KEY environment variable not set[/red]") - console.print("\nGet your API key from: https://app.pinecone.io/?sessionType=signup") - raise typer.Exit(1) - - try: - # Initialize Pinecone client - pc = Pinecone(api_key=api_key, source_tag="claude_code_plugin:assistant") - asst = pc.assistant.Assistant(assistant_name=assistant) - - # Display query - if not json: - console.print(Panel(f"[bold cyan]Query:[/bold cyan] {query}", border_style="cyan")) - - # Retrieve context - with console.status("[bold blue]Searching knowledge base...[/bold blue]", spinner="dots"): - response = asst.context(query=query, top_k=top_k, snippet_size=snippet_size) - - # Get snippets from response - snippets = response.snippets if hasattr(response, 'snippets') else [] - - if json: - # JSON output - results = [] - for snippet in snippets: - file_name = "Unknown" - pages = [] - if hasattr(snippet, 'reference') and snippet.reference: - ref = snippet.reference - if hasattr(ref, 'file') and hasattr(ref.file, 'name'): - file_name = ref.file.name - if hasattr(ref, 'pages') and ref.pages: - pages = ref.pages - - results.append({ - "file_name": file_name, - "pages": pages, - "content": getattr(snippet, 'content', ''), - "score": getattr(snippet, 'score', 0.0), - "type": getattr(snippet, 'type', 'text'), - }) - print(json_module.dumps({"snippets": results, "count": len(results)}, indent=2)) - else: - # Rich formatted output - if not snippets or len(snippets) == 0: - console.print("[yellow]No context found for this query[/yellow]") - return - - console.print(f"\n[bold]Found {len(snippets)} relevant snippet(s):[/bold]\n") - - for idx, snippet in enumerate(snippets, 1): - # Extract file info from reference - file_name = "Unknown" - pages = [] - if hasattr(snippet, 'reference') and snippet.reference: - ref = snippet.reference - if hasattr(ref, 'file') and hasattr(ref.file, 'name'): - file_name = ref.file.name - if hasattr(ref, 'pages') and ref.pages: - pages = ref.pages - - score = getattr(snippet, 'score', 0.0) - content = getattr(snippet, 'content', '') - - # Create header - header = f"#{idx} - {file_name}" - if pages: - pages_str = ", ".join(str(p) for p in pages) - header += f" (Page {pages_str})" - header += f" - Score: {score:.3f}" if isinstance(score, (int, float)) else f" - Score: {score}" - - console.print(Panel( - content, - title=header, - border_style="blue", - subtitle=f"[dim]Relevance: {score:.2%}[/dim]" if isinstance(score, (int, float)) else None - )) - console.print() - - # Suggest next action - next_action = f"""[bold]Next steps:[/bold] -• Ask a question: [cyan]/pinecone:assistant-chat assistant {assistant} message [your question][/cyan] -• Upload more files: [cyan]/pinecone:assistant-upload assistant {assistant} source [path][/cyan]""" - console.print(Panel(next_action, title="What's Next?", border_style="green")) - - except AttributeError as e: - # Handle case where context method doesn't exist or response structure is different - console.print(f"[red]Error: Context retrieval failed[/red]") - console.print(f"[dim]Details: {e}[/dim]") - console.print("\n[yellow]Note:[/yellow] Context API requires SDK version with assistant.context() support") - console.print("\n[yellow]Try using chat instead:[/yellow]") - console.print(f" /pinecone:assistant-chat assistant {assistant} message \"{query}\"") - raise typer.Exit(1) - except Exception as e: - console.print(f"[red]Error: {e}[/red]") - raise typer.Exit(1) - - -if __name__ == "__main__": - app() diff --git a/skills/pinecone-assistant/scripts/create.py b/skills/pinecone-assistant/scripts/create.py deleted file mode 100755 index c135b56..0000000 --- a/skills/pinecone-assistant/scripts/create.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python3 -# /// script -# dependencies = [ -# "pinecone>=8.0.0", -# "typer>=0.15.0", -# "rich>=13.0.0", -# ] -# /// -""" -Create a new Pinecone Assistant. - -Usage: - uv run create.py --name ASSISTANT_NAME [--instructions TEXT] [--region us|eu] [--timeout SECONDS] - -Environment Variables: - PINECONE_API_KEY: Required Pinecone API key - -Output: - Success message with assistant details including host URL for MCP configuration -""" - -import os -import typer -from rich.console import Console -from rich.panel import Panel -from rich.table import Table -from pinecone import Pinecone - -app = typer.Typer() -console = Console() - - -@app.command() -def main( - name: str = typer.Option(..., "--name", "-n", help="Unique name for the assistant"), - instructions: str = typer.Option( - "", - "--instructions", - "-i", - help="Instructions for assistant behavior (max 16KB)", - ), - region: str = typer.Option( - "us", - "--region", - "-r", - help="Deployment region: 'us' or 'eu'", - ), - timeout: int = typer.Option( - 30, - "--timeout", - "-t", - help="Seconds to wait for ready status", - ), -): - """Create a new Pinecone Assistant for document Q&A with citations.""" - - # Validate region - if region not in ["us", "eu"]: - console.print("[red]Error: Region must be 'us' or 'eu'[/red]") - raise typer.Exit(1) - - # Check for API key - api_key = os.environ.get("PINECONE_API_KEY") - if not api_key: - console.print("[red]Error: PINECONE_API_KEY environment variable not set[/red]") - console.print("\nGet your API key from: https://app.pinecone.io/?sessionType=signup") - raise typer.Exit(1) - - try: - # Initialize Pinecone client - with console.status(f"[bold blue]Creating assistant '{name}'...[/bold blue]"): - pc = Pinecone(api_key=api_key, source_tag="claude_code_plugin:assistant") - - # Create assistant - assistant = pc.assistant.create_assistant( - assistant_name=name, - instructions=instructions if instructions else None, - region=region, - timeout=timeout, - metadata={"agentic-ide-source":"claude-code-plugin"} - ) - - # Success message - console.print(f"\n[bold green]✓ Assistant '{name}' created successfully![/bold green]\n") - - # Display assistant details in a table - table = Table(show_header=False, box=None) - table.add_column("Property", style="cyan") - table.add_column("Value", style="white") - - table.add_row("Name", assistant.name) - table.add_row("Region", region) - table.add_row("Status", f"[yellow]{assistant.status}[/yellow]") - table.add_row("Host", getattr(assistant, "host", "N/A")) - if instructions: - instructions_preview = instructions[:80] + "..." if len(instructions) > 80 else instructions - table.add_row("Instructions", instructions_preview) - - console.print(table) - - # MCP configuration info - host = getattr(assistant, "host", "") - if host: - mcp_info = f"""[bold]MCP Endpoint:[/bold] -{host}/mcp/assistants/{name} - -[bold]Set environment variable:[/bold] -export PINECONE_ASSISTANT_HOST="{host}" -""" - console.print(Panel(mcp_info, title="MCP Configuration", border_style="blue")) - - # Next steps - next_steps = f"""[bold]Next steps:[/bold] -1. Upload files: [cyan]/pinecone:assistant-upload assistant {name} source [path][/cyan] -2. Chat: [cyan]/pinecone:assistant-chat assistant {name} message [your question][/cyan] -3. Get context: [cyan]/pinecone:assistant-context assistant {name} query [search][/cyan]""" - - console.print(Panel(next_steps, title="What's Next?", border_style="green")) - - except Exception as e: - console.print(f"[red]Error creating assistant: {e}[/red]") - raise typer.Exit(1) - - -if __name__ == "__main__": - app() diff --git a/skills/pinecone-assistant/scripts/list.py b/skills/pinecone-assistant/scripts/list.py deleted file mode 100755 index bd8a373..0000000 --- a/skills/pinecone-assistant/scripts/list.py +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/env python3 -# /// script -# dependencies = [ -# "pinecone>=8.0.0", -# "typer>=0.15.0", -# "rich>=13.0.0", -# ] -# /// -""" -List all Pinecone Assistants in the account. - -Usage: - uv run list.py [--json] [--files] - -Environment Variables: - PINECONE_API_KEY: Required Pinecone API key - -Output: - Formatted table or JSON list of assistants with name, region, status, and host - Optionally include files for each assistant with --files flag -""" - -import os -import sys -import json -import typer -from rich.console import Console -from rich.table import Table -from rich.panel import Panel -from pinecone import Pinecone - -app = typer.Typer() -console = Console() - - -@app.command() -def main( - json_output: bool = typer.Option(False, "--json", help="Output in JSON format"), - files: bool = typer.Option(False, "--files", "-f", help="Include file listing for each assistant"), -): - """List all Pinecone Assistants in your account.""" - - # Check for API key - api_key = os.environ.get('PINECONE_API_KEY') - if not api_key: - console.print("[red]Error: PINECONE_API_KEY environment variable not set[/red]") - console.print("\nGet your API key from: https://app.pinecone.io/?sessionType=signup") - raise typer.Exit(1) - - try: - # Initialize Pinecone client - pc = Pinecone(api_key=api_key, source_tag="claude_code_plugin:assistant") - - # List assistants - assistants = pc.assistant.list_assistants() - - if not assistants: - if json_output: - print(json.dumps({"assistants": [], "count": 0})) - else: - console.print("[yellow]No assistants found.[/yellow]\n") - console.print("Create your first assistant with:") - console.print(" [cyan]/pinecone:assistant-create name [assistant-name][/cyan]") - return - - if json_output: - # JSON output - assistants_data = [] - for asst in assistants: - asst_data = { - "name": asst.name, - "region": getattr(asst, 'region', 'unknown'), - "status": asst.status, - "host": getattr(asst, 'host', ''), - } - - if files: - # Get files for this assistant - try: - assistant_instance = pc.assistant.Assistant(assistant_name=asst.name) - file_list = assistant_instance.list_files() - asst_data["files"] = [ - { - "name": f.name, - "id": f.id, - "status": f.status, - "metadata": getattr(f, 'metadata', {}), - } - for f in file_list - ] - asst_data["file_count"] = len(file_list) - except Exception as e: - asst_data["files"] = [] - asst_data["file_count"] = 0 - asst_data["file_error"] = str(e) - - assistants_data.append(asst_data) - - result = { - "assistants": assistants_data, - "count": len(assistants) - } - print(json.dumps(result, indent=2)) - else: - # Rich table output - console.print(f"\n[bold]Found {len(assistants)} assistant(s):[/bold]\n") - - # Assistants table - table = Table(show_header=True, header_style="bold cyan") - table.add_column("Name", style="green", width=30) - table.add_column("Region", style="blue", width=10) - table.add_column("Status", style="yellow", width=15) - if files: - table.add_column("Files", style="magenta", width=10) - table.add_column("Host", style="dim", width=40 if files else 50) - - for asst in assistants: - name = asst.name - region = getattr(asst, 'region', 'unknown') - status = asst.status - host = getattr(asst, 'host', '') - - # Color code status - if status == 'ready': - status_display = f"[green]{status}[/green]" - elif status == 'indexing': - status_display = f"[yellow]{status}[/yellow]" - else: - status_display = status - - if files: - # Get file count for this assistant - try: - assistant_instance = pc.assistant.Assistant(assistant_name=asst.name) - file_list = assistant_instance.list_files() - file_count = str(len(file_list)) - except Exception: - file_count = "?" - - table.add_row(name, region, status_display, file_count, host) - else: - table.add_row(name, region, status_display, host) - - console.print(table) - console.print() - - # If --files flag is set, show detailed file listing for each assistant - if files: - console.print("[bold]File Details:[/bold]\n") - for asst in assistants: - try: - assistant_instance = pc.assistant.Assistant(assistant_name=asst.name) - file_list = assistant_instance.list_files() - - if file_list: - # Create a table for this assistant's files - file_table = Table(show_header=True, header_style="bold blue", title=f"[cyan]{asst.name}[/cyan]") - file_table.add_column("#", style="dim", width=4) - file_table.add_column("File Name", style="green", width=50) - file_table.add_column("Status", style="yellow", width=15) - file_table.add_column("ID", style="dim", width=30) - - for idx, file_obj in enumerate(file_list, 1): - file_name = file_obj.name - file_id = file_obj.id - file_status = file_obj.status - - # Color code file status - if file_status == 'available': - file_status_display = f"[green]{file_status}[/green]" - elif file_status == 'processing': - file_status_display = f"[yellow]{file_status}[/yellow]" - else: - file_status_display = file_status - - file_table.add_row(str(idx), file_name, file_status_display, file_id) - - console.print(file_table) - console.print() - else: - console.print(f"[dim]{asst.name}: No files uploaded[/dim]\n") - except Exception as e: - console.print(f"[red]Error listing files for {asst.name}: {e}[/red]\n") - - # Next steps panel - next_steps = """[bold]Next steps:[/bold] -• List with files: [cyan]/pinecone:assistant-list --files[/cyan] -• Chat: [cyan]/pinecone:assistant-chat assistant [name] message [your question][/cyan] -• Upload: [cyan]/pinecone:assistant-upload assistant [name] source [path][/cyan] -• Context: [cyan]/pinecone:assistant-context assistant [name] query [search][/cyan]""" - - console.print(Panel(next_steps, title="Available Commands", border_style="blue")) - - except Exception as e: - console.print(f"[red]Error listing assistants: {e}[/red]") - raise typer.Exit(1) - - -if __name__ == "__main__": - app() diff --git a/skills/pinecone-assistant/scripts/sync.py b/skills/pinecone-assistant/scripts/sync.py deleted file mode 100644 index 13c98a5..0000000 --- a/skills/pinecone-assistant/scripts/sync.py +++ /dev/null @@ -1,354 +0,0 @@ -#!/usr/bin/env python3 -# /// script -# dependencies = [ -# "pinecone>=8.0.0", -# "typer>=0.15.0", -# "rich>=13.0.0", -# ] -# /// -""" -Sync local files to a Pinecone Assistant, only uploading new or changed files. - -Usage: - uv run sync.py --assistant NAME --source PATH [--delete-missing] [--dry-run] - -Environment Variables: - PINECONE_API_KEY: Required Pinecone API key - -Output: - Shows files to add, update, and optionally delete, with confirmation prompt -""" - -import os -import hashlib -from pathlib import Path -from datetime import datetime, timezone -import typer -from rich.console import Console -from rich.panel import Panel -from rich.table import Table -from rich.progress import Progress, SpinnerColumn, TextColumn -from pinecone import Pinecone - -app = typer.Typer() -console = Console() - -# Supported file extensions -SUPPORTED_EXTENSIONS = {'.md', '.txt', '.pdf', '.docx', '.json'} - -# Directories to exclude -EXCLUDE_DIRS = {'node_modules', '.venv', '.git', 'build', 'dist', '__pycache__', '.pytest_cache'} - - -def should_exclude_path(path: Path, source_root: Path) -> bool: - """Check if path should be excluded based on directory patterns.""" - try: - rel_path = path.relative_to(source_root) - for part in rel_path.parts: - if part in EXCLUDE_DIRS or part.startswith('.'): - return True - except ValueError: - return True - return False - - -def find_files(source_path: Path) -> list[Path]: - """Find all supported files in source directory, excluding common build/dependency dirs.""" - files = [] - - if source_path.is_file(): - if source_path.suffix.lower() in SUPPORTED_EXTENSIONS: - return [source_path] - else: - return [] - - for file_path in source_path.rglob('*'): - if file_path.is_file(): - if file_path.suffix.lower() in SUPPORTED_EXTENSIONS: - if not should_exclude_path(file_path, source_path): - files.append(file_path) - - return sorted(files) - - -def get_file_info(file_path: Path): - """Get file modification time and size.""" - stat = file_path.stat() - return { - 'mtime': stat.st_mtime, - 'size': stat.st_size, - } - - -def file_changed(local_info: dict, remote_metadata: dict) -> bool: - """Check if local file differs from remote using mtime and size.""" - remote_mtime = remote_metadata.get('mtime') - remote_size = remote_metadata.get('size') - - if remote_mtime is None or remote_size is None: - # No stored metadata, assume changed - return True - - return (local_info['mtime'] != float(remote_mtime) or - local_info['size'] != int(remote_size)) - - -@app.command() -def main( - assistant: str = typer.Option(..., "--assistant", "-a", help="Name of the assistant"), - source: str = typer.Option(..., "--source", "-s", help="Local file or directory path"), - delete_missing: bool = typer.Option(False, "--delete-missing", help="Delete files from assistant that don't exist locally"), - dry_run: bool = typer.Option(False, "--dry-run", help="Show what would change without making changes"), - yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), -): - """Sync local files to Pinecone Assistant, only uploading new or changed files.""" - - # Check for API key - api_key = os.environ.get("PINECONE_API_KEY") - if not api_key: - console.print("[red]Error: PINECONE_API_KEY environment variable not set[/red]") - console.print("\nGet your API key from: https://app.pinecone.io/?sessionType=signup") - raise typer.Exit(1) - - # Validate source path - source_path = Path(source).resolve() - if not source_path.exists(): - console.print(f"[red]Error: Source path does not exist: {source}[/red]") - raise typer.Exit(1) - - try: - # Initialize Pinecone client - pc = Pinecone(api_key=api_key, source_tag="claude_code_plugin:assistant") - asst = pc.assistant.Assistant(assistant_name=assistant) - - console.print(Panel( - f"[bold cyan]Assistant:[/bold cyan] {assistant}\n" - f"[bold cyan]Source:[/bold cyan] {source_path}", - title="Sync Configuration", - border_style="cyan" - )) - - # Step 1: Get current files in assistant - with console.status("[bold blue]Fetching assistant files...[/bold blue]", spinner="dots"): - remote_files = asst.list_files() - - # Build map of file_path -> file object - remote_file_map = {} - for f in remote_files: - metadata = getattr(f, 'metadata', {}) or {} - file_path = metadata.get('file_path', f.name) - remote_file_map[file_path] = { - 'file_obj': f, - 'metadata': metadata - } - - console.print(f"[dim]Found {len(remote_files)} file(s) in assistant[/dim]\n") - - # Step 2: Find local files - with console.status("[bold blue]Scanning local files...[/bold blue]", spinner="dots"): - local_files = find_files(source_path) - - if not local_files: - console.print("[yellow]No supported files found in source path[/yellow]") - console.print(f"Supported extensions: {', '.join(sorted(SUPPORTED_EXTENSIONS))}") - raise typer.Exit(0) - - console.print(f"[dim]Found {len(local_files)} local file(s)[/dim]\n") - - # Step 3: Determine what needs syncing - to_upload = [] # New files - to_update = [] # Changed files (delete + re-upload) - to_delete = [] # Files in assistant but not local - unchanged = [] # Files that match - - # Track which remote files we've seen - seen_remote_paths = set() - - for local_file in local_files: - # Get relative path from source root - if source_path.is_file(): - rel_path = local_file.name - else: - rel_path = str(local_file.relative_to(source_path)) - - local_info = get_file_info(local_file) - - if rel_path in remote_file_map: - # File exists remotely, check if changed - seen_remote_paths.add(rel_path) - remote_info = remote_file_map[rel_path] - - if file_changed(local_info, remote_info['metadata']): - to_update.append({ - 'local_path': local_file, - 'rel_path': rel_path, - 'remote_file_id': remote_info['file_obj'].id, - 'local_info': local_info - }) - else: - unchanged.append(rel_path) - else: - # New file - to_upload.append({ - 'local_path': local_file, - 'rel_path': rel_path, - 'local_info': local_info - }) - - # Find files to delete (in remote but not local) - if delete_missing: - for rel_path, remote_info in remote_file_map.items(): - if rel_path not in seen_remote_paths: - to_delete.append({ - 'rel_path': rel_path, - 'remote_file_id': remote_info['file_obj'].id - }) - - # Step 4: Show summary - console.print("[bold]Sync Summary:[/bold]\n") - - summary_table = Table(show_header=True, header_style="bold cyan") - summary_table.add_column("Action", style="yellow", width=15) - summary_table.add_column("Count", style="green", width=10) - - summary_table.add_row("New files", str(len(to_upload))) - summary_table.add_row("Updated files", str(len(to_update))) - if delete_missing: - summary_table.add_row("Deleted files", str(len(to_delete))) - summary_table.add_row("Unchanged", str(len(unchanged))) - - console.print(summary_table) - console.print() - - # Show details if there are changes - if to_upload: - console.print("[bold green]Files to upload:[/bold green]") - for item in to_upload[:10]: # Show first 10 - console.print(f" + {item['rel_path']}") - if len(to_upload) > 10: - console.print(f" ... and {len(to_upload) - 10} more") - console.print() - - if to_update: - console.print("[bold yellow]Files to update:[/bold yellow]") - for item in to_update[:10]: - console.print(f" ~ {item['rel_path']}") - if len(to_update) > 10: - console.print(f" ... and {len(to_update) - 10} more") - console.print() - - if to_delete: - console.print("[bold red]Files to delete:[/bold red]") - for item in to_delete[:10]: - console.print(f" - {item['rel_path']}") - if len(to_delete) > 10: - console.print(f" ... and {len(to_delete) - 10} more") - console.print() - - # If no changes, exit early - if not (to_upload or to_update or to_delete): - console.print("[green]✓ All files are up to date![/green]") - return - - # Dry run mode - if dry_run: - console.print("[yellow]Dry run mode: No changes made[/yellow]") - return - - # Confirmation prompt - if not yes: - proceed = typer.confirm("\nProceed with sync?") - if not proceed: - console.print("[yellow]Sync cancelled[/yellow]") - return - - console.print() - - # Step 5: Execute sync - uploaded_count = 0 - updated_count = 0 - deleted_count = 0 - - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - console=console - ) as progress: - - # Upload new files - if to_upload: - task = progress.add_task(f"Uploading {len(to_upload)} new file(s)...", total=len(to_upload)) - for item in to_upload: - try: - asst.upload_file( - file_path=str(item['local_path']), - metadata={ - 'file_path': item['rel_path'], - 'mtime': item['local_info']['mtime'], - 'size': item['local_info']['size'], - 'uploaded_at': datetime.now(timezone.utc).isoformat(), - 'source': 'sync_script', - }, - timeout=None - ) - uploaded_count += 1 - progress.advance(task) - except Exception as e: - console.print(f"[red]Failed to upload {item['rel_path']}: {e}[/red]") - - # Update changed files (delete old + upload new) - if to_update: - task = progress.add_task(f"Updating {len(to_update)} file(s)...", total=len(to_update) * 2) - for item in to_update: - try: - # Delete old version - asst.delete_file(file_id=item['remote_file_id']) - progress.advance(task) - - # Upload new version - asst.upload_file( - file_path=str(item['local_path']), - metadata={ - 'file_path': item['rel_path'], - 'mtime': item['local_info']['mtime'], - 'size': item['local_info']['size'], - 'uploaded_at': datetime.now(timezone.utc).isoformat(), - 'source': 'sync_script', - }, - timeout=None - ) - updated_count += 1 - progress.advance(task) - except Exception as e: - console.print(f"[red]Failed to update {item['rel_path']}: {e}[/red]") - - # Delete missing files - if to_delete: - task = progress.add_task(f"Deleting {len(to_delete)} file(s)...", total=len(to_delete)) - for item in to_delete: - try: - asst.delete_file(file_id=item['remote_file_id']) - deleted_count += 1 - progress.advance(task) - except Exception as e: - console.print(f"[red]Failed to delete {item['rel_path']}: {e}[/red]") - - # Final summary - console.print() - console.print(Panel( - f"[green]✓ Sync complete![/green]\n\n" - f"Uploaded: {uploaded_count}\n" - f"Updated: {updated_count}\n" - + (f"Deleted: {deleted_count}\n" if delete_missing else "") + - f"Unchanged: {len(unchanged)}", - title="Results", - border_style="green" - )) - - except Exception as e: - console.print(f"[red]Error: {e}[/red]") - raise typer.Exit(1) - - -if __name__ == "__main__": - app() diff --git a/skills/pinecone-assistant/scripts/upload.py b/skills/pinecone-assistant/scripts/upload.py deleted file mode 100755 index 4484e75..0000000 --- a/skills/pinecone-assistant/scripts/upload.py +++ /dev/null @@ -1,222 +0,0 @@ -#!/usr/bin/env python3 -# /// script -# dependencies = [ -# "pinecone>=8.0.0", -# "typer>=0.15.0", -# "rich>=13.0.0", -# ] -# /// -""" -Upload files or repository contents to a Pinecone Assistant. - -IMPORTANT: Only uploads DOCUMENTATION and DATA files. -Supported: DOCX (.docx), JSON (.json), Markdown (.md), PDF (.pdf), Text (.txt) -Code files are NOT supported by Pinecone Assistant. - -Usage: - uv run upload.py --assistant NAME --source PATH [--patterns "*.md,*.pdf,*.docx"] - -Environment Variables: - PINECONE_API_KEY: Required Pinecone API key - -Output: - Progress updates and summary of uploaded files -""" - -import os -import glob -from pathlib import Path -from typing import List -from datetime import datetime, timezone -import typer -from rich.console import Console -from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn -from rich.table import Table -from rich.panel import Panel -from pinecone import Pinecone - -app = typer.Typer() -console = Console() - -# Default file patterns - DOCUMENTATION ONLY -# Assistant supports: DOCX, JSON, Markdown, PDF, Text -DEFAULT_PATTERNS = ["**/*.md", "**/*.txt", "**/*.pdf", "**/*.docx", "**/*.json"] - -# Default directories to exclude -DEFAULT_EXCLUDES = ["node_modules", ".venv", "venv", ".git", "build", "dist", "__pycache__", ".next", ".cache"] - - -def find_files(source_path: str, patterns: List[str], excludes: List[str]) -> List[Path]: - """Find files matching patterns, excluding certain directories.""" - source = Path(source_path) - - if not source.exists(): - console.print(f"[red]Error: Path '{source_path}' does not exist[/red]") - raise typer.Exit(1) - - # If it's a single file, return it - if source.is_file(): - return [source] - - # Otherwise, scan directory - files = [] - for pattern in patterns: - matched = glob.glob(str(source / pattern), recursive=True) - files.extend([Path(f) for f in matched]) - - # Filter out excluded directories - filtered_files = [] - for file_path in files: - # Check if any exclude pattern is in the path - if not any(excl in str(file_path) for excl in excludes): - filtered_files.append(file_path) - - return sorted(set(filtered_files)) - - -@app.command() -def main( - assistant: str = typer.Option(..., "--assistant", "-a", help="Name of the assistant to upload to"), - source: str = typer.Option(..., "--source", "-s", help="File or directory path to upload"), - patterns: str = typer.Option( - ",".join(DEFAULT_PATTERNS), - "--patterns", - "-p", - help="Comma-separated glob patterns for documentation files (e.g., '*.md,*.pdf')", - ), - exclude: str = typer.Option( - ",".join(DEFAULT_EXCLUDES), - "--exclude", - "-e", - help="Comma-separated directories to exclude", - ), - metadata_json: str = typer.Option( - "", - "--metadata", - "-m", - help="Additional metadata as JSON string", - ), -): - """Upload documentation files to a Pinecone Assistant. - - NOTE: Only documentation files (markdown, text, PDF) are supported. - Code files are not recommended for Pinecone Assistant. - """ - - # Check for API key - api_key = os.environ.get("PINECONE_API_KEY") - if not api_key: - console.print("[red]Error: PINECONE_API_KEY environment variable not set[/red]") - console.print("\nGet your API key from: https://app.pinecone.io/?sessionType=signup") - raise typer.Exit(1) - - # Parse patterns and excludes - pattern_list = [p.strip() for p in patterns.split(",")] - exclude_list = [e.strip() for e in exclude.split(",")] - - # Parse additional metadata if provided - extra_metadata = {} - if metadata_json: - import json - try: - extra_metadata = json.loads(metadata_json) - except json.JSONDecodeError: - console.print("[red]Error: Invalid JSON in --metadata parameter[/red]") - raise typer.Exit(1) - - try: - # Initialize Pinecone client - pc = Pinecone(api_key=api_key, source_tag="claude_code_plugin:assistant") - asst = pc.assistant.Assistant(assistant_name=assistant) - - # Find files to upload - console.print(f"\n[bold]Scanning for documentation files in:[/bold] {source}") - console.print(f"[dim]Patterns: {', '.join(pattern_list)}[/dim]\n") - - files = find_files(source, pattern_list, exclude_list) - - if not files: - console.print("[yellow]No documentation files found matching the specified patterns[/yellow]") - console.print("\n[dim]Tip: Pinecone Assistant works with .md, .txt, and .pdf files[/dim]") - return - - console.print(f"[green]Found {len(files)} documentation file(s) to upload[/green]\n") - - # Upload files with progress bar - uploaded = 0 - failed = 0 - failed_files = [] - - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - BarColumn(), - TaskProgressColumn(), - console=console, - ) as progress: - task = progress.add_task("[cyan]Uploading files...", total=len(files)) - - for file_path in files: - try: - # Build metadata - rel_path = os.path.relpath(str(file_path), source) - stat = file_path.stat() - metadata = { - "source": "upload_script", - "file_path": rel_path, - "file_type": file_path.suffix, - "content_type": "documentation", - "mtime": stat.st_mtime, - "size": stat.st_size, - "uploaded_at": datetime.now(timezone.utc).isoformat(), - **extra_metadata, - } - - # Upload file - asst.upload_file( - file_path=str(file_path), - metadata=metadata, - timeout=None, - ) - uploaded += 1 - progress.update(task, advance=1, description=f"[cyan]Uploaded: {rel_path}") - - except Exception as e: - failed += 1 - failed_files.append((str(file_path), str(e))) - progress.update(task, advance=1) - - # Summary table - console.print() - summary = Table(show_header=False, box=None) - summary.add_column("Status", style="bold") - summary.add_column("Count") - - summary.add_row("[green]✓ Uploaded[/green]", str(uploaded)) - if failed > 0: - summary.add_row("[red]✗ Failed[/red]", str(failed)) - - console.print(Panel(summary, title="Upload Summary", border_style="blue")) - - # Show failed files if any - if failed_files: - console.print("\n[bold red]Failed uploads:[/bold red]") - for file_path, error in failed_files: - console.print(f" • {file_path}: [red]{error}[/red]") - - # Next steps - if uploaded > 0: - next_steps = f"""[bold]Next steps:[/bold] -• Chat: [cyan]/pinecone:assistant-chat assistant {assistant} message [your question][/cyan] -• Context: [cyan]/pinecone:assistant-context assistant {assistant} query [search][/cyan] - -[dim]Note: Files are being processed and will be available shortly[/dim]""" - console.print(Panel(next_steps, title="What's Next?", border_style="green")) - - except Exception as e: - console.print(f"[red]Error: {e}[/red]") - raise typer.Exit(1) - - -if __name__ == "__main__": - app() diff --git a/skills/pinecone-cli/SKILL.md b/skills/pinecone-cli/SKILL.md deleted file mode 100644 index 161aaa1..0000000 --- a/skills/pinecone-cli/SKILL.md +++ /dev/null @@ -1,156 +0,0 @@ ---- -name: pinecone-cli -description: Guide for using the Pinecone CLI (pc) to manage Pinecone resources from the terminal. The CLI supports ALL index types (standard, integrated, sparse) and all vector operations — unlike the MCP which only supports integrated indexes. Use for batch operations, vector management, backups, namespaces, CI/CD automation, and full control over Pinecone resources. -argument-hint: install | auth | index [op] | vector [op] | backup | namespace ---- - -# Pinecone CLI (`pc`) - -Manage Pinecone from the terminal. The CLI is especially valuable for vector operations across **all index types** — something the MCP currently can't do. - -## CLI vs MCP - -| | CLI | MCP | -|---|---|---| -| Index types | All (standard, integrated, sparse) | Integrated only | -| Vector ops (upsert, query, fetch, update, delete) | ✅ | ❌ | -| Text search on integrated indexes | ✅ | ✅ | -| Backups, namespaces, org/project mgmt | ✅ | ❌ | -| CI/CD / scripting | ✅ | ❌ | - ---- - -## Setup - -### Install (macOS) -```bash -brew tap pinecone-io/tap -brew install pinecone-io/tap/pinecone -``` - -Other platforms (Linux, Windows) — download from [GitHub Releases](https://github.com/pinecone-io/cli/releases). - -### Authenticate - -```bash -# Interactive (recommended for local dev) -pc login -pc target -o "my-org" -p "my-project" - -# Service account (recommended for CI/CD) -pc auth configure --client-id "$PINECONE_CLIENT_ID" --client-secret "$PINECONE_CLIENT_SECRET" - -# API key (quick testing) -pc config set-api-key $PINECONE_API_KEY -``` - -Check status: `pc auth status` · `pc target --show` - -> **Note for agent sessions**: If you need to run `pc login` inside an agent loop, the browser auth link may not surface correctly. It's best to authenticate **before** starting an agent session. Run `pc login` in your terminal directly, then invoke the agent once you're authenticated. - -### Authenticating the CLI does not set `PINECONE_API_KEY` - -`pc login` authenticates the CLI tool itself — it does **not** set `PINECONE_API_KEY` in your environment. Python scripts, Node.js SDKs, and other tools that use the Pinecone SDK need `PINECONE_API_KEY` set separately. - -Use the CLI to create a key and export it in one step: - -```bash -KEY=$(pc api-key create --name agent-sdk-key --json | jq -r '.value') -export PINECONE_API_KEY="$KEY" -``` - -Without `jq`: run `pc api-key create --name agent-sdk-key --json` and copy the `"value"` field manually. - ---- - -## Common Commands - -| Task | Command | -|---|---| -| List indexes | `pc index list` | -| Create serverless index | `pc index create -n my-index -d 1536 -m cosine -c aws -r us-east-1` | -| Index stats | `pc index stats -n my-index` | -| Upload vectors from file | `pc index vector upsert -n my-index --file ./vectors.json` | -| Query by vector | `pc index vector query -n my-index --vector '[0.1, ...]' -k 10 --include-metadata` | -| Query by vector ID | `pc index vector query -n my-index --id "doc-123" -k 10` | -| Fetch vectors by ID | `pc index vector fetch -n my-index --ids '["vec1","vec2"]'` | -| List vector IDs | `pc index vector list -n my-index` | -| Delete vectors by filter | `pc index vector delete -n my-index --filter '{"genre":"classical"}'` | -| List namespaces | `pc index namespace list -n my-index` | -| Create backup | `pc backup create -i my-index -n "my-backup"` | -| JSON output (for scripting) | Add `-j` to any command | - ---- - -## Interesting Things You Can Do - -### Query with custom vectors (not just text) -Unlike the MCP, the CLI lets you query any index with raw vector values — useful when you generate embeddings externally (OpenAI, HuggingFace, etc.): -```bash -pc index vector query -n my-index \ - --vector '[0.1, 0.2, ..., 0.9]' \ - --filter '{"source":{"$eq":"docs"}}' \ - -k 20 --include-metadata -``` - -### Pipe embeddings directly into queries -```bash -jq -c '.embedding' doc.json | pc index vector query -n my-index --vector - -k 10 -``` - -### Bulk metadata update with preview -```bash -# Preview first -pc index vector update -n my-index \ - --filter '{"env":{"$eq":"staging"}}' \ - --metadata '{"env":"production"}' \ - --dry-run - -# Apply -pc index vector update -n my-index \ - --filter '{"env":{"$eq":"staging"}}' \ - --metadata '{"env":"production"}' -``` - -### Backup and restore -```bash -# Snapshot before a migration -pc backup create -i my-index -n "pre-migration" - -# Restore to a new index if something goes wrong -pc backup restore -i -n my-index-restored -``` - -### Automate in CI/CD -```bash -export PINECONE_CLIENT_ID="..." -export PINECONE_CLIENT_SECRET="..." -pc auth configure --client-id "$PINECONE_CLIENT_ID" --client-secret "$PINECONE_CLIENT_SECRET" -pc index vector upsert -n my-index --file ./vectors.jsonl --batch-size 1000 -``` - -### Script against JSON output -```bash -# Get all index names as a list -pc index list -j | jq -r '.[] | .name' - -# Check if an index exists before creating -if ! pc index describe -n my-index -j 2>/dev/null | jq -e '.name' > /dev/null; then - pc index create -n my-index -d 1536 -m cosine -c aws -r us-east-1 -fi -``` - ---- - -## Reference Files - -- [Full command reference](references/command-reference.md) — all commands with flags and examples -- [Troubleshooting & best practices](references/troubleshooting.md) - -## Documentation - -- [CLI Quickstart](https://docs.pinecone.io/reference/cli/quickstart) -- [Command Reference](https://docs.pinecone.io/reference/cli/command-reference) -- [Authentication](https://docs.pinecone.io/reference/cli/authentication) -- [Target Context](https://docs.pinecone.io/reference/cli/target-context) -- [GitHub Releases](https://github.com/pinecone-io/cli/releases) diff --git a/skills/pinecone-cli/references/command-reference.md b/skills/pinecone-cli/references/command-reference.md deleted file mode 100644 index 0091ad2..0000000 --- a/skills/pinecone-cli/references/command-reference.md +++ /dev/null @@ -1,237 +0,0 @@ -# Pinecone CLI — Full Command Reference - -## Index Management - -### Create Index -```bash -# Serverless index -pc index create -n my-index -d 1536 -m cosine -c aws -r us-east-1 - -# With integrated embedding model -pc index create -n my-index -m cosine -c aws -r us-east-1 \ - --model multilingual-e5-large \ - --field-map text=chunk_text - -# Sparse vector index -pc index create -n sparse-index -m dotproduct -c aws -r us-east-1 --vector-type sparse - -# With deletion protection -pc index create -n my-index -d 1536 -m cosine -c aws -r us-east-1 --deletion-protection enabled - -# From collection -pc index create -n my-index -d 1536 -m cosine -c aws -r us-east-1 --source-collection my-collection -``` - -### List / Describe / Stats -```bash -pc index list # Summary view -pc index list --wide # Additional columns (host, embed, tags) -pc index list -j # JSON output - -pc index describe -n my-index -pc index describe -n my-index -j - -pc index stats -n my-index -pc index stats -n my-index --filter '{"genre":{"$eq":"rock"}}' -``` - -### Configure / Delete -```bash -# Enable deletion protection -pc index configure -n my-index --deletion-protection enabled - -# Add tags -pc index configure -n my-index --tags environment=production,team=ml - -# Switch to dedicated read capacity -pc index configure -n my-index \ - --read-mode dedicated \ - --read-node-type b1 \ - --read-shards 2 \ - --read-replicas 2 - -pc index delete -n my-index -``` - ---- - -## Vector Operations - -### Upsert -```bash -# From JSON file (with "vectors" array) -pc index vector upsert -n my-index --file ./vectors.json - -# From JSONL file (one vector per line) -pc index vector upsert -n my-index --file ./vectors.jsonl - -# Inline JSON -pc index vector upsert -n my-index --file '{"vectors": [{"id": "vec1", "values": [0.1, 0.2, 0.3]}]}' - -# From stdin -cat vectors.json | pc index vector upsert -n my-index --file - - -# With namespace, custom batch size -pc index vector upsert -n my-index --namespace tenant-a --file ./vectors.json --batch-size 1000 -``` - -**File formats:** -```json -// JSON (vectors.json) -{"vectors": [{"id": "vec1", "values": [0.1, 0.2, 0.3], "metadata": {"genre": "comedy"}}]} - -// JSONL (vectors.jsonl) -{"id": "vec1", "values": [0.1, 0.2, 0.3], "metadata": {"genre": "comedy"}} -{"id": "vec2", "values": [0.4, 0.5, 0.6], "metadata": {"genre": "drama"}} -``` - -### Query -```bash -# By vector values -pc index vector query -n my-index --vector '[0.1, 0.2, 0.3]' -k 10 --include-metadata - -# By vector ID -pc index vector query -n my-index --id "doc-123" -k 10 --include-metadata - -# With metadata filter -pc index vector query -n my-index \ - --vector '[0.1, 0.2, 0.3]' \ - --filter '{"genre":{"$eq":"sci-fi"}}' \ - --include-metadata - -# Sparse vectors -pc index vector query -n my-index \ - --sparse-indices '[0, 5, 12]' \ - --sparse-values '[0.5, 0.3, 0.8]' \ - -k 15 - -# From stdin -jq -c '.embedding' doc.json | pc index vector query -n my-index --vector - -k 10 -``` - -### Fetch -```bash -pc index vector fetch -n my-index --ids '["vec1","vec2","vec3"]' -pc index vector fetch -n my-index --filter '{"genre":{"$eq":"rock"}}' -pc index vector fetch -n my-index --namespace tenant-a --ids '["doc-123"]' -pc index vector fetch -n my-index --filter '{"genre":{"$eq":"rock"}}' --limit 100 -``` - -### List / Update / Delete -```bash -# List vector IDs -pc index vector list -n my-index -pc index vector list -n my-index --namespace tenant-a --limit 50 - -# Update metadata or values -pc index vector update -n my-index --id "vec1" --metadata '{"category":"updated"}' -pc index vector update -n my-index --id "vec1" --values '[0.2, 0.3, 0.4]' - -# Bulk update with dry-run -pc index vector update -n my-index \ - --filter '{"genre":{"$eq":"sci-fi"}}' \ - --metadata '{"genre":"fantasy"}' \ - --dry-run - -# Delete by IDs or filter -pc index vector delete -n my-index --ids '["vec1","vec2"]' -pc index vector delete -n my-index --filter '{"genre":"classical"}' -pc index vector delete -n my-index --namespace old-data --all-vectors -``` - ---- - -## Namespace Management - -```bash -pc index namespace create -n my-index --name tenant-a -pc index namespace create -n my-index --name tenant-b --schema "category,brand" -pc index namespace list -n my-index -pc index namespace list -n my-index --prefix "tenant-" -pc index namespace describe -n my-index --name tenant-a -pc index namespace delete -n my-index --name tenant-a # WARNING: deletes all vectors -``` - ---- - -## Backup and Restore - -```bash -# Create / list / describe -pc backup create -i my-index -n "nightly-backup" -d "Backup before deployment" -pc backup list -pc backup list --index-name my-index -pc backup describe -i - -# Restore (creates a new index) -pc backup restore -i -n restored-index -pc backup restore -i -n restored-index --deletion-protection enabled - -# Check restore job status -pc backup restore list -pc backup restore describe -i rj-abc123 - -# Delete backup -pc backup delete -i -``` - ---- - -## Project Management - -```bash -pc project list -pc project create -n "demo-project" -pc project create -n "demo-project" --target -pc project describe -i proj-abc123 -pc project update -i proj-abc123 -n "new-name" -pc project delete -i proj-abc123 -``` - ---- - -## Organization Management - -```bash -pc organization list -pc organization describe -i org-abc123 -pc organization update -i org-abc123 -n "new-name" -pc organization delete -i org-abc123 # WARNING: highly destructive -``` - ---- - -## API Key Management - -```bash -pc api-key create -n "my-key" -pc api-key create -n "my-key" --store -pc api-key create -n "my-key" -i proj-abc123 -pc api-key list -pc api-key describe -i key-abc123 -pc api-key update -i key-abc123 --roles ProjectEditor -pc api-key delete -i key-abc123 -``` - ---- - -## Global Flags - -Available on all commands: -- `-h, --help` — Show help -- `-j, --json` — JSON output (great for scripting) -- `-q, --quiet` — Suppress output -- `--timeout` — Command timeout (default: 60s, 0 to disable) - -## Exit Codes - -- `0` — success -- `1` — error - -```bash -if pc index describe -n my-index 2>/dev/null; then - echo "Index exists" -else - pc index create -n my-index -d 1536 -m cosine -c aws -r us-east-1 -fi -``` diff --git a/skills/pinecone-cli/references/troubleshooting.md b/skills/pinecone-cli/references/troubleshooting.md deleted file mode 100644 index 6458bd8..0000000 --- a/skills/pinecone-cli/references/troubleshooting.md +++ /dev/null @@ -1,136 +0,0 @@ -# Pinecone CLI — Troubleshooting & Best Practices - -## Troubleshooting - -### Authentication Issues - -**"Not authenticated" or "Invalid credentials"** -```bash -pc auth status -pc logout -pc login -pc target -o "my-org" -p "my-project" -``` - -**Service account can't access resources** -```bash -pc target --show # Verify correct project is targeted -``` - -### API Key Issues - -**API key not working** -```bash -pc config get-api-key # Verify key is set -# API keys are scoped to org + project — get a new one if needed -pc api-key create -n "new-key" --store -``` - -### Target Context Issues - -**"Project not found" or "Organization not found"** -```bash -pc target --show -pc target --clear -pc target -o "my-org" -p "my-project" -``` - -### Index Issues - -**Index operations failing** -```bash -pc index describe -n my-index -# "Initializing" → wait and retry -# "Terminating" → recreate it -``` - -**Can't delete index** -```bash -# Check if deletion protection is on -pc index describe -n my-index -pc index configure -n my-index --deletion-protection disabled -pc index delete -n my-index -``` - -### Vector Upload Issues - -**Upsert fails with dimension mismatch** -```bash -pc index describe -n my-index # Check configured dimension -# Ensure all vectors have exactly that many values -``` - -**Large file upload is slow** -```bash -# Use max batch size -pc index vector upsert -n my-index --file ./large.json --batch-size 1000 - -# Or split JSONL and loop -split -l 10000 large.jsonl chunk- -for file in chunk-*; do - pc index vector upsert -n my-index --file "$file" -done -``` - -### Query Issues - -**Query returns no results** -```bash -pc index stats -n my-index # Check if data exists -pc index namespace list -n my-index # Verify namespace -# Filters use MongoDB query syntax — double-check filter format -``` - -### Backup Issues - -**Backup creation fails** -```bash -pc index describe -n my-index -# Backups are only supported for serverless indexes in "Ready" state -``` - -**Can't find backup ID** -```bash -pc backup list --index-name my-index -# Use the UUID (e.g. c84725e5-...) not the name for restore/delete -``` - ---- - -## Best Practices - -### Use the right auth method -- **Interactive dev**: `pc login` -- **CI/CD pipelines**: service accounts -- **Quick testing**: `pc api-key create -n "my-key" --store` - -### Check status before operating -```bash -pc auth status -pc target --show -pc index describe -n my-index -``` - -### Use JSON output for scripts -```bash -pc index list -j | jq -r '.[] | .name' -``` - -### Preview destructive operations -```bash -pc index vector update -n my-index \ - --filter '{"genre":{"$eq":"old"}}' \ - --metadata '{"genre":"new"}' \ - --dry-run -``` - -### Protect production indexes -```bash -pc index create -n prod-index -d 1536 -m cosine -c aws -r us-east-1 \ - --deletion-protection enabled -``` - -### Automate backups -```bash -pc backup create -i my-index -n "daily-backup-$(date +%Y%m%d)" -``` diff --git a/skills/pinecone-docs/SKILL.md b/skills/pinecone-docs/SKILL.md deleted file mode 100644 index 9a681c9..0000000 --- a/skills/pinecone-docs/SKILL.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -name: pinecone-docs -description: Curated documentation reference for developers building with Pinecone. Contains links to official docs organized by topic and data format references. Use when writing Pinecone code, looking up API parameters, or needing the correct format for vectors or records. ---- - -# Pinecone Developer Reference - -A curated index of Pinecone documentation. Fetch the relevant page(s) for the task at hand rather than relying on training data. - ---- - -## NOTE TO AGENT -Please attempt to fetch the url listed when relevant. If you run into an error, please attempt to append ".md" to the url to retrieve the markdown version of the Docs page. - -In case you need it: A full reference to ALL relevant URLs can be found here: https://docs.pinecone.io/llms.txt - -Use this as a last resort if you cannot find the relevant page below. - ---- - -## Getting Started - -| Topic | URL | -|---|---| -| Quickstart for all languages and coding environments (Cursor, Claude Code, n8n, Python, JavaScript, Java, Go, C#) | https://docs.pinecone.io/guides/get-started/quickstart | -| Pinecone concepts — namespaces, terminology, and key database concepts | https://docs.pinecone.io/guides/get-started/concepts | -| Data modeling for text and vectors | https://docs.pinecone.io/guides/index-data/data-modeling | -| Architecture of Pinecone | https://docs.pinecone.io/guides/get-started/database-architecture | -| Pinecone Assistant overview | https://docs.pinecone.io/guides/assistant/overview | - ---- - -## Indexes - -| Topic | URL | -|---|---| -| Create an index | https://docs.pinecone.io/guides/index-data/create-an-index | -| Index types and conceptual overview | https://docs.pinecone.io/guides/index-data/indexing-overview | -| Integrated inference (built-in embedding models) | https://docs.pinecone.io/guides/index-data/indexing-overview#integrated-embedding | -| Dedicated read nodes — predictable low-latency performance at high query volumes | https://docs.pinecone.io/guides/index-data/dedicated-read-nodes | - ---- - -## Upsert & Data - -| Topic | URL | -|---|---| -| Upsert vectors and text | https://docs.pinecone.io/guides/index-data/upsert-data | -| Multitenancy with namespaces | https://docs.pinecone.io/guides/index-data/implement-multitenancy | - ---- - -## Search - -| Topic | URL | -|---|---| -| Semantic search | https://docs.pinecone.io/guides/search/semantic-search | -| Hybrid search | https://docs.pinecone.io/guides/search/hybrid-search | -| Lexical search | https://docs.pinecone.io/guides/search/lexical-search | -| Full-text search (preview) — document-schema FTS indexes with `text` / `query_string` / dense / sparse scoring | https://docs.pinecone.io/guides/search/full-text-search | -| Metadata filtering — narrow results and speed up searches | https://docs.pinecone.io/guides/search/filter-by-metadata | - ---- - -## API & SDK Reference - -| Topic | URL | -|---|---| -| Python SDK reference | https://docs.pinecone.io/reference/sdks/python/overview | -| Example Colab notebooks | https://docs.pinecone.io/examples/notebooks | - ---- - -## Production - -| Topic | URL | -|---|---| -| Production checklist — preparing your index for production | https://docs.pinecone.io/guides/production/production-checklist | -| Common errors and what they mean | https://docs.pinecone.io/guides/production/error-handling | -| Targeting indexes correctly — don't use index names in prod | https://docs.pinecone.io/guides/manage-data/target-an-index#target-by-index-host-recommended | - ---- - -## Data Formats - -See [references/data-formats.md](references/data-formats.md) for vector and record schemas. diff --git a/skills/pinecone-docs/references/data-formats.md b/skills/pinecone-docs/references/data-formats.md deleted file mode 100644 index 2a55f45..0000000 --- a/skills/pinecone-docs/references/data-formats.md +++ /dev/null @@ -1,81 +0,0 @@ -# Data Formats - -## Integrated Index Records - -Used with `upsert_records()` (Python SDK) or `upsert-records` (MCP). Records are automatically embedded using the index's configured model. - -**JSON** -```json -[ - { - "_id": "rec1", - "chunk_text": "Your text content here.", - "category": "example" - }, - { - "_id": "rec2", - "chunk_text": "Another piece of text.", - "category": "example" - } -] -``` - -- `_id` — unique record identifier (required) -- The text field name must match the index's `fieldMap` (e.g. `chunk_text` if `fieldMap: {text: "chunk_text"}`) -- All other fields are stored as metadata and can be used for filtering -- Do **not** nest extra fields under a `metadata` key — put them directly on the record - ---- - -## Standard Index Vectors - -Used with `upsert()` (Python SDK) or `pc index vector upsert` (CLI). - -**JSON (with `vectors` array)** -```json -{ - "vectors": [ - { - "id": "vec1", - "values": [0.1, 0.2, 0.3], - "metadata": { "genre": "comedy", "year": 2021 } - }, - { - "id": "vec2", - "values": [0.4, 0.5, 0.6], - "metadata": { "genre": "drama", "year": 2019 } - } - ] -} -``` - -**JSONL (one vector per line)** -```jsonl -{"id": "vec1", "values": [0.1, 0.2, 0.3], "metadata": {"genre": "comedy"}} -{"id": "vec2", "values": [0.4, 0.5, 0.6], "metadata": {"genre": "drama"}} -``` - -- `id` — unique vector identifier (required) -- `values` — dense vector as float array, length must match index dimension (required) -- `metadata` — arbitrary key-value pairs for filtering (optional) - ---- - -## Sparse Vectors - -Used for keyword or hybrid search with sparse indexes. - -```json -{ - "id": "vec1", - "values": [0.1, 0.2, 0.3], - "sparse_values": { - "indices": [10, 45, 316], - "values": [0.5, 0.3, 0.8] - }, - "metadata": { "genre": "comedy" } -} -``` - -- `sparse_values.indices` — non-zero dimension indices -- `sparse_values.values` — corresponding float values, same length as `indices` diff --git a/skills/pinecone-help/SKILL.md b/skills/pinecone-help/SKILL.md deleted file mode 100644 index 38234bf..0000000 --- a/skills/pinecone-help/SKILL.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -name: pinecone-help -description: Overview of all available Pinecone skills and what a user needs to get started. Invoke when a user asks what skills are available, how to get started with Pinecone, or what they need to set up before using any Pinecone skill. ---- - -# Pinecone Skills — Help & Overview - -Pinecone is the leading vector database for building accurate and performant AI applications at scale in production. It's useful for building semantic search, retrieval augmented generation, recommendation systems, and agentic applications. - -Here's everything you need to get started and a summary of all available skills. - ---- - -## What You Need - -### Required -- **Pinecone account** — free to create at https://app.pinecone.io/?sessionType=signup -- **API key** — create one in the Pinecone console after signing up, then either export it in your terminal: - ```bash - export PINECONE_API_KEY="your-key" - ``` - Or add it to a `.env` file if your IDE doesn't inherit shell variables: `PINECONE_API_KEY=your-key` - -### Optional (unlock more capabilities) - -| Tool | What it enables | Install | -|---|---|---| -| **Pinecone MCP server** | Use Pinecone directly inside your AI agent/IDE without writing code | [Setup guide](https://docs.pinecone.io/guides/operations/mcp-server#tools) | -| **Pinecone CLI (`pc`)** | Manage all index types from the terminal, batch operations, backups, CI/CD | `brew tap pinecone-io/tap && brew install pinecone-io/tap/pinecone` | -| **uv** | Run the packaged Python scripts included in these skills | [Install uv](https://docs.astral.sh/uv/getting-started/installation/) | - ---- - -## Available Skills - -| Skill | What it does | -|---|---| -| `pinecone-quickstart` | Step-by-step onboarding — create an index, upload data, and run your first search | -| `pinecone-query` | Search integrated indexes using natural language text via the Pinecone MCP | -| `pinecone-cli` | Use the Pinecone CLI (`pc`) for terminal-based index and vector management | -| `pinecone-assistant` | Create, manage, and chat with Pinecone Assistants for document Q&A with citations | -| `pinecone-mcp` | Reference for all Pinecone MCP server tools and their parameters | -| `pinecone-full-text-search` | Build a full-text-search index — schema design, safe bulk ingestion, and query construction (`text` / `query_string` / dense / sparse scoring with text-match and metadata filters). **Preview API (`2026-01.alpha`); requires `pinecone` Python SDK ≥ 9.0.** | -| `pinecone-docs` | Curated links to official Pinecone documentation, organized by topic | - ---- - -## Which skill should I use? - -**Just getting started?** → `pinecone-quickstart` - -**Want to search an index you already have?** -- Integrated index (built-in embedding model) → `pinecone-query` (uses MCP) -- Any other index type → `pinecone-cli` - -**Working with documents and Q&A?** → `pinecone-assistant` - -**Building a full-text search index (BM25-style keyword/phrase matching, optionally combined with dense or sparse vectors)?** → `pinecone-full-text-search` (preview API, needs `pinecone` Python SDK ≥ 9.0) - -**Need to manage indexes, bulk upload vectors, or automate workflows?** → `pinecone-cli` - -**Looking up API parameters or SDK usage?** → `pinecone-docs` - -**Need to understand what MCP tools are available?** → `pinecone-mcp` diff --git a/skills/pinecone-mcp/SKILL.md b/skills/pinecone-mcp/SKILL.md deleted file mode 100644 index 5fbd7ae..0000000 --- a/skills/pinecone-mcp/SKILL.md +++ /dev/null @@ -1,106 +0,0 @@ ---- -name: pinecone-mcp -description: Reference for the Pinecone MCP server tools. Documents all available tools - list-indexes, describe-index, describe-index-stats, create-index-for-model, upsert-records, search-records, cascading-search, and rerank-documents. Use when an agent needs to understand what Pinecone MCP tools are available, how to use them, or what parameters they accept. ---- - -# Pinecone MCP Tools Reference - -The Pinecone MCP server exposes the following tools to AI agents and IDEs. For setup and installation instructions, see the [MCP server guide](https://docs.pinecone.io/guides/operations/mcp-server#tools). - -> **Key Limitation:** The Pinecone MCP only supports **integrated indexes** — indexes created with a built-in Pinecone embedding model. It does not work with standard indexes using external embedding models. For those, use the Pinecone CLI. - ---- - -## `list-indexes` - -List all indexes in the current Pinecone project. - ---- - -## `describe-index` - -Get configuration details for a specific index — cloud, region, dimension, metric, embedding model, field map, and status. - -**Parameters:** -- `name` (required) — Index name - ---- - -## `describe-index-stats` - -Get statistics for an index including total record count and per-namespace breakdown. - -**Parameters:** -- `name` (required) — Index name - ---- - -## `create-index-for-model` - -Create a new serverless index with an integrated embedding model. Pinecone handles embedding automatically — no external model needed. - -**Parameters:** -- `name` (required) — Index name -- `cloud` (required) — `aws`, `gcp`, or `azure` -- `region` (required) — Cloud region (e.g. `us-east-1`) -- `embed.model` (required) — Embedding model: `llama-text-embed-v2`, `multilingual-e5-large`, or `pinecone-sparse-english-v0` -- `embed.fieldMap.text` (required) — The record field that contains text to embed (e.g. `chunk_text`) - ---- - -## `upsert-records` - -Insert or update records in an integrated index. Records are automatically embedded using the index's configured model. - -**Parameters:** -- `name` (required) — Index name -- `namespace` (required) — Namespace to upsert into -- `records` (required) — Array of records. Each record must have an `id` or `_id` field and contain the text field specified in the index's `fieldMap`. Do not nest fields under `metadata` — put them directly on the record. - -**Example record:** -```json -{ "_id": "rec1", "chunk_text": "The Eiffel Tower was built in 1889.", "category": "architecture" } -``` - ---- - -## `search-records` - -Semantic text search against an integrated index. Pass plain text — the MCP embeds the query automatically using the index's model. - -**Parameters:** -- `name` (required) — Index name -- `namespace` (required) — Namespace to search -- `query.inputs.text` (required) — The text query -- `query.topK` (required) — Number of results to return -- `query.filter` (optional) — Metadata filter using MongoDB-style operators (`$eq`, `$ne`, `$in`, `$gt`, `$gte`, `$lt`, `$lte`) -- `rerank.model` (optional) — Reranking model: `bge-reranker-v2-m3`, `cohere-rerank-3.5`, or `pinecone-rerank-v0` -- `rerank.rankFields` (optional) — Fields to rerank on (e.g. `["chunk_text"]`) -- `rerank.topN` (optional) — Number of results to return after reranking - ---- - -## `cascading-search` - -Search across multiple indexes simultaneously, then deduplicate and rerank results into a single ranked list. - -**Parameters:** -- `indexes` (required) — Array of `{ name, namespace }` objects to search across -- `query.inputs.text` (required) — The text query -- `query.topK` (required) — Number of results to retrieve per index before reranking -- `rerank.model` (required) — Reranking model: `bge-reranker-v2-m3`, `cohere-rerank-3.5`, or `pinecone-rerank-v0` -- `rerank.rankFields` (required) — Fields to rerank on -- `rerank.topN` (optional) — Final number of results to return after reranking - ---- - -## `rerank-documents` - -Rerank a set of documents or records against a query without performing a vector search first. - -**Parameters:** -- `model` (required) — `bge-reranker-v2-m3`, `cohere-rerank-3.5`, or `pinecone-rerank-v0` -- `query` (required) — The query to rerank against -- `documents` (required) — Array of strings or records to rerank -- `options.topN` (required) — Number of results to return -- `options.rankFields` (optional) — If documents are records, the field(s) to rerank on diff --git a/skills/pinecone-query/SKILL.md b/skills/pinecone-query/SKILL.md deleted file mode 100644 index f73abff..0000000 --- a/skills/pinecone-query/SKILL.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -name: pinecone-query -description: Query integrated indexes using text with Pinecone MCP. IMPORTANT - This skill ONLY works with integrated indexes (indexes with built-in Pinecone embedding models like multilingual-e5-large). For standard indexes or advanced vector operations, use the CLI skill instead. Requires PINECONE_API_KEY environment variable and Pinecone MCP server to be configured. -argument-hint: query [q] index [indexName] namespace [ns] topK [k] reranker [rerankModel] ---- - -# Pinecone Query Skill - -Search for records in Pinecone integrated indexes using natural language text queries via the Pinecone MCP server. - -## What is this skill for? - -This skill provides a simple way to query **integrated indexes** (indexes with built-in Pinecone embedding models) using text queries. The MCP server automatically converts your text into embeddings and searches the index. - -### Prerequisites - -**Required:** -1. ✅ **Pinecone MCP server must be configured** - Check if MCP tools are available -2. ✅ **PINECONE_API_KEY environment variable must be set** - Get a free API key at https://app.pinecone.io/?sessionType=signup -3. ✅ **Index must be an integrated index** - Uses Pinecone embedding models (e.g., multilingual-e5-large, llama-text-embed-v2, pinecone-sparse-english-v0) - -### When NOT to use this skill - -**Use the CLI skill instead if:** -- ❌ Your index is a standard index (no integrated embedding model) -- ❌ You need to query with custom vector values (not text) -- ❌ You need advanced vector operations (fetch by ID, list vectors, bulk operations) -- ❌ Your index uses third-party embedding models (OpenAI, HuggingFace, Cohere) - -**MCP Limitation**: The Pinecone MCP currently only supports integrated indexes. For all other use cases, use the Pinecone CLI skill. - -## How it works - -Utilize Pinecone MCP's `search-records` tool to search for records within a specified Pinecone integrated index using a text query. - -## Workflow - -**IMPORTANT: Before proceeding, verify the Pinecone MCP tools are available.** If MCP tools are not accessible: -- Inform the user that the Pinecone MCP server needs to be configured -- Check if `PINECONE_API_KEY` environment variable is set -- Direct them to the MCP setup documentation or the `pinecone-help` skill - -1. Parse the user's input for: - - `query` (required): The text to search for. - - `index` (required): The name of the Pinecone index to search. - - `namespace` (optional): The namespace within the index. - - `reranker` (optional): The reranking model to use for improved relevance. - -2. If the user omits required arguments: - - If only the index name is provided, use the `describe-index` tool to retrieve available namespaces and ask the user to choose. - - If only a query is provided, use `list-indexes` to get available indexes, ask the user to pick one, then use `describe-index` for namespaces if needed. - -3. Call the `search-records` tool with the gathered arguments to perform the search. - -4. Format and display the returned results in a clear, readable table including field highlights (such as ID, score, and relevant metadata). - ---- - -## Troubleshooting - -**`PINECONE_API_KEY` is required.** Get a free key at https://app.pinecone.io/?sessionType=signup - -If you get an access error, the key is likely missing. Ask the user to set it and restart their IDE or agent session: -- Terminal: `export PINECONE_API_KEY="your-key"` -- IDE without shell inheritance: add `PINECONE_API_KEY=your-key` to a `.env` file - -**IMPORTANT** At the moment, the /query command can only be used with integrated indexes, which use hosted Pinecone embedding models to embed and search for data. -If a user attempts to query an index that uses a third party API model such as OpenAI, or HuggingFace embedding models, remind them that this capability is not available yet -with the Pinecone MCP server. - -- If required arguments are missing, prompt the user to supply them, using Pinecone MCP tools as needed (e.g., `list-indexes`, `describe-index`). -- Guide the user interactively through argument selection until the search can be completed. -- If an invalid value is provided for any argument (e.g., nonexistent index or namespace), surface the error and suggest valid options. - -## Tools Reference - -- `search-records`: Search records in a given index with optional metadata filtering and reranking. -- `list-indexes`: List all available Pinecone indexes. -- `describe-index`: Get index configuration and namespaces. -- `describe-index-stats`: Get stats including record counts and namespaces. -- `rerank-documents`: Rerank returned documents using a specified reranking model. -- Ask the user interactively to clarify missing information when needed. - ---- diff --git a/skills/pinecone-quickstart/SKILL.md b/skills/pinecone-quickstart/SKILL.md deleted file mode 100644 index 139cb4c..0000000 --- a/skills/pinecone-quickstart/SKILL.md +++ /dev/null @@ -1,230 +0,0 @@ ---- -name: pinecone-quickstart -description: Interactive Pinecone quickstart for new developers. Choose between two paths - Database (create an integrated index, upsert data, and query using Pinecone MCP + Python) or Assistant (create a Pinecone Assistant for document Q&A). Use when a user wants to get started with Pinecone for the first time or wants a guided tour of Pinecone's tools. ---- - -# Pinecone Quickstart - -Welcome! This skill walks you through your first Pinecone experience using the tools available to you. In this quickstart, -you will learn how to do a simple form of semantic search over some example data. - -## Prerequisites - -Before starting either path, verify the API key works by calling `list-indexes` via the Pinecone MCP. If it succeeds, proceed. If it fails, ask the user to set their key: - -- Terminal: `export PINECONE_API_KEY="your-key"` -- Or create a `.env` file in the project root: `PINECONE_API_KEY=your-key` - -Then retry `list-indexes` to confirm. - -## Step 0: Choose Your Path - -Ask the user which path they want: - -- **Database** – Build a vector search index. Best for developers who want to store and search embeddings. Uses the Pinecone MCP + a Python upsert script. -- **Assistant** – Build a document Q&A assistant. Best for users who want to upload files and ask questions with cited answers. No code required. - ---- - -## Path A: Database Quickstart - -For each step, explain to the user what will happen. An overview is here: - -1. Check if MCP is set -2. Create an integrated index with MCP -3. Upsert sample data using the bundled script (9 sentences across productivity, health, and nature themes) -4. Run a semantic search query and explore further queries -5. Optionally try reranking -6. Offer the complete standalone script - -### Step 1 – Verify MCP is Available - -The prerequisite check already called `list-indexes`. If it succeeded, the MCP is working — proceed to Step 2. - -If it failed because MCP tools were unavailable (not an auth error): -- Tell the user the MCP server needs to be configured -- Point them to: https://docs.pinecone.io/reference/tools/mcp - -### Step 2 – Create an Integrated Index - -Use the MCP `create-index-for-model` tool to create a serverless index with integrated embeddings: - -``` -name: quickstart-skills -cloud: aws -region: us-east-1 -embed: - model: llama-text-embed-v2 - fieldMap: - text: chunk_text -``` - -**Explain to the user what's happening:** -- An *integrated index* uses a built-in Pinecone embedding model (`llama-text-embed-v2`) -- This means you send plain text and Pinecone handles the embedding automatically -- The `field_map` tells Pinecone which field in your records contains the text to embed - -Wait for the index to become ready before proceeding. Waiting a few seconds is sufficient. - -### Step 3 – Upsert Sample Data - -Run the bundled upsert script to seed the index with sample records. - -If `PINECONE_API_KEY` is set in the environment: -```bash -uv run scripts/upsert.py --index quickstart-skills -``` - -If using a `.env` file: -```bash -uv run --env-file .env scripts/upsert.py --index quickstart-skills -``` - -**Explain to the user what's happening:** -- The script uploads 9 sample records across three themes: **productivity** (getting work done), **health** (feeling unwell), and **nature** (outdoors/wildlife) -- The dataset is intentionally varied so semantic search can show its value — the queries below use completely different words than the records, but the right ones still surface -- Each record has an `_id`, a `chunk_text` field (the text that gets embedded), and a `category` field -- This is the same structure you'd use for your own data — just replace the records - -### Step 4 – Query with the MCP - -Use the MCP `search-records` tool to run the first semantic search: - -``` -index: quickstart-skills -namespace: example-namespace -query: - topK: 3 - inputs: - text: "getting things done efficiently" -``` - -Display the results in a clean table: ID, score, and `chunk_text`. - -**Explain to the user what's happening:** -- Notice the query shares no keywords with the records — but it surfaces the productivity sentences -- That's semantic search: it finds meaning, not just matching words -- You sent plain text — Pinecone embedded the query using the same model as the index - -**Offer to explore further:** Ask the user if they'd like to try another query to see the effect more clearly: -- Option A: `"feeling under the weather"` — should surface the health records -- Option B: `"wildlife spotting outside"` — should surface the nature records -- Option C: No thanks, move on - -Run whichever query they choose and display the results the same way. If they want to try both, do both. After each result, point out which theme surfaced and why. - -If they decline or are done exploring, proceed to Step 5 or offer to skip ahead to the complete script. - -### Step 5 – Try Reranking (Optional) - -Ask the user if they want to try reranking. - -If yes, use `search-records` again with reranking enabled: - -``` -rerank: - model: bge-reranker-v2-m3 - rankFields: [chunk_text] - topN: 3 -``` - -**Explain**: Reranking runs a second-pass model over the results to improve relevance ordering. - -### Step 6 – Wrap Up - -Congratulate the user on completing the quickstart. Ask if they'd like a standalone Python script that does everything in one go — create index, upsert, query, and rerank. - -If yes, copy it to their working directory: - -```bash -cp scripts/quickstart_complete.py ./pinecone_quickstart.py -``` - -Tell the user: -- The script is at `./pinecone_quickstart.py` -- Run it with: `uv run pinecone_quickstart.py` -- It uses `uv` inline dependencies — no separate install needed -- They can swap in their own `records` list to build something real - ---- - -## Path B: Assistant Quickstart - -Guide the user through the Pinecone Assistant workflow using the existing assistant skills: - -### Step 1 – Check for Documents - -Before anything else, ask the user if they have files to upload. Pinecone Assistant accepts `.pdf`, `.md`, `.txt`, and `.docx` files — a single file or a folder of files both work. - -**If they have files:** ask for the path and proceed to Step 2. - -**If they don't have files:** offer two options: -- **Generate sample docs** — create a few short markdown files in `./sample-docs/` so they can complete the quickstart right now. Ask what topics they'd like (or default to: a product FAQ, a short how-to guide, and a brief company overview). Write 3 files, each 150–250 words. -- **Come back later** — let them know they can return once they have documents and pick up from Step 2. - -### Step 2 – Create an Assistant - -Invoke `pinecone-assistant` or run (add `--env-file .env` if using a `.env` file): -```bash -uv run ../pinecone-assistant/scripts/create.py --name my-assistant -``` - -Explain: The assistant is a fully managed RAG service — upload documents, ask questions, get cited answers. - -### Step 3 – Upload Documents - -Invoke `pinecone-assistant` or run (add `--env-file .env` if using a `.env` file): -```bash -uv run ../pinecone-assistant/scripts/upload.py --assistant my-assistant --source ./your-docs -``` - -Explain: Pinecone handles chunking, embedding, and indexing automatically — no configuration needed. - -### Step 4 – Chat with the Assistant - -Invoke `pinecone-assistant` or run (add `--env-file .env` if using a `.env` file): -```bash -uv run ../pinecone-assistant/scripts/chat.py --assistant my-assistant --message "What are the main topics in these documents?" -``` - -Explain: Responses include citations with source file and page number. - -### Next Steps for Assistant - -- Invoke `pinecone-assistant` to keep the assistant up to date as documents change -- Use the assistant skill to retrieve raw context snippets for custom workflows -- Every assistant is also an MCP server — see https://docs.pinecone.io/guides/assistant/mcp-server - ---- - -## Troubleshooting - -**`PINECONE_API_KEY` not set** - -Terminal environments: -```bash -export PINECONE_API_KEY="your-key" -``` -IDEs that don't inherit shell variables: create a `.env` file in the project root: -``` -PINECONE_API_KEY=your-key -``` -Then use `uv run --env-file .env` when running scripts. Restart your IDE/agent session after setting. - -**MCP tools not available** -- Verify the Pinecone MCP server is configured in your IDE's MCP settings -- Check that `PINECONE_API_KEY` is set before the MCP server starts - -**Index already exists** -- The upsert script is safe to re-run — it will upsert over existing records -- Or delete and recreate: use `pc index delete -n quickstart-skills` via the CLI - -**`uv` not installed** -See the [uv installation guide](https://docs.astral.sh/uv/getting-started/installation/). - -## Further Reading - -- Quickstart docs: https://docs.pinecone.io/guides/get-started/quickstart -- Integrated indexes: https://docs.pinecone.io/guides/index-data/create-an-index -- Python SDK: https://docs.pinecone.io/guides/get-started/python-sdk -- MCP server: https://docs.pinecone.io/reference/tools/mcp diff --git a/skills/pinecone-quickstart/scripts/quickstart_complete.py b/skills/pinecone-quickstart/scripts/quickstart_complete.py deleted file mode 100644 index c07643a..0000000 --- a/skills/pinecone-quickstart/scripts/quickstart_complete.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 -# /// script -# dependencies = [ -# "pinecone>=8.0.0", -# ] -# /// - -import os -from pinecone import Pinecone - -api_key = os.environ.get("PINECONE_API_KEY") -if not api_key: - raise ValueError("PINECONE_API_KEY environment variable not set") - -pc = Pinecone(api_key=api_key, source_tag="pinecone_skills:index_quickstart") - -# 1. Create a serverless index with an integrated embedding model -index_name = "quickstart" - -if not pc.has_index(index_name): - pc.create_index_for_model( - name=index_name, - cloud="aws", - region="us-east-1", - embed={ - "model": "llama-text-embed-v2", - "field_map": {"text": "chunk_text"} - } - ) - -# 2. Upsert records -# Three distinct themes — notice the queries below use different words than the records. -# That's semantic search: finding meaning, not just matching keywords. -records = [ - # Health / feeling unwell - {"_id": "rec1", "chunk_text": "I've been sneezing all day and my nose won't stop running.", "category": "health"}, - {"_id": "rec2", "chunk_text": "She stayed home with a pounding headache and a low-grade fever.", "category": "health"}, - {"_id": "rec3", "chunk_text": "He felt completely drained after waking up with a sore throat and chills.", "category": "health"}, - # Productivity / work - {"_id": "rec4", "chunk_text": "She blocked off two hours in the morning to focus without interruptions.", "category": "productivity"}, - {"_id": "rec5", "chunk_text": "He finished all his tasks ahead of schedule by prioritizing the hardest ones first.", "category": "productivity"}, - {"_id": "rec6", "chunk_text": "Turning off notifications helped her get into a deep flow state.", "category": "productivity"}, - # Outdoors / nature - {"_id": "rec7", "chunk_text": "A red fox darted across the trail and disappeared into the underbrush.", "category": "nature"}, - {"_id": "rec8", "chunk_text": "The hikers paused to watch a bald eagle circle lazily over the valley.", "category": "nature"}, - {"_id": "rec9", "chunk_text": "Fireflies lit up the meadow as the sun dipped below the treeline.", "category": "nature"}, -] - -dense_index = pc.Index(index_name) -dense_index.upsert_records("example-namespace", records) - -# 3. Search records -# The query uses different words than the records — semantic search finds meaning, not keywords. -query = "feeling ill and run down" - -results = dense_index.search( - namespace="example-namespace", - query={"top_k": 3, "inputs": {"text": query}} -) - -print("Search results:") -for hit in results["result"]["hits"]: - print(f" id: {hit['_id']} | score: {round(hit['_score'], 2)} | text: {hit['fields']['chunk_text']}") - -# 4. Search with reranking -reranked_results = dense_index.search( - namespace="example-namespace", - query={"top_k": 3, "inputs": {"text": query}}, - rerank={"model": "bge-reranker-v2-m3", "top_n": 3, "rank_fields": ["chunk_text"]} -) - -print("\nReranked results:") -for hit in reranked_results["result"]["hits"]: - print(f" id: {hit['_id']} | score: {round(hit['_score'], 2)} | text: {hit['fields']['chunk_text']}") diff --git a/skills/pinecone-quickstart/scripts/upsert.py b/skills/pinecone-quickstart/scripts/upsert.py deleted file mode 100644 index 1642bef..0000000 --- a/skills/pinecone-quickstart/scripts/upsert.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python3 -# /// script -# dependencies = [ -# "pinecone>=8.0.0", -# "typer>=0.15.0", -# ] -# /// - -import os -import typer -from pinecone import Pinecone - -app = typer.Typer() - -@app.command() -def main( - index: str = typer.Option(..., "--index", help="Name of the Pinecone index to upsert into"), - namespace: str = typer.Option("example-namespace", "--namespace", help="Namespace to upsert into"), -): - api_key = os.environ.get("PINECONE_API_KEY") - if not api_key: - typer.echo("Error: PINECONE_API_KEY environment variable not set", err=True) - raise typer.Exit(1) - - pc = Pinecone(api_key=api_key, source_tag="pinecone_skills:upsert") - - records = [ - # Health / feeling unwell - {"_id": "rec1", "chunk_text": "I've been sneezing all day and my nose won't stop running.", "category": "health"}, - {"_id": "rec2", "chunk_text": "She stayed home with a pounding headache and a low-grade fever.", "category": "health"}, - {"_id": "rec3", "chunk_text": "He felt completely drained after waking up with a sore throat and chills.", "category": "health"}, - # Productivity / work - {"_id": "rec4", "chunk_text": "She blocked off two hours in the morning to focus without interruptions.", "category": "productivity"}, - {"_id": "rec5", "chunk_text": "He finished all his tasks ahead of schedule by prioritizing the hardest ones first.", "category": "productivity"}, - {"_id": "rec6", "chunk_text": "Turning off notifications helped her get into a deep flow state.", "category": "productivity"}, - # Outdoors / nature - {"_id": "rec7", "chunk_text": "A red fox darted across the trail and disappeared into the underbrush.", "category": "nature"}, - {"_id": "rec8", "chunk_text": "The hikers paused to watch a bald eagle circle lazily over the valley.", "category": "nature"}, - {"_id": "rec9", "chunk_text": "Fireflies lit up the meadow as the sun dipped below the treeline.", "category": "nature"}, - ] - - idx = pc.Index(index) - idx.upsert_records(namespace, records) - typer.echo(f"Upserted {len(records)} records into '{index}' (namespace: '{namespace}')") - -if __name__ == "__main__": - app() diff --git a/skills/query/.gitkeep b/skills/query/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/skills/query/SKILL.md b/skills/query/SKILL.md index 612083a..3a8c546 100644 --- a/skills/query/SKILL.md +++ b/skills/query/SKILL.md @@ -4,6 +4,8 @@ description: Query integrated indexes using text with Pinecone MCP. IMPORTANT - argument-hint: query [q] index [indexName] namespace [ns] topK [k] reranker [rerankModel] --- +> Packaged as a Cursor plugin skill ([pinecone-io/pinecone-cursor-plugin](https://github.com/pinecone-io/pinecone-cursor-plugin)). + # Pinecone Query Skill Search for records in Pinecone integrated indexes using natural language text queries via the Pinecone MCP server. @@ -60,9 +62,11 @@ Utilize Pinecone MCP's `search-records` tool to search for records within a spec **`PINECONE_API_KEY` is required.** Get a free key at https://app.pinecone.io/?sessionType=signup -If you get an access error, the key is likely missing. Ask the user to add `PINECONE_API_KEY=your-key` to a `.env` file at their workspace root — the bundled MCP config loads it via Cursor's `envFile` field — then restart their IDE or agent session. For terminal-only scripts, `export PINECONE_API_KEY="your-key"` also works. +If you get an access error, the key is likely missing. Ask the user to set it and restart their IDE or agent session: +- Add `PINECONE_API_KEY=your-key` to a `.env` file at the workspace root (the bundled MCP configuration for this Cursor plugin loads it via Cursor's `envFile` setting) +- Terminal: `export PINECONE_API_KEY="your-key"` -**IMPORTANT** At the moment, the /query command can only be used with integrated indexes, which use hosted Pinecone embedding models to embed and search for data. +**IMPORTANT** In Cursor Agent chat, `/query` (this skill) works only with integrated indexes, which use hosted Pinecone embedding models to embed and search for data. If a user attempts to query an index that uses a third party API model such as OpenAI, or HuggingFace embedding models, remind them that this capability is not available yet with the Pinecone MCP server. diff --git a/skills/quickstart/.gitkeep b/skills/quickstart/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/skills/quickstart/SKILL.md b/skills/quickstart/SKILL.md index c3df901..eec7ef7 100644 --- a/skills/quickstart/SKILL.md +++ b/skills/quickstart/SKILL.md @@ -3,6 +3,8 @@ name: quickstart description: Interactive Pinecone quickstart for new developers. Choose between two paths - Database (create an integrated index, upsert data, and query using Pinecone MCP + Python) or Assistant (create a Pinecone Assistant for document Q&A). Use when a user wants to get started with Pinecone for the first time or wants a guided tour of Pinecone's tools. --- +> Packaged as a Cursor plugin skill ([pinecone-io/pinecone-cursor-plugin](https://github.com/pinecone-io/pinecone-cursor-plugin)). + # Pinecone Quickstart Welcome! This skill walks you through your first Pinecone experience using the tools available to you. In this quickstart, @@ -10,7 +12,7 @@ you will learn how to do a simple form of semantic search over some example data ## Prerequisites -Before starting either path, verify the API key works by calling `list-indexes` via the Pinecone MCP. If it succeeds, proceed. If it fails, ask the user to add `PINECONE_API_KEY=your-key` to a `.env` file at their workspace root (the bundled MCP config loads it via Cursor's `envFile` field) and restart their IDE/agent session. For terminal-only scripts, `export PINECONE_API_KEY="your-key"` also works. +Before starting either path, verify the API key works by calling `list-indexes` via the Pinecone MCP. If it succeeds, proceed. If it fails, ask the user to add `PINECONE_API_KEY=your-key` to a `.env` file at their workspace root (the bundled MCP configuration for this Cursor plugin loads it via Cursor's `envFile` setting) and restart the IDE/agent session. For terminal-only scripts, `export PINECONE_API_KEY="your-key"` also works. Then retry `list-indexes` to confirm. @@ -161,7 +163,7 @@ Before anything else, ask the user if they have files to upload. Pinecone Assist ### Step 2 – Create an Assistant -Invoke `assistant` or run (add `--env-file .env` if using a `.env` file): +Invoke the `assistant` skill (for example `/assistant` in Cursor Agent chat) or run (add `--env-file .env` if using a `.env` file): ```bash uv run ../assistant/scripts/create.py --name my-assistant ``` @@ -170,7 +172,7 @@ Explain: The assistant is a fully managed RAG service — upload documents, ask ### Step 3 – Upload Documents -Invoke `assistant` or run (add `--env-file .env` if using a `.env` file): +Invoke the `assistant` skill or run (add `--env-file .env` if using a `.env` file): ```bash uv run ../assistant/scripts/upload.py --assistant my-assistant --source ./your-docs ``` @@ -179,7 +181,7 @@ Explain: Pinecone handles chunking, embedding, and indexing automatically — no ### Step 4 – Chat with the Assistant -Invoke `assistant` or run (add `--env-file .env` if using a `.env` file): +Invoke the `assistant` skill or run (add `--env-file .env` if using a `.env` file): ```bash uv run ../assistant/scripts/chat.py --assistant my-assistant --message "What are the main topics in these documents?" ``` @@ -188,7 +190,7 @@ Explain: Responses include citations with source file and page number. ### Next Steps for Assistant -- Invoke `assistant` to keep the assistant up to date as documents change +- Invoke the `assistant` skill to keep the assistant up to date as documents change - Use the `assistant` skill to retrieve raw context snippets for custom workflows - Every assistant is also an MCP server — see https://docs.pinecone.io/guides/assistant/mcp-server @@ -202,7 +204,7 @@ Create a `.env` file at the workspace root: ``` PINECONE_API_KEY=your-key ``` -The bundled MCP config loads it via Cursor's `envFile` field. For running scripts directly, use `uv run --env-file .env scripts/...` (or `export PINECONE_API_KEY="your-key"` for terminal use). Restart your IDE/agent session after setting. +The bundled MCP configuration for this Cursor plugin loads it via Cursor's `envFile` setting. For running scripts directly, use `uv run --env-file .env scripts/...` (or `export PINECONE_API_KEY="your-key"` for terminal use). Restart your IDE/agent session after setting. **MCP tools not available** - Verify the Pinecone MCP server is configured in your IDE's MCP settings diff --git a/skills/quickstart/scripts/.gitkeep b/skills/quickstart/scripts/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/skills/quickstart/scripts/quickstart_complete.py b/skills/quickstart/scripts/quickstart_complete.py index 0404873..c07643a 100644 --- a/skills/quickstart/scripts/quickstart_complete.py +++ b/skills/quickstart/scripts/quickstart_complete.py @@ -12,7 +12,7 @@ if not api_key: raise ValueError("PINECONE_API_KEY environment variable not set") -pc = Pinecone(api_key=api_key, source_tag="cursor_plugin:index_quickstart") +pc = Pinecone(api_key=api_key, source_tag="pinecone_skills:index_quickstart") # 1. Create a serverless index with an integrated embedding model index_name = "quickstart" diff --git a/skills/quickstart/scripts/upsert.py b/skills/quickstart/scripts/upsert.py index ac428ee..1642bef 100644 --- a/skills/quickstart/scripts/upsert.py +++ b/skills/quickstart/scripts/upsert.py @@ -22,7 +22,7 @@ def main( typer.echo("Error: PINECONE_API_KEY environment variable not set", err=True) raise typer.Exit(1) - pc = Pinecone(api_key=api_key, source_tag="cursor_plugin:upsert") + pc = Pinecone(api_key=api_key, source_tag="pinecone_skills:upsert") records = [ # Health / feeling unwell