diff --git a/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts b/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts index 00f0c766897..5bf56d07b57 100644 --- a/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts +++ b/packages/components/nodes/vectorstores/Pinecone/Pinecone.ts @@ -4,10 +4,14 @@ import { PineconeStoreParams, PineconeStore } from '@langchain/pinecone' import { Embeddings } from '@langchain/core/embeddings' import { Document } from '@langchain/core/documents' import { VectorStore } from '@langchain/core/vectorstores' +import { ScoreThresholdRetriever } from '@langchain/classic/retrievers/score_threshold' +import { ContextualCompressionRetriever } from '@langchain/classic/retrievers/contextual_compression' import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams, IndexingResult } from '../../../src/Interface' import { FLOWISE_CHATID, getBaseClasses, getCredentialData, getCredentialParam, parseJsonBody } from '../../../src/utils' -import { addMMRInputParams, howToUseFileUpload, resolveVectorStoreOrRetriever } from '../VectorStoreUtils' +import { howToUseFileUpload } from '../VectorStoreUtils' import { index } from '../../../src/indexing' +import { PineconeHybridRetriever, generateSparseVectorsBatch } from './PineconeHybridRetriever' +import { PineconeRerankCompressor } from './PineconeRerankCompressor' class Pinecone_VectorStores implements INode { label: string @@ -26,11 +30,11 @@ class Pinecone_VectorStores implements INode { constructor() { this.label = 'Pinecone' this.name = 'pinecone' - this.version = 5.0 + this.version = 6.0 this.type = 'Pinecone' this.icon = 'pinecone.svg' this.category = 'Vector Stores' - this.description = `Upsert embedded data and perform similarity or mmr search using Pinecone, a leading fully managed hosted vector database` + this.description = `Upsert embedded data and perform similarity, hybrid, or sparse search with optional reranking using Pinecone, a leading fully managed hosted vector database` this.baseClasses = [this.type, 'VectorStoreRetriever', 'BaseRetriever'] this.credential = { label: 'Connect Credential', @@ -108,9 +112,154 @@ class Pinecone_VectorStores implements INode { type: 'number', additionalParams: true, optional: true + }, + // ── Search Type ───────────────────────────────────────── + { + label: 'Search Type', + name: 'searchType', + type: 'options', + default: 'dense', + options: [ + { + label: 'Dense (Similarity)', + name: 'dense', + description: 'Standard semantic similarity search using dense embeddings' + }, + { + label: 'Max Marginal Relevance (MMR)', + name: 'mmr', + description: 'Balances relevance and diversity in results' + }, + { + label: 'Sparse (Keyword / BM25)', + name: 'sparse', + description: 'Pure keyword-based search using sparse vectors via Pinecone Inference' + }, + { + label: 'Hybrid (Dense + Sparse)', + name: 'hybrid', + description: 'Combines semantic and keyword search with configurable weighting' + } + ], + additionalParams: true, + optional: true + }, + // ── MMR Parameters ────────────────────────────────────── + { + label: 'Fetch K (for MMR Search)', + name: 'fetchK', + description: 'Number of initial documents to fetch for MMR reranking. Default to 20. Used only when the search type is MMR', + placeholder: '20', + type: 'number', + additionalParams: true, + optional: true, + show: { + searchType: 'mmr' + } + }, + { + label: 'Lambda (for MMR Search)', + name: 'lambda', + description: + 'Number between 0 and 1 that determines the degree of diversity among the results, where 0 corresponds to maximum diversity and 1 to minimum diversity. Used only when the search type is MMR', + placeholder: '0.5', + type: 'number', + additionalParams: true, + optional: true, + show: { + searchType: 'mmr' + } + }, + // ── Hybrid / Sparse Parameters ────────────────────────── + { + label: 'Sparse Embedding Model', + name: 'sparseModel', + description: + 'Pinecone Inference sparse embedding model used for keyword/BM25 encoding. Required for sparse and hybrid search types.', + type: 'string', + default: 'pinecone-sparse-english-v0', + placeholder: 'pinecone-sparse-english-v0', + additionalParams: true, + optional: true, + show: { + searchType: ['hybrid', 'sparse'] + } + }, + { + label: 'Alpha (Hybrid Search Weighting)', + name: 'alpha', + description: + 'Number between 0.0 and 1.0 that determines the weighting between dense and sparse search. ' + + '1.0 = pure dense (semantic), 0.0 = pure sparse (keyword). Default is 0.5 (equal weighting).', + placeholder: '0.5', + type: 'number', + step: 0.1, + additionalParams: true, + optional: true, + show: { + searchType: 'hybrid' + } + }, + // ── Similarity Threshold ──────────────────────────────── + { + label: 'Similarity Threshold', + name: 'similarityThreshold', + description: + 'Minimum similarity score to filter retrieved documents. ' + + 'Score range depends on your index metric (e.g. 0.0–1.0 for cosine). ' + + 'Leave empty to return all Top K results.', + type: 'number', + step: 0.05, + additionalParams: true, + optional: true + }, + // ── Reranking ─────────────────────────────────────────── + { + label: 'Reranking Model', + name: 'rerankModel', + description: + 'Apply Pinecone reranking to improve result quality after initial retrieval. ' + + 'Uses Pinecone Inference API — no extra credentials needed.', + type: 'options', + options: [ + { + label: 'None', + name: 'none', + description: 'No reranking applied' + }, + { + label: 'bge-reranker-v2-m3', + name: 'bge-reranker-v2-m3', + description: 'Multilingual BGE reranker model' + }, + { + label: 'pinecone-rerank-v0', + name: 'pinecone-rerank-v0', + description: 'Pinecone native reranking model' + }, + { + label: 'cohere-rerank-3.5', + name: 'cohere-rerank-3.5', + description: 'Cohere reranking model via Pinecone' + } + ], + default: 'none', + additionalParams: true, + optional: true + }, + { + label: 'Rerank Top N', + name: 'rerankTopN', + description: 'Number of results to return after reranking. Should be ≤ Top K. Defaults to Top K value if not specified.', + placeholder: '3', + type: 'number', + additionalParams: true, + optional: true, + hide: { + rerankModel: 'none' + } } ] - addMMRInputParams(this.inputs) this.outputs = [ { label: 'Pinecone Retriever', @@ -135,12 +284,13 @@ class Pinecone_VectorStores implements INode { const recordManager = nodeData.inputs?.recordManager const pineconeTextKey = nodeData.inputs?.pineconeTextKey as string const isFileUploadEnabled = nodeData.inputs?.fileUpload as boolean + const searchType = (nodeData.inputs?.searchType as string) || 'dense' + const sparseModel = (nodeData.inputs?.sparseModel as string) || 'pinecone-sparse-english-v0' const credentialData = await getCredentialData(nodeData.credential ?? '', options) const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData) const client = new Pinecone({ apiKey: pineconeApiKey }) - const pineconeIndex = client.Index(_index) const flattenDocs = docs && docs.length ? flatten(docs) : [] @@ -154,9 +304,74 @@ class Pinecone_VectorStores implements INode { } } + const textKey = pineconeTextKey || 'text' + + // ── Hybrid/Sparse upsert: store both dense + sparse vectors ── + if ((searchType === 'hybrid' || searchType === 'sparse') && finalDocs.length > 0) { + try { + const texts = finalDocs.map((doc) => doc.pageContent) + + // Always generate dense embeddings — Pinecone indexes require a dense vector for every record, + // and storing them ensures users can switch search types without re-upserting + const denseVectors = await embeddings.embedDocuments(texts) + + // Generate sparse embeddings via Pinecone Inference API + const sparseVectors = await generateSparseVectorsBatch(pineconeApiKey, sparseModel, texts) + + // Build Pinecone records with both dense and sparse vectors + const ns = pineconeNamespace ? pineconeIndex.namespace(pineconeNamespace) : pineconeIndex + + // Upsert in batches of 100 + const batchSize = 100 + for (let i = 0; i < finalDocs.length; i += batchSize) { + const batchDocs = finalDocs.slice(i, i + batchSize) + const records = batchDocs.map((doc, j) => { + const idx = i + j + // Sanitize metadata: Pinecone only allows string, number, boolean, or list of strings + const rawMeta: Record = { + ...doc.metadata, + [textKey]: doc.pageContent + } + const metadata: Record = {} + for (const [key, val] of Object.entries(rawMeta)) { + if (val === null || val === undefined) continue + if (typeof val === 'string' || typeof val === 'number' || typeof val === 'boolean') { + metadata[key] = val + } else if (Array.isArray(val) && val.every((v) => typeof v === 'string')) { + metadata[key] = val + } else if (typeof val === 'object') { + // Flatten nested objects to JSON string + metadata[key] = JSON.stringify(val) + } + } + + const record: Record = { + id: doc.id || doc.metadata?.id || `${Date.now()}-${idx}`, + metadata, + values: denseVectors[idx] + } + + // Add sparse vector + if (sparseVectors[idx]) { + record.sparseValues = sparseVectors[idx] + } + + return record + }) + + await (ns as any).upsert(records) + } + + return { numAdded: finalDocs.length, addedDocs: finalDocs } + } catch (e) { + throw new Error(e) + } + } + + // ── Standard dense-only upsert (unchanged from v5.0) ───────── const obj: PineconeStoreParams = { pineconeIndex, - textKey: pineconeTextKey || 'text' + textKey } if (pineconeNamespace) obj.namespace = pineconeNamespace @@ -231,23 +446,37 @@ class Pinecone_VectorStores implements INode { } async init(nodeData: INodeData, _: string, options: ICommonObject): Promise { - const index = nodeData.inputs?.pineconeIndex as string + const indexName = nodeData.inputs?.pineconeIndex as string const pineconeNamespace = nodeData.inputs?.pineconeNamespace as string const pineconeMetadataFilter = nodeData.inputs?.pineconeMetadataFilter const embeddings = nodeData.inputs?.embeddings as Embeddings const pineconeTextKey = nodeData.inputs?.pineconeTextKey as string const isFileUploadEnabled = nodeData.inputs?.fileUpload as boolean + // New feature inputs + const searchType = (nodeData.inputs?.searchType as string) || 'dense' + const topK = nodeData.inputs?.topK as string + const k = topK ? parseFloat(topK) : 4 + const similarityThreshold = nodeData.inputs?.similarityThreshold as string + const threshold = similarityThreshold ? parseFloat(similarityThreshold) : undefined + const rerankModel = (nodeData.inputs?.rerankModel as string) || 'none' + const rerankTopN = nodeData.inputs?.rerankTopN as string + const alpha = nodeData.inputs?.alpha as string + const sparseModel = (nodeData.inputs?.sparseModel as string) || 'pinecone-sparse-english-v0' + const fetchK = nodeData.inputs?.fetchK as string + const lambda = nodeData.inputs?.lambda as string + const credentialData = await getCredentialData(nodeData.credential ?? '', options) const pineconeApiKey = getCredentialParam('pineconeApiKey', credentialData, nodeData) const client = new Pinecone({ apiKey: pineconeApiKey }) + const pineconeIndex = client.Index(indexName) + const textKey = pineconeTextKey || 'text' - const pineconeIndex = client.Index(index) - + // Build metadata filter const obj: PineconeStoreParams = { pineconeIndex, - textKey: pineconeTextKey || 'text' + textKey } if (pineconeNamespace) obj.namespace = pineconeNamespace @@ -265,9 +494,87 @@ class Pinecone_VectorStores implements INode { ] } - const vectorStore = (await PineconeStore.fromExistingIndex(embeddings, obj)) as unknown as VectorStore + const output = nodeData.outputs?.output as string + const metadataFilter = obj.filter + + // ──────────────────────────────────────────────────────────── + // OUTPUT: Retriever + // ──────────────────────────────────────────────────────────── + if (output === 'retriever') { + let retriever - return resolveVectorStoreOrRetriever(nodeData, vectorStore, obj.filter) + if (searchType === 'sparse' || searchType === 'hybrid') { + // ── Sparse / Hybrid: use custom PineconeHybridRetriever ── + retriever = new PineconeHybridRetriever({ + client, + apiKey: pineconeApiKey, + indexName, + embeddings, + k, + alpha: alpha != null && alpha !== '' ? parseFloat(alpha) : 0.5, + sparseModel, + searchType: searchType as 'sparse' | 'hybrid', + filter: metadataFilter, + namespace: pineconeNamespace, + textKey + }) + } else { + // ── Dense / MMR: use LangChain PineconeStore ──────────── + const vectorStore = (await PineconeStore.fromExistingIndex(embeddings, obj)) as unknown as VectorStore + const filter = vectorStore?.lc_kwargs?.filter ? undefined : metadataFilter + + if (threshold !== undefined && threshold > 0) { + // ── Similarity Threshold Retriever ─────────────────── + retriever = ScoreThresholdRetriever.fromVectorStore(vectorStore, { + minSimilarityScore: threshold, + maxK: k, + kIncrement: 2, + filter + }) + } else if (searchType === 'mmr') { + // ── MMR Search ────────────────────────────────────── + const f = fetchK ? parseInt(fetchK) : 20 + const l = lambda != null && lambda !== '' ? parseFloat(lambda) : 0.5 + retriever = vectorStore.asRetriever({ + searchType: 'mmr', + k, + filter, + searchKwargs: { + fetchK: f, + lambda: l + } + }) + } else { + // ── Standard Dense (Similarity) Search ────────────── + retriever = vectorStore.asRetriever({ + k, + filter + }) + } + } + + // ── Apply Reranking (wraps any retriever type) ────────── + if (rerankModel && rerankModel !== 'none') { + const topN = rerankTopN ? parseInt(rerankTopN) : k + const compressor = new PineconeRerankCompressor(client, rerankModel, topN) + return new ContextualCompressionRetriever({ + baseCompressor: compressor, + baseRetriever: retriever + }) + } + + return retriever + } + + // ──────────────────────────────────────────────────────────── + // OUTPUT: Vector Store + // ──────────────────────────────────────────────────────────── + if (output === 'vectorStore') { + const vectorStore = (await PineconeStore.fromExistingIndex(embeddings, obj)) as unknown as VectorStore + ;(vectorStore as any).k = k + ;(vectorStore as any).filter = metadataFilter + return vectorStore + } } } diff --git a/packages/components/nodes/vectorstores/Pinecone/PineconeHybridRetriever.ts b/packages/components/nodes/vectorstores/Pinecone/PineconeHybridRetriever.ts new file mode 100644 index 00000000000..af4babaca34 --- /dev/null +++ b/packages/components/nodes/vectorstores/Pinecone/PineconeHybridRetriever.ts @@ -0,0 +1,220 @@ +import type { Pinecone } from '@pinecone-database/pinecone' +import axios from 'axios' +import { BaseRetriever } from '@langchain/core/retrievers' +import { CallbackManagerForRetrieverRun } from '@langchain/core/callbacks/manager' +import { Embeddings } from '@langchain/core/embeddings' +import { Document } from '@langchain/core/documents' + +export interface PineconeHybridRetrieverConfig { + client: Pinecone + apiKey: string + indexName: string + embeddings: Embeddings + k: number + alpha: number + sparseModel: string + searchType: 'dense' | 'sparse' | 'hybrid' + filter?: Record + namespace?: string + textKey: string +} + +/** + * A custom LangChain BaseRetriever that performs hybrid (dense + sparse), + * pure sparse, or pure dense search using the raw Pinecone SDK. + * + * - Dense vectors come from the connected Embeddings node. + * - Sparse vectors are generated via Pinecone's inference.embed() API + * using the configured sparse model (e.g. pinecone-sparse-english-v0). + * - Alpha (0.0–1.0) controls the weighting: 1.0 = pure dense, 0.0 = pure sparse. + */ +export class PineconeHybridRetriever extends BaseRetriever { + lc_namespace = ['flowise', 'retrievers', 'pinecone-hybrid'] + + private client: Pinecone + private apiKey: string + private indexName: string + private embeddings: Embeddings + private k: number + private alpha: number + private sparseModel: string + private searchType: 'dense' | 'sparse' | 'hybrid' + private filter?: Record + private namespace?: string + private textKey: string + + constructor(config: PineconeHybridRetrieverConfig) { + super() + this.client = config.client + this.apiKey = config.apiKey + this.indexName = config.indexName + this.embeddings = config.embeddings + this.k = config.k + this.alpha = config.alpha + this.sparseModel = config.sparseModel + this.searchType = config.searchType + this.filter = config.filter + this.namespace = config.namespace + this.textKey = config.textKey + } + + async _getRelevantDocuments(query: string, _runManager?: CallbackManagerForRetrieverRun): Promise { + const pineconeIndex = this.client.Index(this.indexName) + const index = this.namespace ? pineconeIndex.namespace(this.namespace) : pineconeIndex + + let queryResponse + + if (this.searchType === 'dense') { + // Pure dense search — only use the embeddings model + const vector = await this.embeddings.embedQuery(query) + queryResponse = await index.query({ + topK: this.k, + includeMetadata: true, + vector, + ...(this.filter ? { filter: this.filter } : {}) + }) + } else if (this.searchType === 'sparse') { + // Pure sparse search — combine dense + sparse with alpha = 0 to avoid dimension mismatch + const [denseVector, rawSparseVector] = await Promise.all([this.embeddings.embedQuery(query), this.generateSparseVector(query)]) + const vector = denseVector.map(() => 0) + const sparseVector = { + indices: rawSparseVector.indices, + values: rawSparseVector.values + } + queryResponse = await index.query({ + topK: this.k, + includeMetadata: true, + vector, + sparseVector, + ...(this.filter ? { filter: this.filter } : {}) + }) + } else { + // Hybrid search — combine dense + sparse with alpha weighting + const [denseVector, rawSparseVector] = await Promise.all([this.embeddings.embedQuery(query), this.generateSparseVector(query)]) + + // Apply alpha weighting: alpha=1.0 → pure dense, alpha=0.0 → pure sparse + const vector = denseVector.map((v: number) => v * this.alpha) + const sparseVector = { + indices: rawSparseVector.indices, + values: rawSparseVector.values.map((v: number) => v * (1 - this.alpha)) + } + + queryResponse = await index.query({ + topK: this.k, + includeMetadata: true, + vector, + sparseVector, + ...(this.filter ? { filter: this.filter } : {}) + }) + } + + // Convert Pinecone matches to LangChain Documents + const documents: Document[] = [] + for (const match of queryResponse.matches || []) { + const metadata: Record = { ...(match.metadata || {}) } + const pageContent = (metadata[this.textKey] as string) || '' + delete metadata[this.textKey] + metadata.score = match.score + documents.push(new Document({ pageContent, metadata })) + } + + return documents + } + + /** + * Generate a sparse vector for the given text using Pinecone's inference API. + */ + /** + * Generate a sparse vector for the given text using Pinecone's REST API directly. + * SDK v4 doesn't support sparse embeddings, so we call the REST endpoint. + */ + private async generateSparseVector(text: string): Promise<{ indices: number[]; values: number[] }> { + const result = await callPineconeEmbedApi(this.apiKey, this.sparseModel, [text], 'query') + return result[0] + } +} + +/** + * Call the Pinecone Inference embed REST API directly. + * SDK v4.0.0 doesn't expose sparse embedding values, so we bypass it. + */ +async function callPineconeEmbedApi( + apiKey: string, + model: string, + inputs: string[], + inputType: 'passage' | 'query' +): Promise> { + if (!apiKey) { + throw new Error('Pinecone API key is required for sparse embedding call.') + } + + const body = { + model, + inputs: inputs.map((text) => ({ text })), + parameters: { input_type: inputType, return_type: 'sparse' } + } + + let response + try { + response = await axios.post('https://api.pinecone.io/embed', body, { + headers: { + 'Content-Type': 'application/json', + 'Api-Key': apiKey, + 'X-Pinecone-API-Version': '2025-01' + } + }) + } catch (error: any) { + const errorText = error.response?.data ? JSON.stringify(error.response.data) : error.message + throw new Error( + 'Pinecone embed API returned error: ' + + errorText + + '. ' + + "Model: '" + + model + + "'. Ensure the model supports sparse encoding and is available on your Pinecone plan." + ) + } + const json = response.data + const data = json.data ?? json + + const results: Array<{ indices: number[]; values: number[] }> = [] + for (let i = 0; i < inputs.length; i++) { + const embedding = data?.[i] + // Pinecone REST API returns sparse_indices and sparse_values as separate flat arrays + const indices = embedding?.sparse_indices + const values = embedding?.sparse_values + + if (!indices || !values || !Array.isArray(indices) || !Array.isArray(values)) { + throw new Error( + `Sparse embedding model '${model}' did not return sparse values for input at index ${i}. ` + + `Response: ${JSON.stringify(embedding)?.substring(0, 300)}` + ) + } + + results.push({ indices, values }) + } + + return results +} + +/** + * Generate sparse vectors for a batch of texts using Pinecone's REST API. + * Used during upsert to store sparse vectors alongside dense vectors. + */ +export async function generateSparseVectorsBatch( + apiKey: string, + sparseModel: string, + texts: string[] +): Promise> { + // Process in batches of 96 (Pinecone inference API limit) + const batchSize = 96 + const allSparseVectors: Array<{ indices: number[]; values: number[] }> = [] + + for (let i = 0; i < texts.length; i += batchSize) { + const batch = texts.slice(i, i + batchSize) + const batchResults = await callPineconeEmbedApi(apiKey, sparseModel, batch, 'passage') + allSparseVectors.push(...batchResults) + } + + return allSparseVectors +} diff --git a/packages/components/nodes/vectorstores/Pinecone/PineconeRerankCompressor.ts b/packages/components/nodes/vectorstores/Pinecone/PineconeRerankCompressor.ts new file mode 100644 index 00000000000..622e4d83766 --- /dev/null +++ b/packages/components/nodes/vectorstores/Pinecone/PineconeRerankCompressor.ts @@ -0,0 +1,70 @@ +import { Pinecone } from '@pinecone-database/pinecone' +import { Callbacks } from '@langchain/core/callbacks/manager' +import { Document } from '@langchain/core/documents' +import { BaseDocumentCompressor } from '@langchain/classic/retrievers/document_compressors' + +/** + * A BaseDocumentCompressor that re-scores documents using Pinecone's + * inference.rerank() API. This is used as the compressor inside a + * ContextualCompressionRetriever to apply reranking after initial retrieval. + * + * Uses the user's existing Pinecone API key — no extra credentials needed. + */ +export class PineconeRerankCompressor extends BaseDocumentCompressor { + private client: Pinecone + private model: string + private topN: number + + constructor(client: Pinecone, model: string, topN: number) { + super() + this.client = client + this.model = model + this.topN = topN + } + + async compressDocuments( + documents: Document>[], + query: string, + _?: Callbacks | undefined + ): Promise>[]> { + // Avoid empty API calls + if (documents.length === 0) { + return [] + } + + try { + const result = await this.client.inference.rerank( + this.model, + query, + documents.map((doc) => doc.pageContent), + { topN: this.topN, returnDocuments: false } + ) + + const rerankData = (result as any)?.data ?? result + const finalResults: Document>[] = [] + + for (const item of rerankData) { + const idx = (item as any).index + const score = (item as any).score + if (idx !== undefined && idx < documents.length) { + const doc = documents[idx] + finalResults.push( + new Document({ + pageContent: doc.pageContent, + metadata: { + ...doc.metadata, + relevance_score: score + } + }) + ) + } + } + + return finalResults.slice(0, this.topN) + } catch (error) { + // On reranking failure, return original documents (graceful degradation) + console.warn('Pinecone reranking failed, returning original documents:', error) + return documents + } + } +}