From 54a32588bb7f986412948f582d08b966a4a3fb53 Mon Sep 17 00:00:00 2001 From: Adam Quigley Date: Mon, 3 Aug 2026 14:34:16 -0400 Subject: [PATCH] SOLR-18267: Add scalar quantized flat vector index support --- .../SOLR-18267-scalar-quantized-flat.yml | 8 + .../apache/solr/schema/DenseVectorField.java | 17 +- .../ScalarQuantizedDenseVectorField.java | 19 +- .../apache/solr/search/vector/KnnQParser.java | 7 +- .../vector/VectorSimilarityQParser.java | 7 +- ...densevector-flat-scalarQuantized-byte.xml} | 8 +- ...chema-densevector-flat-scalarQuantized.xml | 31 +++ .../ScalarQuantizedDenseVectorFieldTest.java | 188 +++++++++++++++++- .../pages/dense-vector-search.adoc | 27 ++- 9 files changed, 277 insertions(+), 35 deletions(-) create mode 100644 changelog/unreleased/SOLR-18267-scalar-quantized-flat.yml rename solr/core/src/test-files/solr/collection1/conf/{bad-schema-densevector-flat-scalarQuantized.xml => bad-schema-densevector-flat-scalarQuantized-byte.xml} (75%) create mode 100644 solr/core/src/test-files/solr/collection1/conf/schema-densevector-flat-scalarQuantized.xml diff --git a/changelog/unreleased/SOLR-18267-scalar-quantized-flat.yml b/changelog/unreleased/SOLR-18267-scalar-quantized-flat.yml new file mode 100644 index 000000000000..b16cf1c46fd6 --- /dev/null +++ b/changelog/unreleased/SOLR-18267-scalar-quantized-flat.yml @@ -0,0 +1,8 @@ +title: Support knnAlgorithm="flat" for ScalarQuantizedDenseVectorField to store scalar-quantized vectors + without building an HNSW graph. +type: added +authors: +- name: Adam Quigley +links: +- name: SOLR-18267 + url: https://issues.apache.org/jira/browse/SOLR-18267 diff --git a/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java index 797b6bfc9a7e..0a53cafd401f 100644 --- a/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java +++ b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java @@ -499,6 +499,16 @@ public ValueSource getValueSource(SchemaField field, QParser parser) { SolrException.ErrorCode.BAD_REQUEST, "Vector encoding not supported for function queries."); } + /** Throws if this field type does not support KNN vector queries. */ + public void checkKnnQuerySupported() { + if (FLAT_ALGORITHM.equals(knnAlgorithm)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "KNN vector queries are not supported for fields using knnAlgorithm=\"flat\". " + + "Use vectorSimilarity() function queries instead."); + } + } + public Query getKnnVectorQuery( String fieldName, String vectorToSearch, @@ -509,12 +519,7 @@ public Query getKnnVectorQuery( EarlyTerminationParams earlyTermination, Integer filteredSearchThreshold) { - if (FLAT_ALGORITHM.equals(knnAlgorithm)) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "KNN vector queries are not supported for fields using knnAlgorithm=\"flat\". " - + "Use vectorSimilarity() function queries instead."); - } + checkKnnQuerySupported(); DenseVectorParser vectorBuilder = getVectorBuilder(vectorToSearch, DenseVectorParser.BuilderPhase.QUERY); diff --git a/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java index 26a33be48a24..80bd4fdfae0f 100644 --- a/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java +++ b/solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java @@ -22,6 +22,7 @@ import java.util.Map; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat; import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; @@ -123,18 +124,28 @@ public void init(IndexSchema schema, Map args) { super.init(schema, args); - if (FLAT_ALGORITHM.equals(getKnnAlgorithm())) { + if (VectorEncoding.BYTE.equals(getVectorEncoding()) + && FLAT_ALGORITHM.equals(getKnnAlgorithm())) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, - "knnAlgorithm 'flat' is not supported for ScalarQuantizedDenseVectorField"); + "vectorEncoding 'BYTE' is not supported for ScalarQuantizedDenseVectorField" + + " with knnAlgorithm 'flat'"); } } + // Unlike DenseVectorField, the scalar-quantized flat format supports KNN queries. + @Override + public void checkKnnQuerySupported() {} + @Override public KnnVectorsFormat buildKnnVectorsFormat() { ScalarEncoding encoding = ScalarEncoding.fromNumBits(getBits()); - return new Lucene104HnswScalarQuantizedVectorsFormat( - encoding, getHnswM(), getHnswEfConstruction()); + if (FLAT_ALGORITHM.equals(getKnnAlgorithm())) { + return new Lucene104ScalarQuantizedVectorsFormat(encoding); + } else { + return new Lucene104HnswScalarQuantizedVectorsFormat( + encoding, getHnswM(), getHnswEfConstruction()); + } } @Override diff --git a/solr/core/src/java/org/apache/solr/search/vector/KnnQParser.java b/solr/core/src/java/org/apache/solr/search/vector/KnnQParser.java index d376bfa5a40f..15bf22f5d85a 100644 --- a/solr/core/src/java/org/apache/solr/search/vector/KnnQParser.java +++ b/solr/core/src/java/org/apache/solr/search/vector/KnnQParser.java @@ -120,12 +120,7 @@ public Query parse() throws SyntaxError { final SchemaField schemaField = req.getCore().getLatestSchema().getField(getFieldName()); final DenseVectorField denseVectorType = getCheckedFieldType(schemaField); - if (DenseVectorField.FLAT_ALGORITHM.equals(denseVectorType.getKnnAlgorithm())) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "The {!knn} query parser is not supported for fields using knnAlgorithm=\"flat\". " - + "Use vectorSimilarity() function queries instead."); - } + denseVectorType.checkKnnQuerySupported(); final String vectorToSearch = getVectorToSearch(); final int topK = localParams.getInt(TOP_K, DEFAULT_TOP_K); diff --git a/solr/core/src/java/org/apache/solr/search/vector/VectorSimilarityQParser.java b/solr/core/src/java/org/apache/solr/search/vector/VectorSimilarityQParser.java index a88d1dfb3062..074240b48ca3 100644 --- a/solr/core/src/java/org/apache/solr/search/vector/VectorSimilarityQParser.java +++ b/solr/core/src/java/org/apache/solr/search/vector/VectorSimilarityQParser.java @@ -47,12 +47,7 @@ public Query parse() throws SyntaxError { final SchemaField schemaField = req.getCore().getLatestSchema().getField(fieldName); final DenseVectorField denseVectorType = getCheckedFieldType(schemaField); - if (DenseVectorField.FLAT_ALGORITHM.equals(denseVectorType.getKnnAlgorithm())) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "The {!vectorSimilarity} query parser is not supported for fields using knnAlgorithm=\"flat\". " - + "Use vectorSimilarity() function queries instead."); - } + denseVectorType.checkKnnQuerySupported(); final String vectorToSearch = getVectorToSearch(); final float minTraverse = localParams.getFloat(MIN_TRAVERSE, DEFAULT_MIN_TRAVERSE); diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-flat-scalarQuantized.xml b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-flat-scalarQuantized-byte.xml similarity index 75% rename from solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-flat-scalarQuantized.xml rename to solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-flat-scalarQuantized-byte.xml index 365bddd03e77..a1c8b84892b4 100644 --- a/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-flat-scalarQuantized.xml +++ b/solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-flat-scalarQuantized-byte.xml @@ -16,14 +16,14 @@ limitations under the License. --> - + - + - + - + id diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-densevector-flat-scalarQuantized.xml b/solr/core/src/test-files/solr/collection1/conf/schema-densevector-flat-scalarQuantized.xml new file mode 100644 index 000000000000..821a8df076c6 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-densevector-flat-scalarQuantized.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + id + diff --git a/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java b/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java index 94ebc63003be..396fbe55b8b8 100644 --- a/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java +++ b/solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java @@ -18,7 +18,11 @@ import static org.hamcrest.core.Is.is; +import java.util.ArrayList; +import java.util.List; +import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat; import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.solr.common.SolrInputDocument; import org.apache.solr.core.AbstractBadConfigTestBase; import org.junit.Test; @@ -31,6 +35,15 @@ public void fieldTypeDefinition_invalidBitSize_shouldThrowException() throws Exc "ScalarQuantizedDenseVectorField No encoding for 6 bits: v_scalar_bits"); } + @Test + public void fieldTypeDefinition_flatAlgorithm_byteEncoding_shouldThrowException() + throws Exception { + assertConfigs( + "solrconfig-basic.xml", + "bad-schema-densevector-flat-scalarQuantized-byte.xml", + "vectorEncoding 'BYTE' is not supported"); + } + @Test public void fieldDefinition_default_shouldLoadSchemaField() throws Exception { try { @@ -130,10 +143,175 @@ public void fieldDefinition_dynamicConfidenceInterval_shouldLoadSchemaField() th } @Test - public void fieldDefinition_flatAlgorithm_shouldThrowException() throws Exception { - assertConfigs( - "solrconfig-basic.xml", - "bad-schema-densevector-flat-scalarQuantized.xml", - "knnAlgorithm 'flat' is not supported for ScalarQuantizedDenseVectorField"); + public void fieldDefinition_flatAlgorithm_shouldLoadSchemaField() throws Exception { + try { + initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml"); + IndexSchema schema = h.getCore().getLatestSchema(); + + SchemaField vector = schema.getField("vector_sq_flat"); + assertNotNull(vector); + + ScalarQuantizedDenseVectorField type = (ScalarQuantizedDenseVectorField) vector.getType(); + assertThat(type.getKnnAlgorithm(), is("flat")); + assertThat(type.getDimension(), is(4)); + assertThat(type.getSimilarityFunction(), is(VectorSimilarityFunction.COSINE)); + assertThat(type.getBits(), is(ScalarQuantizedDenseVectorField.DEFAULT_BITS)); + + assertTrue(vector.indexed()); + assertTrue(vector.stored()); + } finally { + deleteCore(); + } + } + + @Test + public void flatAlgorithm_buildKnnVectorsFormat_shouldReturnScalarQuantizedFormat() + throws Exception { + try { + initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml"); + IndexSchema schema = h.getCore().getLatestSchema(); + + SchemaField vector = schema.getField("vector_sq_flat"); + ScalarQuantizedDenseVectorField type = (ScalarQuantizedDenseVectorField) vector.getType(); + + assertThat( + type.buildKnnVectorsFormat() instanceof Lucene104ScalarQuantizedVectorsFormat, is(true)); + } finally { + deleteCore(); + } + } + + @Test + public void flatAlgorithm_vectorSimilarityFunction_shouldReturnResults() throws Exception { + try { + initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml"); + + addDoc("0", 1.0f, 2.0f, 3.0f, 4.0f); + addDoc("1", 2.0f, 3.0f, 4.0f, 5.0f); + addDoc("2", 100.0f, 200.0f, 50.0f, 25.0f); + + assertU(commit()); + + assertJQ( + req( + "q", "{!func}vectorSimilarity(vector_sq_flat,[1.0, 2.0, 3.0, 4.0])", + "fl", "id,score"), + "/response/numFound==3", + "/response/docs/[0]/id=='0'"); + + assertJQ( + req( + "q", "{!func}vectorSimilarity(vector_sq_flat,[1.0, 2.0, 3.0, 4.0])", + "fq", "id:(0 2)", + "fl", "id,score"), + "/response/numFound==2", + "/response/docs/[0]/id=='0'"); + } finally { + deleteCore(); + } + } + + @Test + public void flatAlgorithm_knnQuery_shouldReturnResults() throws Exception { + try { + initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml"); + + addDoc("0", 1.0f, 2.0f, 3.0f, 4.0f); + addDoc("1", 2.0f, 3.0f, 4.0f, 5.0f); + addDoc("2", 100.0f, 200.0f, 50.0f, 25.0f); + + assertU(commit()); + + assertJQ( + req( + "q", "{!knn f=vector_sq_flat topK=2}[1.0, 2.0, 3.0, 4.0]", + "fl", "id,score"), + "/response/numFound==2", + "/response/docs/[0]/id=='0'", + "/response/docs/[1]/id=='1'"); + } finally { + deleteCore(); + } + } + + @Test + public void flatAlgorithm_knnQuery_preFilter_shouldReturnFilteredResults() throws Exception { + try { + initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml"); + + addDoc("0", 1.0f, 2.0f, 3.0f, 4.0f); + addDoc("1", 2.0f, 3.0f, 4.0f, 5.0f); + addDoc("2", 100.0f, 200.0f, 50.0f, 25.0f); + + assertU(commit()); + + assertJQ( + req( + "q", "{!knn f=vector_sq_flat topK=2 preFilter='id:(1 2)'}[1.0, 2.0, 3.0, 4.0]", + "fl", "id,score"), + "/response/numFound==2", + "/response/docs/[0]/id=='1'", + "/response/docs/[1]/id=='2'"); + } finally { + deleteCore(); + } + } + + @Test + public void flatAlgorithm_knnQuery_hnswParamsIgnored_shouldReturnResults() throws Exception { + try { + initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml"); + + addDoc("0", 1.0f, 2.0f, 3.0f, 4.0f); + addDoc("1", 2.0f, 3.0f, 4.0f, 5.0f); + + assertU(commit()); + + assertJQ( + req( + "q", + "{!knn f=vector_sq_flat topK=1 efSearchScaleFactor=2.0" + + " earlyTermination=true saturationThreshold=0.95 patience=3" + + " filteredSearchThreshold=60}[1.0, 2.0, 3.0, 4.0]", + "fl", + "id,score"), + "/response/numFound==1", + "/response/docs/[0]/id=='0'"); + } finally { + deleteCore(); + } + } + + @Test + public void flatAlgorithm_vectorSimilarityQParser_shouldReturnResults() throws Exception { + try { + initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml"); + + addDoc("0", 1.0f, 2.0f, 3.0f, 4.0f); + addDoc("1", 2.0f, 3.0f, 4.0f, 5.0f); + addDoc("2", 100.0f, 200.0f, 50.0f, 25.0f); + + assertU(commit()); + + assertJQ( + req( + "q", "{!vectorSimilarity f=vector_sq_flat minReturn=0.0}[1.0, 2.0, 3.0, 4.0]", + "fl", "id,score"), + "/response/numFound==3", + "/response/docs/[0]/id=='0'"); + } finally { + deleteCore(); + } + } + + private void addDoc(String id, float... v) { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", id); + List vector = new ArrayList<>(v.length); + for (float value : v) { + vector.add(value); + } + doc.addField("vector_sq_flat", vector); + assertU(adoc(doc)); } } diff --git a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc index c91a362f740f..07db12c4f5f3 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/dense-vector-search.adoc @@ -128,7 +128,7 @@ Here's how `DenseVectorField` can be configured with the advanced hyperparameter + Accepted values: `hnsw`, `flat`, `cagra_hnsw` (requires GPU acceleration setup). + -The `flat` algorithm stores vectors without building an HNSW graph. This avoids the indexing overhead of graph construction, but does not support the `{!knn}` query parser. Use `vectorSimilarity()` function queries to score and rank documents by vector similarity. See <> for details. +The `flat` algorithm stores vectors without building an HNSW graph, avoiding the indexing overhead of graph construction. Supported query parsers differ by field type. See <> for details. + Please note that the `knnAlgorithm` accepted values may change in future releases. @@ -300,8 +300,9 @@ with 8 bits we can store up to 256 discrete values, so a float dimension with va [0.0, 0.0039) => 0, [0.0039, 0.0078) => 1 ... etc -As a specific type of DenseVectorField, this field type supports all the same configurable properties outlined above as well -as some additional ones. +As a specific type of DenseVectorField, this field type supports `knnAlgorithm` (`hnsw` or `flat`) and other configurable properties outlined above, as well as some additional ones. + +NOTE: `vectorEncoding="BYTE"` is not supported for `ScalarQuantizedDenseVectorField`. Scalar quantization operates on `FLOAT32` vectors. Here is how a ScalarQuantizedDenseVectorField can be defined in the schema: @@ -348,6 +349,8 @@ BinaryQuantizedDenseVectorField accepts the same parameters as `DenseVectorField `similarityFunction`. Bit quantization uses its own distance calculation and so does not require nor use the `similarityFunction` param. +NOTE: `knnAlgorithm="flat"` is not supported for `BinaryQuantizedDenseVectorField`. + [[query-hnsw-fields]] == Querying Vectors in Navigable Small-world Graphs @@ -871,12 +874,28 @@ The final ranked list of results will have the first pass score(main query `q`) Setting `knnAlgorithm="flat"` stores vectors without building an HNSW graph, avoiding the indexing cost of graph construction. -Flat fields do not support the `{!knn}`, `{!knn_text_to_vector}`, or `{!vectorSimilarity}` query parsers. +The flat algorithm is supported by both `DenseVectorField` and `ScalarQuantizedDenseVectorField`. When used with `ScalarQuantizedDenseVectorField`, vectors are still quantized at index time. + +[source,xml] +---- + + + + + +---- + +For `DenseVectorField`, flat fields do not support the `{!knn}`, `{!knn_text_to_vector}`, or `{!vectorSimilarity}` query parsers. Use `vectorSimilarity()` function queries to score and rank by similarity: [source] q={!func}vectorSimilarity(vector,[1.0, 2.0, 3.0, 4.0])&fl=id,score +For `ScalarQuantizedDenseVectorField`, flat fields also support the `{!knn}`, `{!knn_text_to_vector}`, and `{!vectorSimilarity}` query parsers, which score against the quantized vectors. +Note that `vectorSimilarity()` function queries score against the raw (unquantized) float vectors, so the two paths can return slightly different scores for the same field. + == Indexing Multi-Vectors for Late Interaction