SOLR-18267: Add scalar quantized flat vector index support with no HNSW

changelog/unreleased/SOLR-18267-scalar-quantized-flat.yml

@@ -0,0 +1,8 @@
+title: Support knnAlgorithm="flat" for ScalarQuantizedDenseVectorField to store scalar-quantized vectors
+  without building an HNSW graph.
+type: added
+authors:
+- name: Adam Quigley
+links:
+- name: SOLR-18267
+  url: https://issues.apache.org/jira/browse/SOLR-18267

solr/core/src/java/org/apache/solr/schema/DenseVectorField.java

@@ -499,6 +499,16 @@ public ValueSource getValueSource(SchemaField field, QParser parser) {
         SolrException.ErrorCode.BAD_REQUEST, "Vector encoding not supported for function queries.");
   }
 
+  /** Throws if this field type does not support KNN vector queries. */
+  public void checkKnnQuerySupported() {
+    if (FLAT_ALGORITHM.equals(knnAlgorithm)) {
+      throw new SolrException(
+          SolrException.ErrorCode.BAD_REQUEST,
+          "KNN vector queries are not supported for fields using knnAlgorithm=\"flat\". "
+              + "Use vectorSimilarity() function queries instead.");
+    }
+  }
+
   public Query getKnnVectorQuery(
       String fieldName,
       String vectorToSearch,
@@ -509,12 +519,7 @@ public Query getKnnVectorQuery(
       EarlyTerminationParams earlyTermination,
       Integer filteredSearchThreshold) {
 
-    if (FLAT_ALGORITHM.equals(knnAlgorithm)) {
-      throw new SolrException(
-          SolrException.ErrorCode.BAD_REQUEST,
-          "KNN vector queries are not supported for fields using knnAlgorithm=\"flat\". "
-              + "Use vectorSimilarity() function queries instead.");
-    }
+    checkKnnQuerySupported();
 
     DenseVectorParser vectorBuilder =
         getVectorBuilder(vectorToSearch, DenseVectorParser.BuilderPhase.QUERY);

solr/core/src/java/org/apache/solr/schema/ScalarQuantizedDenseVectorField.java

@@ -22,6 +22,7 @@
 import java.util.Map;
 import org.apache.lucene.codecs.KnnVectorsFormat;
 import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;
+import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat;
 import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat.ScalarEncoding;
 import org.apache.lucene.index.VectorEncoding;
 import org.apache.lucene.index.VectorSimilarityFunction;
@@ -123,18 +124,28 @@ public void init(IndexSchema schema, Map<String, String> args) {
 
     super.init(schema, args);
 
-    if (FLAT_ALGORITHM.equals(getKnnAlgorithm())) {
+    if (VectorEncoding.BYTE.equals(getVectorEncoding())
+        && FLAT_ALGORITHM.equals(getKnnAlgorithm())) {
       throw new SolrException(
           SolrException.ErrorCode.BAD_REQUEST,
-          "knnAlgorithm 'flat' is not supported for ScalarQuantizedDenseVectorField");
+          "vectorEncoding 'BYTE' is not supported for ScalarQuantizedDenseVectorField"
+              + " with knnAlgorithm 'flat'");
     }
   }
 
+  // Unlike DenseVectorField, the scalar-quantized flat format supports KNN queries.
+  @Override
+  public void checkKnnQuerySupported() {}
+
   @Override
   public KnnVectorsFormat buildKnnVectorsFormat() {
     ScalarEncoding encoding = ScalarEncoding.fromNumBits(getBits());
-    return new Lucene104HnswScalarQuantizedVectorsFormat(
-        encoding, getHnswM(), getHnswEfConstruction());
+    if (FLAT_ALGORITHM.equals(getKnnAlgorithm())) {
+      return new Lucene104ScalarQuantizedVectorsFormat(encoding);
+    } else {
+      return new Lucene104HnswScalarQuantizedVectorsFormat(
+          encoding, getHnswM(), getHnswEfConstruction());
+    }
   }
 
   @Override

solr/core/src/java/org/apache/solr/search/vector/KnnQParser.java

@@ -120,12 +120,7 @@ public Query parse() throws SyntaxError {
     final SchemaField schemaField = req.getCore().getLatestSchema().getField(getFieldName());
     final DenseVectorField denseVectorType = getCheckedFieldType(schemaField);
 
-    if (DenseVectorField.FLAT_ALGORITHM.equals(denseVectorType.getKnnAlgorithm())) {
-      throw new SolrException(
-          SolrException.ErrorCode.BAD_REQUEST,
-          "The {!knn} query parser is not supported for fields using knnAlgorithm=\"flat\". "
-              + "Use vectorSimilarity() function queries instead.");
-    }
+    denseVectorType.checkKnnQuerySupported();
 
     final String vectorToSearch = getVectorToSearch();
     final int topK = localParams.getInt(TOP_K, DEFAULT_TOP_K);

solr/core/src/java/org/apache/solr/search/vector/VectorSimilarityQParser.java

@@ -47,12 +47,7 @@ public Query parse() throws SyntaxError {
     final SchemaField schemaField = req.getCore().getLatestSchema().getField(fieldName);
     final DenseVectorField denseVectorType = getCheckedFieldType(schemaField);
 
-    if (DenseVectorField.FLAT_ALGORITHM.equals(denseVectorType.getKnnAlgorithm())) {
-      throw new SolrException(
-          SolrException.ErrorCode.BAD_REQUEST,
-          "The {!vectorSimilarity} query parser is not supported for fields using knnAlgorithm=\"flat\". "
-              + "Use vectorSimilarity() function queries instead.");
-    }
+    denseVectorType.checkKnnQuerySupported();
 
     final String vectorToSearch = getVectorToSearch();
     final float minTraverse = localParams.getFloat(MIN_TRAVERSE, DEFAULT_MIN_TRAVERSE);

solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-flat-scalarQuantized.xml → solr/core/src/test-files/solr/collection1/conf/bad-schema-densevector-flat-scalarQuantized-byte.xml

@@ -16,14 +16,14 @@
  limitations under the License.
 -->
 
-<!-- Test schema file for DenseVectorField -->
+<!-- Test schema file for ScalarQuantizedDenseVectorField -->
 
-<schema name="bad-schema-densevector-flat-quantized" version="1.7">
+<schema name="bad-schema-densevector-flat-scalarQuantized-byte" version="1.7">
   <fieldType name="string" class="solr.StrField" multiValued="true"/>
-  <fieldType name="knn_vector_flat_sq" class="solr.ScalarQuantizedDenseVectorField" vectorDimension="4" similarityFunction="cosine" knnAlgorithm="flat"/>
+  <fieldType name="knn_vector_sq_byte" class="solr.ScalarQuantizedDenseVectorField" vectorDimension="4" similarityFunction="cosine" vectorEncoding="BYTE" knnAlgorithm="flat"/>
 
   <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
-  <field name="vector" type="knn_vector_flat_sq" indexed="true" stored="true"/>
+  <field name="vector_sq_byte" type="knn_vector_sq_byte" indexed="true" stored="true"/>
 
   <uniqueKey>id</uniqueKey>
 </schema>

solr/core/src/test-files/solr/collection1/conf/schema-densevector-flat-scalarQuantized.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Test schema file for ScalarQuantizedDenseVectorField with flat algorithm -->
+
+<schema name="schema-densevector-flat-scalarQuantized" version="1.7">
+  <fieldType name="string" class="solr.StrField" multiValued="true"/>
+  <fieldType name="knn_vector_flat_sq" class="solr.ScalarQuantizedDenseVectorField" vectorDimension="4" similarityFunction="cosine" knnAlgorithm="flat"/>
+  <fieldType name="knn_vector_flat_sq_4bit" class="solr.ScalarQuantizedDenseVectorField" vectorDimension="4" similarityFunction="cosine" knnAlgorithm="flat" bits="4"/>
+
+  <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+  <field name="vector_sq_flat" type="knn_vector_flat_sq" indexed="true" stored="true"/>
+  <field name="vector_sq_flat_4bit" type="knn_vector_flat_sq_4bit" indexed="true" stored="true"/>
+
+  <uniqueKey>id</uniqueKey>
+</schema>

solr/core/src/test/org/apache/solr/schema/ScalarQuantizedDenseVectorFieldTest.java

@@ -18,7 +18,11 @@
 
 import static org.hamcrest.core.Is.is;
 
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsFormat;
 import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.core.AbstractBadConfigTestBase;
 import org.junit.Test;
 
@@ -31,6 +35,15 @@ public void fieldTypeDefinition_invalidBitSize_shouldThrowException() throws Exc
         "ScalarQuantizedDenseVectorField No encoding for 6 bits: v_scalar_bits");
   }
 
+  @Test
+  public void fieldTypeDefinition_flatAlgorithm_byteEncoding_shouldThrowException()
+      throws Exception {
+    assertConfigs(
+        "solrconfig-basic.xml",
+        "bad-schema-densevector-flat-scalarQuantized-byte.xml",
+        "vectorEncoding 'BYTE' is not supported");
+  }
+
   @Test
   public void fieldDefinition_default_shouldLoadSchemaField() throws Exception {
     try {
@@ -130,10 +143,175 @@ public void fieldDefinition_dynamicConfidenceInterval_shouldLoadSchemaField() th
   }
 
   @Test
-  public void fieldDefinition_flatAlgorithm_shouldThrowException() throws Exception {
-    assertConfigs(
-        "solrconfig-basic.xml",
-        "bad-schema-densevector-flat-scalarQuantized.xml",
-        "knnAlgorithm 'flat' is not supported for ScalarQuantizedDenseVectorField");
+  public void fieldDefinition_flatAlgorithm_shouldLoadSchemaField() throws Exception {
+    try {
+      initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml");
+      IndexSchema schema = h.getCore().getLatestSchema();
+
+      SchemaField vector = schema.getField("vector_sq_flat");
+      assertNotNull(vector);
+
+      ScalarQuantizedDenseVectorField type = (ScalarQuantizedDenseVectorField) vector.getType();
+      assertThat(type.getKnnAlgorithm(), is("flat"));
+      assertThat(type.getDimension(), is(4));
+      assertThat(type.getSimilarityFunction(), is(VectorSimilarityFunction.COSINE));
+      assertThat(type.getBits(), is(ScalarQuantizedDenseVectorField.DEFAULT_BITS));
+
+      assertTrue(vector.indexed());
+      assertTrue(vector.stored());
+    } finally {
+      deleteCore();
+    }
+  }
+
+  @Test
+  public void flatAlgorithm_buildKnnVectorsFormat_shouldReturnScalarQuantizedFormat()
+      throws Exception {
+    try {
+      initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml");
+      IndexSchema schema = h.getCore().getLatestSchema();
+
+      SchemaField vector = schema.getField("vector_sq_flat");
+      ScalarQuantizedDenseVectorField type = (ScalarQuantizedDenseVectorField) vector.getType();
+
+      assertThat(
+          type.buildKnnVectorsFormat() instanceof Lucene104ScalarQuantizedVectorsFormat, is(true));
+    } finally {
+      deleteCore();
+    }
+  }
+
+  @Test
+  public void flatAlgorithm_vectorSimilarityFunction_shouldReturnResults() throws Exception {
+    try {
+      initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml");
+
+      addDoc("0", 1.0f, 2.0f, 3.0f, 4.0f);
+      addDoc("1", 2.0f, 3.0f, 4.0f, 5.0f);
+      addDoc("2", 100.0f, 200.0f, 50.0f, 25.0f);
+
+      assertU(commit());
+
+      assertJQ(
+          req(
+              "q", "{!func}vectorSimilarity(vector_sq_flat,[1.0, 2.0, 3.0, 4.0])",
+              "fl", "id,score"),
+          "/response/numFound==3",
+          "/response/docs/[0]/id=='0'");
+
+      assertJQ(
+          req(
+              "q", "{!func}vectorSimilarity(vector_sq_flat,[1.0, 2.0, 3.0, 4.0])",
+              "fq", "id:(0 2)",
+              "fl", "id,score"),
+          "/response/numFound==2",
+          "/response/docs/[0]/id=='0'");
+    } finally {
+      deleteCore();
+    }
+  }
+
+  @Test
+  public void flatAlgorithm_knnQuery_shouldReturnResults() throws Exception {
+    try {
+      initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml");
+
+      addDoc("0", 1.0f, 2.0f, 3.0f, 4.0f);
+      addDoc("1", 2.0f, 3.0f, 4.0f, 5.0f);
+      addDoc("2", 100.0f, 200.0f, 50.0f, 25.0f);
+
+      assertU(commit());
+
+      assertJQ(
+          req(
+              "q", "{!knn f=vector_sq_flat topK=2}[1.0, 2.0, 3.0, 4.0]",
+              "fl", "id,score"),
+          "/response/numFound==2",
+          "/response/docs/[0]/id=='0'",
+          "/response/docs/[1]/id=='1'");
+    } finally {
+      deleteCore();
+    }
+  }
+
+  @Test
+  public void flatAlgorithm_knnQuery_preFilter_shouldReturnFilteredResults() throws Exception {
+    try {
+      initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml");
+
+      addDoc("0", 1.0f, 2.0f, 3.0f, 4.0f);
+      addDoc("1", 2.0f, 3.0f, 4.0f, 5.0f);
+      addDoc("2", 100.0f, 200.0f, 50.0f, 25.0f);
+
+      assertU(commit());
+
+      assertJQ(
+          req(
+              "q", "{!knn f=vector_sq_flat topK=2 preFilter='id:(1 2)'}[1.0, 2.0, 3.0, 4.0]",
+              "fl", "id,score"),
+          "/response/numFound==2",
+          "/response/docs/[0]/id=='1'",
+          "/response/docs/[1]/id=='2'");
+    } finally {
+      deleteCore();
+    }
+  }
+
+  @Test
+  public void flatAlgorithm_knnQuery_hnswParamsIgnored_shouldReturnResults() throws Exception {
+    try {
+      initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml");
+
+      addDoc("0", 1.0f, 2.0f, 3.0f, 4.0f);
+      addDoc("1", 2.0f, 3.0f, 4.0f, 5.0f);
+
+      assertU(commit());
+
+      assertJQ(
+          req(
+              "q",
+              "{!knn f=vector_sq_flat topK=1 efSearchScaleFactor=2.0"
+                  + " earlyTermination=true saturationThreshold=0.95 patience=3"
+                  + " filteredSearchThreshold=60}[1.0, 2.0, 3.0, 4.0]",
+              "fl",
+              "id,score"),
+          "/response/numFound==1",
+          "/response/docs/[0]/id=='0'");
+    } finally {
+      deleteCore();
+    }
+  }
+
+  @Test
+  public void flatAlgorithm_vectorSimilarityQParser_shouldReturnResults() throws Exception {
+    try {
+      initCore("solrconfig_codec.xml", "schema-densevector-flat-scalarQuantized.xml");
+
+      addDoc("0", 1.0f, 2.0f, 3.0f, 4.0f);
+      addDoc("1", 2.0f, 3.0f, 4.0f, 5.0f);
+      addDoc("2", 100.0f, 200.0f, 50.0f, 25.0f);
+
+      assertU(commit());
+
+      assertJQ(
+          req(
+              "q", "{!vectorSimilarity f=vector_sq_flat minReturn=0.0}[1.0, 2.0, 3.0, 4.0]",
+              "fl", "id,score"),
+          "/response/numFound==3",
+          "/response/docs/[0]/id=='0'");
+    } finally {
+      deleteCore();
+    }
+  }
+
+  private void addDoc(String id, float... v) {
+    SolrInputDocument doc = new SolrInputDocument();
+    doc.addField("id", id);
+    List<Float> vector = new ArrayList<>(v.length);
+    for (float value : v) {
+      vector.add(value);
+    }
+    doc.addField("vector_sq_flat", vector);
+    assertU(adoc(doc));
   }
 }