DRAFT - Refactor Collection Table Matcher

pom.xml

@@ -401,6 +401,10 @@
         <groupId>com.spotify.fmt</groupId>
         <artifactId>fmt-maven-plugin</artifactId>
         <version>2.23</version>
+        <configuration>
+          <!-- Skip the SuperShreddingMetadata java file because it is nicely formatted for readability -->
+          <filesPathPattern>^((?!SuperShreddingMetadata).)*\.java$</filesPathPattern>
+        </configuration>
         <executions>
           <execution>
             <phase>process-sources</phase>

src/main/java/io/stargate/sgv2/jsonapi/config/constants/DocumentConstants.java

@@ -36,44 +36,6 @@ interface Fields {
     String SCORES_FIELD = "scores";
   }
 
-  /** Names of columns in Document-containing Tables */
-  interface Columns {
-    /**
-     * Atomic values are added to the array_contains field to support $eq on both atomic value and
-     * array element
-     */
-    String DATA_CONTAINS_COLUMN_NAME = "array_contains";
-
-    String QUERY_BOOLEAN_MAP_COLUMN_NAME = "query_bool_values";
-
-    String QUERY_DOUBLE_MAP_COLUMN_NAME = "query_dbl_values";
-
-    String QUERY_NULL_MAP_COLUMN_NAME = "query_null_values";
-
-    /** Text map support _id $ne and _id $nin on both atomic value and array element */
-    String QUERY_TEXT_MAP_COLUMN_NAME = "query_text_values";
-
-    String QUERY_TIMESTAMP_MAP_COLUMN_NAME = "query_timestamp_values";
-
-    /** Physical table column name that stores the vector field. */
-    String VECTOR_SEARCH_INDEX_COLUMN_NAME = "query_vector_value";
-
-    /** Document field name to which vector data is stored. */
-    String VECTOR_EMBEDDING_FIELD = "$vector";
-
-    /** Document field name that will have text value for which vectorize method in called */
-    String VECTOR_EMBEDDING_TEXT_FIELD = "$vectorize";
-
-    /** Document field name that will have text value for which vectorize method in called */
-    String BINARY_VECTOR_TEXT_FIELD = "$binary";
-
-    /** Field name used in projection clause to get similarity score in response. */
-    String VECTOR_FUNCTION_SIMILARITY_FIELD = "$similarity";
-
-    /** Physical table column name that stores the lexical content. */
-    String LEXICAL_INDEX_COLUMN_NAME = "query_lexical_value";
-  }
-
   interface KeyTypeId {
     /**
      * Type id are used in key stored in database representing the datatype of the id field. These

src/main/java/io/stargate/sgv2/jsonapi/exception/ErrorFormatters.java

@@ -123,7 +123,8 @@ public static String errFmt(ApiDataType apiDataType) {
   }
 
   public static String errFmt(DataType dataType) {
-    return nullSafe(dataType, d -> d.asCql(true, true));
+    // pass false for includeFrozen to avoid using frozen<> when not needed.
+    return nullSafe(dataType, d -> d.asCql(false, true));
   }
 
   public static Map<String, String> errVars(SchemaObject schemaObject) {

src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/executor/CollectionIndexUsage.java

@@ -3,7 +3,7 @@
 import com.google.common.base.Preconditions;
 import io.micrometer.core.instrument.Tag;
 import io.micrometer.core.instrument.Tags;
-import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants;
+import io.stargate.sgv2.jsonapi.service.schema.collections.spec.SuperShreddingMetadata;
 
 /**
  * This class is used to track the usage of indexes in a query. It is used to generate metrics for
@@ -34,23 +34,15 @@ public Tags getTags() {
         Tag.of("key", String.valueOf(primaryKeyTag)),
         Tag.of("exist_keys", String.valueOf(existKeysIndexTag)),
         Tag.of("array_size", String.valueOf(arraySizeIndexTag)),
+        Tag.of(SuperShreddingMetadata.Names.ARRAY_CONTAINS, String.valueOf(arrayContainsTag)),
+        Tag.of(SuperShreddingMetadata.Names.QUERY_BOOLEAN_VALUES, String.valueOf(booleanIndexTag)),
+        Tag.of(SuperShreddingMetadata.Names.QUERY_DOUBLE_VALUES, String.valueOf(numberIndexTag)),
+        Tag.of(SuperShreddingMetadata.Names.QUERY_NULL_VALUES, String.valueOf(nullIndexTag)),
+        Tag.of(SuperShreddingMetadata.Names.QUERY_TEXT_VALUES, String.valueOf(textIndexTag)),
         Tag.of(
-            DocumentConstants.Columns.DATA_CONTAINS_COLUMN_NAME, String.valueOf(arrayContainsTag)),
-        Tag.of(
-            DocumentConstants.Columns.QUERY_BOOLEAN_MAP_COLUMN_NAME,
-            String.valueOf(booleanIndexTag)),
-        Tag.of(
-            DocumentConstants.Columns.QUERY_DOUBLE_MAP_COLUMN_NAME, String.valueOf(numberIndexTag)),
-        Tag.of(DocumentConstants.Columns.QUERY_NULL_MAP_COLUMN_NAME, String.valueOf(nullIndexTag)),
-        Tag.of(DocumentConstants.Columns.QUERY_TEXT_MAP_COLUMN_NAME, String.valueOf(textIndexTag)),
-        Tag.of(
-            DocumentConstants.Columns.QUERY_TIMESTAMP_MAP_COLUMN_NAME,
-            String.valueOf(timestampIndexTag)),
-        Tag.of(
-            DocumentConstants.Columns.VECTOR_SEARCH_INDEX_COLUMN_NAME,
-            String.valueOf(vectorIndexTag)),
-        Tag.of(
-            DocumentConstants.Columns.LEXICAL_INDEX_COLUMN_NAME, String.valueOf(lexicalIndexTag)));
+            SuperShreddingMetadata.Names.QUERY_TIMESTAMP_VALUES, String.valueOf(timestampIndexTag)),
+        Tag.of(SuperShreddingMetadata.Names.QUERY_VECTOR_VALUE, String.valueOf(vectorIndexTag)),
+        Tag.of(SuperShreddingMetadata.Names.QUERY_LEXICAL_VALUE, String.valueOf(lexicalIndexTag)));
   }
 
   /**

src/main/java/io/stargate/sgv2/jsonapi/service/cqldriver/override/ExtendedVectorType.java

@@ -4,8 +4,9 @@
 import com.datastax.oss.driver.internal.core.type.DefaultVectorType;
 
 /**
- * Extended vector type to support vector size This is needed because java drivers
- * DataTypes.vectorOf() method has a bug
+ * Extended vector type to support vector size.
+ *
+ * <p>Basically a clone of {@link DefaultVectorType} but changes the {@link #asCql} override.
  */
 public class ExtendedVectorType extends DefaultVectorType {
   public ExtendedVectorType(DataType subtype, int vectorSize) {
@@ -14,6 +15,10 @@ public ExtendedVectorType(DataType subtype, int vectorSize) {
 
   @Override
   public String asCql(boolean includeFrozen, boolean pretty) {
-    return "VECTOR<" + getElementType().asCql(includeFrozen, pretty) + "," + getDimensions() + ">";
+    // NOTE: this is very similar to the DefaultVectorType.asCql() method, the difference
+    // is passing along the includeFrozen and pretty parameters. Default sets them to true
+    // which means frozen is included in places we dont want it.
+    return String.format(
+        "vector<%s, %d>", getElementType().asCql(includeFrozen, pretty), getDimensions());
   }
 }

src/main/java/io/stargate/sgv2/jsonapi/service/operation/MetadataDBTask.java

@@ -16,7 +16,7 @@
 import io.stargate.sgv2.jsonapi.service.operation.tasks.Task;
 import io.stargate.sgv2.jsonapi.service.operation.tasks.TaskRetryPolicy;
 import io.stargate.sgv2.jsonapi.service.schema.SchemaObject;
-import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionTableMatcher;
+import io.stargate.sgv2.jsonapi.service.schema.collections.spec.SuperShreddingTablePredicate;
 import io.stargate.sgv2.jsonapi.util.CqlIdentifierUtil;
 import java.util.List;
 import java.util.Map;
@@ -31,7 +31,7 @@ public abstract class MetadataDBTask<SchemaT extends SchemaObject> extends DBTas
 
   // Re-use the matcher for a collection, anything not a collection is a table
   protected static final Predicate<TableMetadata> TABLE_MATCHER =
-      new CollectionTableMatcher().negate();
+      new SuperShreddingTablePredicate().negate();
 
   // this will be set on executeStatement
   // TODO: BETTER CONTROL ON WHEN THIS IS SET AND NOT SET

src/main/java/io/stargate/sgv2/jsonapi/service/operation/collections/CreateCollectionOperation.java

@@ -41,7 +41,7 @@
 import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionLexicalDef;
 import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionRerankDef;
 import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionSchemaObject;
-import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionTableMatcher;
+import io.stargate.sgv2.jsonapi.service.schema.collections.spec.SuperShreddingTablePredicate;
 import io.stargate.sgv2.jsonapi.service.schema.tables.CQLSAIIndex;
 import java.time.Duration;
 import java.util.*;
@@ -69,7 +69,8 @@ public record CreateCollectionOperation(
 
   private static final Logger LOGGER = LoggerFactory.getLogger(CreateCollectionOperation.class);
 
-  private static final CollectionTableMatcher COLLECTION_MATCHER = new CollectionTableMatcher();
+  private static final SuperShreddingTablePredicate COLLECTION_MATCHER =
+      new SuperShreddingTablePredicate();
 
   private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
 

src/main/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionOperation.java

@@ -11,7 +11,6 @@
 import io.stargate.sgv2.jsonapi.api.model.command.CommandResult;
 import io.stargate.sgv2.jsonapi.api.model.command.clause.sort.SortExpression;
 import io.stargate.sgv2.jsonapi.api.request.RequestContext;
-import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants;
 import io.stargate.sgv2.jsonapi.exception.SchemaException;
 import io.stargate.sgv2.jsonapi.service.cql.builder.Query;
 import io.stargate.sgv2.jsonapi.service.cql.builder.QueryBuilder;
@@ -24,6 +23,7 @@
 import io.stargate.sgv2.jsonapi.service.operation.query.DBLogicalExpression;
 import io.stargate.sgv2.jsonapi.service.projection.DocumentProjector;
 import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionSchemaObject;
+import io.stargate.sgv2.jsonapi.service.schema.collections.spec.SuperShreddingMetadata;
 import io.stargate.sgv2.jsonapi.service.shredding.collections.DocumentId;
 import java.util.*;
 import java.util.function.Supplier;
@@ -524,8 +524,7 @@ private List<SimpleStatement> buildSelectQueries(IDCollectionFilter additionalId
             if (bm25Expr != null) {
               qb =
                   qb.bm25Sort(
-                      DocumentConstants.Columns.LEXICAL_INDEX_COLUMN_NAME,
-                      bm25Expr.getLexicalQuery());
+                      SuperShreddingMetadata.Names.QUERY_LEXICAL_VALUE, bm25Expr.getLexicalQuery());
             }
             query = qb.build();
           } else {
@@ -547,14 +546,14 @@ private Query getVectorSearchQueryByExpression(Expression<BuiltCondition> expres
           .select()
           .column(CollectionReadType.DOCUMENT == readType ? documentColumns : documentKeyColumns)
           .similarityFunction(
-              DocumentConstants.Columns.VECTOR_SEARCH_INDEX_COLUMN_NAME,
+              SuperShreddingMetadata.Names.QUERY_VECTOR_VALUE,
               commandContext().schemaObject().similarityFunction())
           .from(
               commandContext.schemaObject().identifier().keyspace(),
               commandContext.schemaObject().identifier().table())
           .where(expression)
           .limit(limit)
-          .vsearch(DocumentConstants.Columns.VECTOR_SEARCH_INDEX_COLUMN_NAME, vector())
+          .vsearch(SuperShreddingMetadata.Names.QUERY_VECTOR_VALUE, vector())
           .build();
     } else {
       return new QueryBuilder()
@@ -565,7 +564,7 @@ private Query getVectorSearchQueryByExpression(Expression<BuiltCondition> expres
               commandContext.schemaObject().identifier().table())
           .where(expression)
           .limit(limit)
-          .vsearch(DocumentConstants.Columns.VECTOR_SEARCH_INDEX_COLUMN_NAME, vector())
+          .vsearch(SuperShreddingMetadata.Names.QUERY_VECTOR_VALUE, vector())
           .build();
     }
   }

src/main/java/io/stargate/sgv2/jsonapi/service/operation/collections/FindCollectionsCollectionOperation.java

@@ -17,32 +17,34 @@
 import io.stargate.sgv2.jsonapi.service.operation.Operation;
 import io.stargate.sgv2.jsonapi.service.schema.KeyspaceSchemaObject;
 import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionSchemaObject;
-import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionTableMatcher;
+import io.stargate.sgv2.jsonapi.service.schema.collections.spec.SuperShreddingTablePredicate;
 import java.util.List;
 import java.util.function.Supplier;
 
 /**
  * Find collection operation. Uses {@link CQLSessionCache} to fetch all valid jsonapi tables for a
- * namespace. The schema check against the table is done in the {@link CollectionTableMatcher}.
+ * namespace. The schema check against the table is done in the {@link
+ * SuperShreddingTablePredicate}.
  *
  * @param explain - returns collection options if `true`; returns only collection names if `false`
  * @param objectMapper {@link ObjectMapper}
  * @param cqlSessionCache {@link CQLSessionCache}
- * @param tableMatcher {@link CollectionTableMatcher}
+ * @param tableMatcher {@link SuperShreddingTablePredicate}
  * @param commandContext {@link CommandContext}
  */
 public record FindCollectionsCollectionOperation(
     boolean explain,
     ObjectMapper objectMapper,
     CQLSessionCache cqlSessionCache,
-    CollectionTableMatcher tableMatcher,
+    SuperShreddingTablePredicate tableMatcher,
     CommandContext<KeyspaceSchemaObject> commandContext)
     implements Operation {
 
   // shared table matcher instance
   // TODO: if this is static why does the record that have an instance variable passed by the ctor
   // below ?
-  private static final CollectionTableMatcher TABLE_MATCHER = new CollectionTableMatcher();
+  private static final SuperShreddingTablePredicate TABLE_MATCHER =
+      new SuperShreddingTablePredicate();
 
   public FindCollectionsCollectionOperation(
       boolean explain,

src/main/java/io/stargate/sgv2/jsonapi/service/operation/filters/collection/AllCollectionFilter.java

@@ -2,12 +2,12 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants;
 import io.stargate.sgv2.jsonapi.exception.ServerException;
 import io.stargate.sgv2.jsonapi.service.operation.builder.BuiltCondition;
 import io.stargate.sgv2.jsonapi.service.operation.builder.BuiltConditionPredicate;
 import io.stargate.sgv2.jsonapi.service.operation.builder.ConditionLHS;
 import io.stargate.sgv2.jsonapi.service.operation.builder.JsonTerm;
+import io.stargate.sgv2.jsonapi.service.schema.collections.spec.SuperShreddingMetadata;
 import io.stargate.sgv2.jsonapi.service.shredding.collections.DocValueHasher;
 import java.util.ArrayList;
 import java.util.List;
@@ -57,7 +57,7 @@ public List<BuiltCondition> getAll() {
       this.collectionIndexUsage.arrayContainsTag = true;
       result.add(
           BuiltCondition.of(
-              ConditionLHS.column(DocumentConstants.Columns.DATA_CONTAINS_COLUMN_NAME),
+              ConditionLHS.column(SuperShreddingMetadata.Names.ARRAY_CONTAINS),
               negation ? BuiltConditionPredicate.NOT_CONTAINS : BuiltConditionPredicate.CONTAINS,
               new JsonTerm(getHashValue(new DocValueHasher(), getPath(), value))));
     }

src/main/java/io/stargate/sgv2/jsonapi/service/operation/filters/collection/InCollectionFilter.java

@@ -4,13 +4,13 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants;
 import io.stargate.sgv2.jsonapi.exception.FilterException;
 import io.stargate.sgv2.jsonapi.exception.ServerException;
 import io.stargate.sgv2.jsonapi.service.operation.builder.BuiltCondition;
 import io.stargate.sgv2.jsonapi.service.operation.builder.BuiltConditionPredicate;
 import io.stargate.sgv2.jsonapi.service.operation.builder.ConditionLHS;
 import io.stargate.sgv2.jsonapi.service.operation.builder.JsonTerm;
+import io.stargate.sgv2.jsonapi.service.schema.collections.spec.SuperShreddingMetadata;
 import io.stargate.sgv2.jsonapi.service.shredding.collections.DocValueHasher;
 import io.stargate.sgv2.jsonapi.service.shredding.collections.DocumentId;
 import java.math.BigDecimal;
@@ -100,7 +100,7 @@ public List<BuiltCondition> getAll() {
             this.collectionIndexUsage.arrayContainsTag = true;
             inResult.add(
                 BuiltCondition.of(
-                    ConditionLHS.column(DocumentConstants.Columns.DATA_CONTAINS_COLUMN_NAME),
+                    ConditionLHS.column(SuperShreddingMetadata.Names.ARRAY_CONTAINS),
                     BuiltConditionPredicate.CONTAINS,
                     new JsonTerm(getHashValue(new DocValueHasher(), getPath(), value))));
           }
@@ -131,7 +131,7 @@ public List<BuiltCondition> getAll() {
               this.collectionIndexUsage.arrayContainsTag = true;
               ninResults.add(
                   BuiltCondition.of(
-                      ConditionLHS.column(DocumentConstants.Columns.DATA_CONTAINS_COLUMN_NAME),
+                      ConditionLHS.column(SuperShreddingMetadata.Names.ARRAY_CONTAINS),
                       BuiltConditionPredicate.NOT_CONTAINS,
                       new JsonTerm(getHashValue(new DocValueHasher(), getPath(), value))));
             }

src/main/java/io/stargate/sgv2/jsonapi/service/operation/filters/collection/MapCollectionFilter.java

@@ -1,11 +1,11 @@
 package io.stargate.sgv2.jsonapi.service.operation.filters.collection;
 
-import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants;
 import io.stargate.sgv2.jsonapi.exception.FilterException;
 import io.stargate.sgv2.jsonapi.service.operation.builder.BuiltCondition;
 import io.stargate.sgv2.jsonapi.service.operation.builder.BuiltConditionPredicate;
 import io.stargate.sgv2.jsonapi.service.operation.builder.ConditionLHS;
 import io.stargate.sgv2.jsonapi.service.operation.builder.JsonTerm;
+import io.stargate.sgv2.jsonapi.service.schema.collections.spec.SuperShreddingMetadata;
 import io.stargate.sgv2.jsonapi.service.shredding.collections.DocValueHasher;
 import java.util.Map;
 import java.util.Objects;
@@ -90,12 +90,12 @@ public BuiltCondition get() {
     return switch (operator) {
       case EQ ->
           BuiltCondition.of(
-              ConditionLHS.column(DocumentConstants.Columns.DATA_CONTAINS_COLUMN_NAME),
+              ConditionLHS.column(SuperShreddingMetadata.Names.ARRAY_CONTAINS),
               BuiltConditionPredicate.CONTAINS,
               new JsonTerm(getHashValue(new DocValueHasher(), key, value)));
       case NE ->
           BuiltCondition.of(
-              ConditionLHS.column(DocumentConstants.Columns.DATA_CONTAINS_COLUMN_NAME),
+              ConditionLHS.column(SuperShreddingMetadata.Names.ARRAY_CONTAINS),
               BuiltConditionPredicate.NOT_CONTAINS,
               new JsonTerm(getHashValue(new DocValueHasher(), key, value)));
       case MAP_EQUALS ->

src/main/java/io/stargate/sgv2/jsonapi/service/operation/filters/collection/MatchCollectionFilter.java

@@ -2,11 +2,11 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
-import io.stargate.sgv2.jsonapi.config.constants.DocumentConstants;
 import io.stargate.sgv2.jsonapi.service.operation.builder.BuiltCondition;
 import io.stargate.sgv2.jsonapi.service.operation.builder.BuiltConditionPredicate;
 import io.stargate.sgv2.jsonapi.service.operation.builder.ConditionLHS;
 import io.stargate.sgv2.jsonapi.service.operation.builder.JsonTerm;
+import io.stargate.sgv2.jsonapi.service.schema.collections.spec.SuperShreddingMetadata;
 import java.util.Objects;
 import java.util.Optional;
 
@@ -23,7 +23,7 @@ public MatchCollectionFilter(String path, String value) {
   @Override
   public BuiltCondition get() {
     return BuiltCondition.of(
-        ConditionLHS.column(DocumentConstants.Columns.LEXICAL_INDEX_COLUMN_NAME),
+        ConditionLHS.column(SuperShreddingMetadata.Names.QUERY_LEXICAL_VALUE),
         BuiltConditionPredicate.TEXT_SEARCH,
         new JsonTerm(value));
   }

src/main/java/io/stargate/sgv2/jsonapi/service/schema/SchemaObjectFactory.java

@@ -11,7 +11,7 @@
 import io.stargate.sgv2.jsonapi.service.cqldriver.CQLSessionCache;
 import io.stargate.sgv2.jsonapi.service.cqldriver.executor.*;
 import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionSchemaObject;
-import io.stargate.sgv2.jsonapi.service.schema.collections.CollectionTableMatcher;
+import io.stargate.sgv2.jsonapi.service.schema.collections.spec.SuperShreddingTablePredicate;
 import io.stargate.sgv2.jsonapi.service.schema.tables.TableBasedSchemaObject;
 import io.stargate.sgv2.jsonapi.service.schema.tables.TableSchemaObject;
 import io.stargate.sgv2.jsonapi.util.CqlIdentifierUtil;
@@ -24,8 +24,8 @@
 public class SchemaObjectFactory implements SchemaObjectCache.SchemaObjectFactory {
   private static final Logger LOGGER = LoggerFactory.getLogger(SchemaObjectFactory.class);
 
-  private static final CollectionTableMatcher IS_COLLECTION_PREDICATE =
-      new CollectionTableMatcher();
+  private static final SuperShreddingTablePredicate IS_COLLECTION_PREDICATE =
+      new SuperShreddingTablePredicate();
 
   private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();