From 5e0b93712ce09c3a0e0ee74696ea10719141ec41 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Wed, 24 Jun 2026 14:18:02 +0100 Subject: [PATCH 1/6] Update SK to use MEVD packages and move MEVD projects out of main solution --- dotnet/Directory.Packages.props | 11 +++ dotnet/MEVD.slnx | 67 +++++++++++++++++++ dotnet/SK-dotnet.slnx | 47 ------------- dotnet/samples/Concepts/Concepts.csproj | 20 +++--- ...gingFace_TextEmbeddingCustomHttpHandler.cs | 7 +- .../ChatWithAgent.ApiService.csproj | 2 +- .../MCPServer/MCPServer.csproj | 2 +- .../Demos/OnnxSimpleRAG/OnnxSimpleRAG.csproj | 2 +- .../VectorStoreRAG/VectorStoreRAG.csproj | 14 ++-- .../GettingStartedWithTextSearch.csproj | 2 +- .../GettingStartedWithVectorStores.csproj | 7 +- .../IntegrationTests/IntegrationTests.csproj | 12 ---- .../SemanticKernel.Abstractions.csproj | 5 +- .../SemanticKernel.UnitTests.csproj | 2 +- 14 files changed, 106 insertions(+), 94 deletions(-) create mode 100644 dotnet/MEVD.slnx diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index 871f35361f2e..4dcc213ea1f1 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -152,6 +152,7 @@ + @@ -200,6 +201,16 @@ + + + + + + + + + + diff --git a/dotnet/MEVD.slnx b/dotnet/MEVD.slnx new file mode 100644 index 000000000000..10a3ac81db18 --- /dev/null +++ b/dotnet/MEVD.slnx @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dotnet/SK-dotnet.slnx b/dotnet/SK-dotnet.slnx index b2ff323d726c..966fbcb713a3 100644 --- a/dotnet/SK-dotnet.slnx +++ b/dotnet/SK-dotnet.slnx @@ -121,24 +121,6 @@ - - - - - - - - - - - - - - - - - - @@ -292,33 +274,4 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/dotnet/samples/Concepts/Concepts.csproj b/dotnet/samples/Concepts/Concepts.csproj index 2aa832dc868c..ad6b3fe12a59 100644 --- a/dotnet/samples/Concepts/Concepts.csproj +++ b/dotnet/samples/Concepts/Concepts.csproj @@ -49,6 +49,15 @@ + + + + + + + + + @@ -71,17 +80,6 @@ - - - - - - - - - - - diff --git a/dotnet/samples/Concepts/Memory/HuggingFace_TextEmbeddingCustomHttpHandler.cs b/dotnet/samples/Concepts/Memory/HuggingFace_TextEmbeddingCustomHttpHandler.cs index db956debca35..7e27f0a25bc3 100644 --- a/dotnet/samples/Concepts/Memory/HuggingFace_TextEmbeddingCustomHttpHandler.cs +++ b/dotnet/samples/Concepts/Memory/HuggingFace_TextEmbeddingCustomHttpHandler.cs @@ -3,7 +3,7 @@ using System.Text.Json; using Microsoft.Extensions.VectorData; using Microsoft.SemanticKernel.Connectors.HuggingFace; -using Microsoft.SemanticKernel.Connectors.SqliteVec; +using Microsoft.SemanticKernel.Connectors.InMemory; using Microsoft.SemanticKernel.Embeddings; #pragma warning disable CS8602 // Dereference of a possibly null reference. @@ -32,12 +32,11 @@ public async Task RunInferenceApiEmbeddingCustomHttpHandlerAsync() }) ); - var sqliteCollection = new SqliteCollection( - "Data Source=./../../../Sqlite.sqlite", + var inMemoryCollection = new InMemoryCollection( name: "Test", new() { EmbeddingGenerator = hf.AsEmbeddingGenerator() }); - await sqliteCollection.UpsertAsync(new Record + await inMemoryCollection.UpsertAsync(new Record { Id = "1", Text = "THIS IS A SAMPLE", diff --git a/dotnet/samples/Demos/AgentFrameworkWithAspire/ChatWithAgent.ApiService/ChatWithAgent.ApiService.csproj b/dotnet/samples/Demos/AgentFrameworkWithAspire/ChatWithAgent.ApiService/ChatWithAgent.ApiService.csproj index e2a7b7725f0f..09abe013258f 100644 --- a/dotnet/samples/Demos/AgentFrameworkWithAspire/ChatWithAgent.ApiService/ChatWithAgent.ApiService.csproj +++ b/dotnet/samples/Demos/AgentFrameworkWithAspire/ChatWithAgent.ApiService/ChatWithAgent.ApiService.csproj @@ -14,13 +14,13 @@ + - diff --git a/dotnet/samples/Demos/ModelContextProtocolClientServer/MCPServer/MCPServer.csproj b/dotnet/samples/Demos/ModelContextProtocolClientServer/MCPServer/MCPServer.csproj index 042eb196402f..55b4a187a65e 100644 --- a/dotnet/samples/Demos/ModelContextProtocolClientServer/MCPServer/MCPServer.csproj +++ b/dotnet/samples/Demos/ModelContextProtocolClientServer/MCPServer/MCPServer.csproj @@ -32,11 +32,11 @@ + - diff --git a/dotnet/samples/Demos/OnnxSimpleRAG/OnnxSimpleRAG.csproj b/dotnet/samples/Demos/OnnxSimpleRAG/OnnxSimpleRAG.csproj index 0f5162c8af6a..9507fd81365c 100644 --- a/dotnet/samples/Demos/OnnxSimpleRAG/OnnxSimpleRAG.csproj +++ b/dotnet/samples/Demos/OnnxSimpleRAG/OnnxSimpleRAG.csproj @@ -8,7 +8,6 @@ - @@ -17,6 +16,7 @@ + diff --git a/dotnet/samples/Demos/VectorStoreRAG/VectorStoreRAG.csproj b/dotnet/samples/Demos/VectorStoreRAG/VectorStoreRAG.csproj index 23fd288e7b97..fab34e67af9d 100644 --- a/dotnet/samples/Demos/VectorStoreRAG/VectorStoreRAG.csproj +++ b/dotnet/samples/Demos/VectorStoreRAG/VectorStoreRAG.csproj @@ -15,17 +15,17 @@ + + + + + + + - - - - - - - diff --git a/dotnet/samples/GettingStartedWithTextSearch/GettingStartedWithTextSearch.csproj b/dotnet/samples/GettingStartedWithTextSearch/GettingStartedWithTextSearch.csproj index a4ebfd31ac80..9eab7a6c81d7 100644 --- a/dotnet/samples/GettingStartedWithTextSearch/GettingStartedWithTextSearch.csproj +++ b/dotnet/samples/GettingStartedWithTextSearch/GettingStartedWithTextSearch.csproj @@ -35,6 +35,7 @@ + @@ -42,7 +43,6 @@ - diff --git a/dotnet/samples/GettingStartedWithVectorStores/GettingStartedWithVectorStores.csproj b/dotnet/samples/GettingStartedWithVectorStores/GettingStartedWithVectorStores.csproj index 2b8c52498767..b08f7cf0ce95 100644 --- a/dotnet/samples/GettingStartedWithVectorStores/GettingStartedWithVectorStores.csproj +++ b/dotnet/samples/GettingStartedWithVectorStores/GettingStartedWithVectorStores.csproj @@ -32,16 +32,15 @@ + + + - - - - diff --git a/dotnet/src/IntegrationTests/IntegrationTests.csproj b/dotnet/src/IntegrationTests/IntegrationTests.csproj index 575a75a0f28a..7ed606ec711d 100644 --- a/dotnet/src/IntegrationTests/IntegrationTests.csproj +++ b/dotnet/src/IntegrationTests/IntegrationTests.csproj @@ -72,18 +72,6 @@ - - - - - - - - - - - - diff --git a/dotnet/src/SemanticKernel.Abstractions/SemanticKernel.Abstractions.csproj b/dotnet/src/SemanticKernel.Abstractions/SemanticKernel.Abstractions.csproj index 02b5009b6194..d6254b9fd51d 100644 --- a/dotnet/src/SemanticKernel.Abstractions/SemanticKernel.Abstractions.csproj +++ b/dotnet/src/SemanticKernel.Abstractions/SemanticKernel.Abstractions.csproj @@ -23,13 +23,10 @@ Semantic Kernel interfaces and abstractions. This package is automatically installed by Semantic Kernel packages if needed. - - - - + diff --git a/dotnet/src/SemanticKernel.UnitTests/SemanticKernel.UnitTests.csproj b/dotnet/src/SemanticKernel.UnitTests/SemanticKernel.UnitTests.csproj index f3e9dff4ab9c..1bc01dd5df06 100644 --- a/dotnet/src/SemanticKernel.UnitTests/SemanticKernel.UnitTests.csproj +++ b/dotnet/src/SemanticKernel.UnitTests/SemanticKernel.UnitTests.csproj @@ -25,11 +25,11 @@ + - From 313ac3b1b9688764b107753a4371ee766efcd6d6 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Wed, 24 Jun 2026 14:22:44 +0100 Subject: [PATCH 2/6] Fix vulnerability issue --- dotnet/samples/Demos/VectorStoreRAG/VectorStoreRAG.csproj | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dotnet/samples/Demos/VectorStoreRAG/VectorStoreRAG.csproj b/dotnet/samples/Demos/VectorStoreRAG/VectorStoreRAG.csproj index fab34e67af9d..97b89013fef8 100644 --- a/dotnet/samples/Demos/VectorStoreRAG/VectorStoreRAG.csproj +++ b/dotnet/samples/Demos/VectorStoreRAG/VectorStoreRAG.csproj @@ -22,6 +22,8 @@ + + From 9fc3f5243ddc2fea6036db4dde9babd257f232c3 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Wed, 24 Jun 2026 14:27:52 +0100 Subject: [PATCH 3/6] Remove MEVD.slnx from build --- .github/workflows/dotnet-build-and-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dotnet-build-and-test.yml b/.github/workflows/dotnet-build-and-test.yml index 8987c2f48e42..5a758f331662 100644 --- a/.github/workflows/dotnet-build-and-test.yml +++ b/.github/workflows/dotnet-build-and-test.yml @@ -81,7 +81,7 @@ jobs: - name: Build dotnet solutions shell: bash run: | - export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | tr '\n' ' ') + export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | grep -v "MEVD.slnx" | tr '\n' ' ') for solution in $SOLUTIONS; do dotnet build $solution -c ${{ matrix.configuration }} --warnaserror done @@ -89,7 +89,7 @@ jobs: - name: Package install check shell: bash run: | - export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | tr '\n' ' ') + export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | grep -v "MEVD.slnx" | tr '\n' ' ') for solution in $SOLUTIONS; do dotnet pack $solution -c ${{ matrix.configuration }} --no-build --no-restore --output ./artifacts done From bc083e939b4ef997cd873eac4646d696951ad37a Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Wed, 24 Jun 2026 14:29:34 +0100 Subject: [PATCH 4/6] Delete the MEVD solution filter, since it is no longer valid. --- dotnet/MEVD.slnf | 50 ------------------------------------------------ 1 file changed, 50 deletions(-) delete mode 100644 dotnet/MEVD.slnf diff --git a/dotnet/MEVD.slnf b/dotnet/MEVD.slnf deleted file mode 100644 index 2a2273ba7f88..000000000000 --- a/dotnet/MEVD.slnf +++ /dev/null @@ -1,50 +0,0 @@ -{ - "solution": { - "path": "SK-dotnet.slnx", - "projects": - [ - "src/VectorData/AzureAISearch/AzureAISearch.csproj", - "src/VectorData/CosmosMongoDB/CosmosMongoDB.csproj", - "src/VectorData/CosmosNoSql/CosmosNoSql.csproj", - "src/VectorData/Chroma/Chroma.csproj", - "src/VectorData/InMemory/InMemory.csproj", - "src/VectorData/Milvus/Milvus.csproj", - "src/VectorData/MongoDB/MongoDB.csproj", - "src/VectorData/Pinecone/Pinecone.csproj", - "src/VectorData/PgVector/PgVector.csproj", - "src/VectorData/Qdrant/Qdrant.csproj", - "src/VectorData/Redis/Redis.csproj", - "src/VectorData/SqliteVec/SqliteVec.csproj", - "src/VectorData/SqlServer/SqlServer.csproj", - "src/VectorData/Weaviate/Weaviate.csproj", - - "src/VectorData/VectorData.Abstractions/VectorData.Abstractions.csproj", - - "test/VectorData/AzureAISearch.UnitTests/AzureAISearch.UnitTests.csproj", - "test/VectorData/AzureAISearch.ConformanceTests/AzureAISearch.ConformanceTests.csproj", - "test/VectorData/CosmosMongoDB.UnitTests/CosmosMongoDB.UnitTests.csproj", - "test/VectorData/CosmosMongoDB.ConformanceTests/CosmosMongoDB.ConformanceTests.csproj", - "test/VectorData/CosmosNoSql.UnitTests/CosmosNoSql.UnitTests.csproj", - "test/VectorData/CosmosNoSql.ConformanceTests/CosmosNoSql.ConformanceTests.csproj", - "test/VectorData/InMemory.UnitTests/InMemory.UnitTests.csproj", - "test/VectorData/InMemory.ConformanceTests/InMemory.ConformanceTests.csproj", - "test/VectorData/MongoDB.UnitTests/MongoDB.UnitTests.csproj", - "test/VectorData/MongoDB.ConformanceTests/MongoDB.ConformanceTests.csproj", - "test/VectorData/Pinecone.UnitTests/Pinecone.UnitTests.csproj", - "test/VectorData/Pinecone.ConformanceTests/Pinecone.ConformanceTests.csproj", - "test/VectorData/PgVector.UnitTests/PgVector.UnitTests.csproj", - "test/VectorData/PgVector.ConformanceTests/PgVector.ConformanceTests.csproj", - "test/VectorData/Qdrant.UnitTests/Qdrant.UnitTests.csproj", - "test/VectorData/Qdrant.ConformanceTests/Qdrant.ConformanceTests.csproj", - "test/VectorData/Redis.UnitTests/Redis.UnitTests.csproj", - "test/VectorData/Redis.ConformanceTests/Redis.ConformanceTests.csproj", - "test/VectorData/SqliteVec.UnitTests/SqliteVec.UnitTests.csproj", - "test/VectorData/SqliteVec.ConformanceTests/SqliteVec.ConformanceTests.csproj", - "test/VectorData/SqlServer.ConformanceTests/SqlServer.ConformanceTests.csproj", - "test/VectorData/Weaviate.UnitTests/Weaviate.UnitTests.csproj", - "test/VectorData/Weaviate.ConformanceTests/Weaviate.ConformanceTests.csproj", - - "test/VectorData/VectorData.ConformanceTests/VectorData.ConformanceTests.csproj" - ] - } -} \ No newline at end of file From 43494ec18e8e70752c10cd5bcc87c2eead5caf67 Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Wed, 24 Jun 2026 16:54:39 +0100 Subject: [PATCH 5/6] Remove MEVD.Abstractions from repo and point implementations to released package --- .github/workflows/dotnet-build-and-test.yml | 4 +- dotnet/MEVD.slnx | 3 +- .../Memory/MongoDB/MongoModelBuilder.cs | 13 +- .../AzureAISearch/AzureAISearch.csproj | 5 +- .../CosmosMongoDB/CosmosMongoDB.csproj | 5 +- .../VectorData/CosmosNoSql/CosmosNoSql.csproj | 5 +- .../CosmosNoSql/CosmosNoSqlMapper.cs | 6 +- .../src/VectorData/InMemory/InMemory.csproj | 5 +- dotnet/src/VectorData/MongoDB/MongoDB.csproj | 5 +- .../src/VectorData/PgVector/PgVector.csproj | 5 +- .../src/VectorData/Pinecone/Pinecone.csproj | 5 +- dotnet/src/VectorData/Qdrant/Qdrant.csproj | 5 +- dotnet/src/VectorData/Redis/Redis.csproj | 5 +- .../src/VectorData/SqlServer/SqlServer.csproj | 5 +- .../src/VectorData/SqliteVec/SqliteVec.csproj | 5 +- .../VectorData.Abstractions/.editorconfig | 3 - .../AnyTagEqualToFilterClause.cs | 33 - .../FilterClauses/EqualToFilterClause.cs | 33 - .../FilterClauses/FilterClause.cs | 23 - .../VectorData.Abstractions/PACKAGE.md | 51 -- .../Properties/AssemblyInfo.cs | 3 - .../CollectionJsonModelBuilder.cs | 104 --- .../ProviderServices/CollectionModel.cs | 243 ------- .../CollectionModelBuilder.cs | 609 ------------------ .../CollectionModelBuildingOptions.cs | 34 - .../ProviderServices/DataPropertyModel.cs | 34 - .../EmbeddingGenerationDispatcher.cs | 86 --- .../Filter/FilterPreprocessingOptions.cs | 22 - .../Filter/FilterTranslatorBase.cs | 403 ------------ .../Filter/QueryParameterExpression.cs | 33 - .../ProviderServices/IRecordCreator.cs | 8 - .../ProviderServices/KeyPropertyModel.cs | 31 - .../ProviderServices/PropertyModel.cs | 162 ----- .../ProviderServices/VectorDataStrings.cs | 79 --- .../ProviderServices/VectorPropertyModel.cs | 198 ------ .../VectorPropertyModel{TInput}.cs | 70 -- .../VectorData.Abstractions/README.md | 1 + .../VectorStoreDataAttribute.cs | 41 -- .../VectorStoreKeyAttribute.cs | 46 -- .../VectorStoreVectorAttribute.cs | 68 -- .../RecordDefinition/DistanceFunction.cs | 80 --- .../RecordDefinition/IndexKind.cs | 60 -- .../VectorStoreCollectionDefinition.cs | 33 - .../VectorStoreDataProperty.cs | 40 -- .../VectorStoreKeyProperty.cs | 32 - .../RecordDefinition/VectorStoreProperty.cs | 71 -- .../VectorStoreVectorProperty.cs | 105 --- .../VectorStoreVectorProperty{TInput}.cs | 36 -- .../FilteredRecordRetrievalOptions.cs | 117 ---- .../RecordOptions/RecordRetrievalOptions.cs | 18 - .../VectorData.Abstractions/Throw.cs | 14 - .../VectorData.Abstractions.csproj | 71 -- .../VectorSearch/HybridSearchOptions.cs | 77 --- .../VectorSearch/IKeywordHybridSearchable.cs | 74 --- .../VectorSearch/IVectorSearchable.cs | 72 --- .../KeywordHybridSearchExtensions.cs | 31 - .../VectorSearch/RecordSearchOptions.cs | 70 -- .../VectorSearch/VectorSearchExtensions.cs | 31 - .../VectorSearch/VectorSearchResult.cs | 31 - .../VectorStorage/VectorStore.cs | 98 --- .../VectorStorage/VectorStoreCollection.cs | 202 ------ .../VectorStoreCollectionMetadata.cs | 25 - .../VectorStoreCollectionOptions.cs | 40 -- .../VectorStorage/VectorStoreException.cs | 58 -- .../VectorStorage/VectorStoreExtensions.cs | 31 - .../VectorStorage/VectorStoreMetadata.cs | 20 - .../VectorData.Abstractions/neticon.png | Bin 7006 -> 0 bytes .../src/VectorData/Weaviate/Weaviate.csproj | 5 +- .../VectorData.ConformanceTests.csproj | 5 +- .../CollectionModelBuilderTests.cs | 525 --------------- .../PropertyModelTests.cs | 139 ---- .../VectorData.UnitTests.csproj | 43 -- 72 files changed, 28 insertions(+), 4725 deletions(-) delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/.editorconfig delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/AnyTagEqualToFilterClause.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/EqualToFilterClause.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/FilterClause.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/PACKAGE.md delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/Properties/AssemblyInfo.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionJsonModelBuilder.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModel.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModelBuilder.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModelBuildingOptions.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/DataPropertyModel.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/EmbeddingGenerationDispatcher.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterPreprocessingOptions.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterTranslatorBase.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/QueryParameterExpression.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/IRecordCreator.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/KeyPropertyModel.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/PropertyModel.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorDataStrings.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorPropertyModel.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorPropertyModel{TInput}.cs create mode 100644 dotnet/src/VectorData/VectorData.Abstractions/README.md delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreDataAttribute.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreKeyAttribute.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreVectorAttribute.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/DistanceFunction.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/IndexKind.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreCollectionDefinition.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreDataProperty.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreKeyProperty.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreProperty.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreVectorProperty.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreVectorProperty{TInput}.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/RecordOptions/FilteredRecordRetrievalOptions.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/RecordOptions/RecordRetrievalOptions.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/Throw.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/VectorData.Abstractions.csproj delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/HybridSearchOptions.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/IKeywordHybridSearchable.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/IVectorSearchable.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/KeywordHybridSearchExtensions.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/RecordSearchOptions.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/VectorSearchExtensions.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/VectorSearchResult.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStore.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollection.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollectionMetadata.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollectionOptions.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreException.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreExtensions.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreMetadata.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/neticon.png delete mode 100644 dotnet/test/VectorData/VectorData.UnitTests/CollectionModelBuilderTests.cs delete mode 100644 dotnet/test/VectorData/VectorData.UnitTests/PropertyModelTests.cs delete mode 100644 dotnet/test/VectorData/VectorData.UnitTests/VectorData.UnitTests.csproj diff --git a/.github/workflows/dotnet-build-and-test.yml b/.github/workflows/dotnet-build-and-test.yml index 5a758f331662..8987c2f48e42 100644 --- a/.github/workflows/dotnet-build-and-test.yml +++ b/.github/workflows/dotnet-build-and-test.yml @@ -81,7 +81,7 @@ jobs: - name: Build dotnet solutions shell: bash run: | - export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | grep -v "MEVD.slnx" | tr '\n' ' ') + export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | tr '\n' ' ') for solution in $SOLUTIONS; do dotnet build $solution -c ${{ matrix.configuration }} --warnaserror done @@ -89,7 +89,7 @@ jobs: - name: Package install check shell: bash run: | - export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | grep -v "MEVD.slnx" | tr '\n' ' ') + export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | tr '\n' ' ') for solution in $SOLUTIONS; do dotnet pack $solution -c ${{ matrix.configuration }} --no-build --no-restore --output ./artifacts done diff --git a/dotnet/MEVD.slnx b/dotnet/MEVD.slnx index 10a3ac81db18..9f782e02e062 100644 --- a/dotnet/MEVD.slnx +++ b/dotnet/MEVD.slnx @@ -17,6 +17,7 @@ + @@ -32,7 +33,6 @@ - @@ -60,7 +60,6 @@ - diff --git a/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoModelBuilder.cs b/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoModelBuilder.cs index 25b77f90839b..69f974cffd19 100644 --- a/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoModelBuilder.cs +++ b/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoModelBuilder.cs @@ -27,14 +27,17 @@ internal class MongoModelBuilder() : CollectionModelBuilder(s_validationOptions) UsesExternalSerializer = true, }; - protected override void ProcessProperty(PropertyInfo? clrProperty, VectorStoreProperty? definitionProperty, Type? type) + [RequiresUnreferencedCode("Traverses the CLR type's properties with reflection, so not compatible with trimming")] + protected override void ProcessTypeProperties(Type type, VectorStoreCollectionDefinition? definition) { - base.ProcessProperty(clrProperty, definitionProperty, type); + base.ProcessTypeProperties(type, definition); - if (clrProperty?.GetCustomAttribute() is { } bsonElementAttribute - && this.PropertyMap.TryGetValue(clrProperty.Name, out var property)) + foreach (var property in this.Properties) { - property.StorageName = bsonElementAttribute.ElementName; + if (property.PropertyInfo?.GetCustomAttribute() is { } bsonElementAttribute) + { + property.StorageName = bsonElementAttribute.ElementName; + } } } diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearch.csproj b/dotnet/src/VectorData/AzureAISearch/AzureAISearch.csproj index d43078f48cee..54258372dc2a 100644 --- a/dotnet/src/VectorData/AzureAISearch/AzureAISearch.csproj +++ b/dotnet/src/VectorData/AzureAISearch/AzureAISearch.csproj @@ -26,6 +26,7 @@ + @@ -37,8 +38,4 @@ - - - - diff --git a/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoDB.csproj b/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoDB.csproj index 0d6bca948aaf..23f8e0cba2db 100644 --- a/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoDB.csproj +++ b/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoDB.csproj @@ -42,10 +42,7 @@ - - - - + diff --git a/dotnet/src/VectorData/CosmosNoSql/CosmosNoSql.csproj b/dotnet/src/VectorData/CosmosNoSql/CosmosNoSql.csproj index 2849a25f118e..05a0574e03c4 100644 --- a/dotnet/src/VectorData/CosmosNoSql/CosmosNoSql.csproj +++ b/dotnet/src/VectorData/CosmosNoSql/CosmosNoSql.csproj @@ -32,13 +32,10 @@ + - - - - diff --git a/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlMapper.cs b/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlMapper.cs index 6f0ffdaea91b..8a8644b80198 100644 --- a/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlMapper.cs +++ b/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlMapper.cs @@ -41,8 +41,8 @@ public JsonObject MapFromDataToStorageModel(TRecord dataModel, int recordIndex, // The key property in Azure CosmosDB NoSQL is always named 'id'. // But the external JSON serializer used just above isn't aware of that, and will produce a JSON object with another name, taking into - // account e.g. naming policies. SerializedKeyName gets populated in the model builder - containing that name - once VectorStoreModelBuildingOptions.ReservedKeyPropertyName is set - RenameJsonProperty(jsonObject, this._keyProperty.SerializedKeyName!, CosmosNoSqlConstants.ReservedKeyPropertyName); + // account e.g. naming policies. TemporaryStorageName gets populated in the model builder - containing that name - once VectorStoreModelBuildingOptions.ReservedKeyPropertyName is set + RenameJsonProperty(jsonObject, this._keyProperty.TemporaryStorageName!, CosmosNoSqlConstants.ReservedKeyPropertyName); // Go over the vector properties; inject any generated embeddings to overwrite the JSON serialized above. // Also, for Embedding properties we also need to overwrite with a simple array (since Embedding gets serialized as a complex object). @@ -116,7 +116,7 @@ public JsonObject MapFromDataToStorageModel(TRecord dataModel, int recordIndex, public TRecord MapFromStorageToDataModel(JsonObject storageModel, bool includeVectors) { // See above comment. - RenameJsonProperty(storageModel, CosmosNoSqlConstants.ReservedKeyPropertyName, this._keyProperty.SerializedKeyName!); + RenameJsonProperty(storageModel, CosmosNoSqlConstants.ReservedKeyPropertyName, this._keyProperty.TemporaryStorageName!); foreach (var vectorProperty in this._model.VectorProperties) { diff --git a/dotnet/src/VectorData/InMemory/InMemory.csproj b/dotnet/src/VectorData/InMemory/InMemory.csproj index aa8cbb340f55..dcd70c709db2 100644 --- a/dotnet/src/VectorData/InMemory/InMemory.csproj +++ b/dotnet/src/VectorData/InMemory/InMemory.csproj @@ -34,11 +34,8 @@ + - - - - diff --git a/dotnet/src/VectorData/MongoDB/MongoDB.csproj b/dotnet/src/VectorData/MongoDB/MongoDB.csproj index 4bc2786c1161..6196ba1c4fee 100644 --- a/dotnet/src/VectorData/MongoDB/MongoDB.csproj +++ b/dotnet/src/VectorData/MongoDB/MongoDB.csproj @@ -37,10 +37,7 @@ - - - - + diff --git a/dotnet/src/VectorData/PgVector/PgVector.csproj b/dotnet/src/VectorData/PgVector/PgVector.csproj index b171819ce395..8b33c42e99f6 100644 --- a/dotnet/src/VectorData/PgVector/PgVector.csproj +++ b/dotnet/src/VectorData/PgVector/PgVector.csproj @@ -32,16 +32,13 @@ + 8.0.7 - - - - diff --git a/dotnet/src/VectorData/Pinecone/Pinecone.csproj b/dotnet/src/VectorData/Pinecone/Pinecone.csproj index d5c5847327a2..f065a03f1462 100644 --- a/dotnet/src/VectorData/Pinecone/Pinecone.csproj +++ b/dotnet/src/VectorData/Pinecone/Pinecone.csproj @@ -28,6 +28,7 @@ + @@ -39,8 +40,4 @@ - - - - diff --git a/dotnet/src/VectorData/Qdrant/Qdrant.csproj b/dotnet/src/VectorData/Qdrant/Qdrant.csproj index 86e2503a2056..248a0a2c0b22 100644 --- a/dotnet/src/VectorData/Qdrant/Qdrant.csproj +++ b/dotnet/src/VectorData/Qdrant/Qdrant.csproj @@ -27,6 +27,7 @@ + @@ -40,8 +41,4 @@ - - - - \ No newline at end of file diff --git a/dotnet/src/VectorData/Redis/Redis.csproj b/dotnet/src/VectorData/Redis/Redis.csproj index 41e673ad61a5..d463c09b373d 100644 --- a/dotnet/src/VectorData/Redis/Redis.csproj +++ b/dotnet/src/VectorData/Redis/Redis.csproj @@ -26,6 +26,7 @@ + @@ -33,8 +34,4 @@ - - - - \ No newline at end of file diff --git a/dotnet/src/VectorData/SqlServer/SqlServer.csproj b/dotnet/src/VectorData/SqlServer/SqlServer.csproj index 7896ffcb03f5..c6075692aa84 100644 --- a/dotnet/src/VectorData/SqlServer/SqlServer.csproj +++ b/dotnet/src/VectorData/SqlServer/SqlServer.csproj @@ -30,13 +30,10 @@ + - - - - diff --git a/dotnet/src/VectorData/SqliteVec/SqliteVec.csproj b/dotnet/src/VectorData/SqliteVec/SqliteVec.csproj index ab5a3f4798dd..d3e2062b157b 100644 --- a/dotnet/src/VectorData/SqliteVec/SqliteVec.csproj +++ b/dotnet/src/VectorData/SqliteVec/SqliteVec.csproj @@ -31,16 +31,13 @@ + - - - - diff --git a/dotnet/src/VectorData/VectorData.Abstractions/.editorconfig b/dotnet/src/VectorData/VectorData.Abstractions/.editorconfig deleted file mode 100644 index acb2cb62caf4..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/.editorconfig +++ /dev/null @@ -1,3 +0,0 @@ -# Suppress missing documentation warnings for generated code (strings) -[*.Designer.cs] -dotnet_diagnostic.CS1591.severity = none diff --git a/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/AnyTagEqualToFilterClause.cs b/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/AnyTagEqualToFilterClause.cs deleted file mode 100644 index c9419d087732..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/AnyTagEqualToFilterClause.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Represents a filter clause that filters by checking if a field consisting of a list of values contains a specific value. -/// -[Obsolete("Use LINQ expressions via VectorSearchOptions.Filter instead. This type will be removed in a future version.")] -public sealed class AnyTagEqualToFilterClause : FilterClause -{ - /// - /// Initializes a new instance of the class. - /// - /// The name of the field with the list of values. - /// The value that the list should contain. - public AnyTagEqualToFilterClause(string fieldName, string value) - { - this.FieldName = fieldName; - this.Value = value; - } - - /// - /// Gets the name of the field with the list of values. - /// - public string FieldName { get; private set; } - - /// - /// Gets the value that the list should contain. - /// - public string Value { get; private set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/EqualToFilterClause.cs b/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/EqualToFilterClause.cs deleted file mode 100644 index 03fc678abc8b..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/EqualToFilterClause.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Represents a filter clause that filters using equality of a field value. -/// -[Obsolete("Use LINQ expressions via VectorSearchOptions.Filter instead. This type will be removed in a future version.")] -public sealed class EqualToFilterClause : FilterClause -{ - /// - /// Initializes a new instance of the class. - /// - /// Field name. - /// Field value. - public EqualToFilterClause(string fieldName, object value) - { - this.FieldName = fieldName; - this.Value = value; - } - - /// - /// Gets the field name to match. - /// - public string FieldName { get; private set; } - - /// - /// Gets the field value to match. - /// - public object Value { get; private set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/FilterClause.cs b/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/FilterClause.cs deleted file mode 100644 index 97c8869a27ea..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/FilterClauses/FilterClause.cs +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines a base class for filter clauses. -/// -/// -/// A is used to request that the underlying search service should -/// filter search results based on the specified criteria. -/// -[Obsolete("Use LINQ expressions via VectorSearchOptions.Filter instead. This type will be removed in a future version.")] -public abstract class FilterClause -{ - /// - /// Initializes a new instance of the class. - /// - protected FilterClause() - { - } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/PACKAGE.md b/dotnet/src/VectorData/VectorData.Abstractions/PACKAGE.md deleted file mode 100644 index 948b8527c6a8..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/PACKAGE.md +++ /dev/null @@ -1,51 +0,0 @@ -## About - -Contains abstractions for accessing Vector Databases and Vector Indexes. - -## Key Features - -- Base abstract classes and interfaces for Vector Database implementation. Vector Database implementations are provided separately in other packages, for example `Microsoft.SemanticKernel.Connectors.AzureAISearch`. -- Abstractions include: - - Creating, listing and deleting collections with custom schema support. - - Creating, retrieving, updating and deleting records. - - Similarty search using vector embeddings. - - Search using filters. - - Hybrid search combining vector similarity and keyword search. - - Built-in embedding generation using `Microsoft.Extensions.AI`. - -## How to Use - -This package is typically used with an implementation of the vector database abstractions such as `Microsoft.SemanticKernel.Connectors.AzureAISearch`. - -## Main Types - -The main types provided by this library are: - -- [Microsoft.Extensions.VectorData.VectorStore](https://learn.microsoft.com/dotnet/api/microsoft.extensions.vectordata.vectorstore) -- [Microsoft.Extensions.VectorData.VectorStoreCollection](https://learn.microsoft.com/dotnet/api/microsoft.extensions.vectordata.vectorstorecollection-2) - -## Additional Documentation - -- [Conceptual documentation](https://learn.microsoft.com/en-us/semantic-kernel/concepts/vector-store-connectors) - -## Related Packages - -Vector Database implementations: - -- [Microsoft.SemanticKernel.Connectors.AzureAISearch](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.AzureAISearch) -- [Microsoft.SemanticKernel.Connectors.CosmosMongoDB](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.CosmosMongoDB) -- [Microsoft.SemanticKernel.Connectors.CosmosNoSQL](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.CosmosNoSQL) -- [Elastic.SemanticKernel.Connectors.Elasticsearch](https://www.nuget.org/packages/Elastic.SemanticKernel.Connectors.Elasticsearch) -- [Microsoft.SemanticKernel.Connectors.InMemory](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.InMemory) -- [Microsoft.SemanticKernel.Connectors.MongoDB](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.MongoDB) -- [Microsoft.SemanticKernel.Connectors.PgVector](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.PgVector) -- [Microsoft.SemanticKernel.Connectors.Pinecone](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.Pinecone) -- [Microsoft.SemanticKernel.Connectors.Qdrant](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.Qdrant) -- [Microsoft.SemanticKernel.Connectors.Redis](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.Redis) -- [Microsoft.SemanticKernel.Connectors.SqliteVec](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.SqliteVec) -- [Microsoft.SemanticKernel.Connectors.SqlServer](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.SqlServer) -- [Microsoft.SemanticKernel.Connectors.Weaviate](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.Weaviate) - -## Feedback & Contributing - -Microsoft.Extensions.VectorData.Abstractions is released as open source under the [MIT license](https://licenses.nuget.org/MIT). Bug reports and contributions are welcome at [the GitHub repository](https://github.com/microsoft/semantic-kernel). diff --git a/dotnet/src/VectorData/VectorData.Abstractions/Properties/AssemblyInfo.cs b/dotnet/src/VectorData/VectorData.Abstractions/Properties/AssemblyInfo.cs deleted file mode 100644 index 09647faa37af..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -[assembly: System.Resources.NeutralResourcesLanguage("en-US")] diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionJsonModelBuilder.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionJsonModelBuilder.cs deleted file mode 100644 index 90a2db9433f3..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionJsonModelBuilder.cs +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Diagnostics.CodeAnalysis; -using System.Reflection; -using System.Text.Json; -using System.Text.Json.Serialization; -using Microsoft.Extensions.AI; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -/// -/// Represents a model builder that performs logic specific to connectors that use System.Text.Json for serialization. -/// This is an internal support type meant for use by connectors only and not by applications. -/// -[Experimental("MEVD9001")] -public abstract class CollectionJsonModelBuilder : CollectionModelBuilder -{ - private JsonSerializerOptions? _jsonSerializerOptions; - - /// - /// Constructs a new . - /// - protected CollectionJsonModelBuilder(CollectionModelBuildingOptions options) - : base(options) - { - } - - /// - /// Builds and returns a from the given and . - /// - [RequiresDynamicCode("This model building variant is not compatible with NativeAOT. See BuildDynamic() for dynamic mapping, and a third variant accepting source-generated delegates will be introduced in the future.")] - [RequiresUnreferencedCode("This model building variant is not compatible with trimming. See BuildDynamic() for dynamic mapping, and a third variant accepting source-generated delegates will be introduced in the future.")] - public virtual CollectionModel Build( - Type recordType, - Type keyType, - VectorStoreCollectionDefinition? definition, - IEmbeddingGenerator? defaultEmbeddingGenerator, - JsonSerializerOptions jsonSerializerOptions) - { - this._jsonSerializerOptions = jsonSerializerOptions; - - return this.Build(recordType, keyType, definition, defaultEmbeddingGenerator); - } - - /// - /// Builds and returns a for dynamic mapping scenarios from the given . - /// - public virtual CollectionModel BuildDynamic( - VectorStoreCollectionDefinition definition, - IEmbeddingGenerator? defaultEmbeddingGenerator, - JsonSerializerOptions jsonSerializerOptions) - { - this._jsonSerializerOptions = jsonSerializerOptions; - - return this.BuildDynamic(definition, defaultEmbeddingGenerator); - } - - /// - protected override void Customize() - { - // This mimics the naming behavior of the System.Text.Json serializer, which we use for serialization/deserialization. - // The property storage names in the model must in sync with the serializer configuration, since the model is used e.g. for filtering - // even if serialization/deserialization doesn't use the model. - var namingPolicy = this._jsonSerializerOptions?.PropertyNamingPolicy; - - foreach (var property in this.Properties) - { - var keyPropertyWithReservedName = this.Options.ReservedKeyStorageName is not null && property is KeyPropertyModel; - string storageName; - - if (property.PropertyInfo?.GetCustomAttribute() is { } jsonPropertyNameAttribute) - { - if (keyPropertyWithReservedName && jsonPropertyNameAttribute.Name != this.Options.ReservedKeyStorageName) - { - throw new InvalidOperationException($"The key property for your connector must always have the reserved name '{this.Options.ReservedKeyStorageName}' and cannot be changed."); - } - - storageName = jsonPropertyNameAttribute.Name; - } - else if (namingPolicy is not null) - { - storageName = namingPolicy.ConvertName(property.ModelName); - } - else - { - storageName = property.ModelName; - } - - if (keyPropertyWithReservedName) - { - // Some providers (Weaviate, Cosmos NoSQL) have a fixed, reserved storage name for keys (id), and at the same time use an external - // JSON serializer to serialize the entire user POCO. Since the serializer is unaware of the reserved storage name, it will produce - // a storage name as usual, based on the .NET property's name, possibly with a naming policy applied to it. The connector then needs - // to look that up and replace with the reserved name. - ((KeyPropertyModel)property).SerializedKeyName = storageName; - } - else - { - property.StorageName = storageName; - } - } - } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModel.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModel.cs deleted file mode 100644 index 7f837038dfa6..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModel.cs +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Linq.Expressions; -using System.Reflection; -using System.Runtime.CompilerServices; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -/// -/// Represents a record in a vector store collection. -/// This is an internal support type meant for use by connectors only and not by applications. -/// -[Experimental("MEVD9001")] -public sealed class CollectionModel -{ - private readonly Type _recordType; - private readonly Func _recordFactory; - - private KeyPropertyModel? _singleKeyProperty; - private VectorPropertyModel? _singleVectorProperty; - private DataPropertyModel? _singleFullTextSearchProperty; - - /// - /// Gets the key properties of the record. - /// - public IReadOnlyList KeyProperties { get; } - - /// - /// Gets the data properties of the record. - /// - public IReadOnlyList DataProperties { get; } - - /// - /// Gets the vector properties of the record. - /// - public IReadOnlyList VectorProperties { get; } - - /// - /// Gets all properties of the record, of all types. - /// - public IReadOnlyList Properties { get; } - - /// - /// Gets all properties of the record, of all types, indexed by their model name. - /// - public IReadOnlyDictionary PropertyMap { get; } - - /// - /// Gets a value that indicates whether any of the vector properties in the model require embedding generation. - /// - public bool EmbeddingGenerationRequired { get; } - - internal CollectionModel( - Type recordType, - Func recordFactory, - IReadOnlyList keyProperties, - IReadOnlyList dataProperties, - IReadOnlyList vectorProperties, - IReadOnlyDictionary propertyMap) - { - this._recordType = recordType; - this._recordFactory = recordFactory; - - this.KeyProperties = keyProperties; - this.DataProperties = dataProperties; - this.VectorProperties = vectorProperties; - this.PropertyMap = propertyMap; - this.Properties = propertyMap.Values.ToList(); - - this.EmbeddingGenerationRequired = vectorProperties.Any(p => p.EmbeddingType != p.Type); - } - - /// - /// Returns the single key property in the model, and throws if there are multiple key properties. - /// - public KeyPropertyModel KeyProperty => this._singleKeyProperty ??= this.KeyProperties.Single(); - - /// - /// Returns the single vector property in the model, and throws if there are multiple vector properties. - /// Suitable for connectors where validation is in place for single vectors only (). - /// - public VectorPropertyModel VectorProperty => this._singleVectorProperty ??= this.VectorProperties.Single(); - - /// - /// Instantiates a new record of the specified type. - /// - // TODO: the pattern of first instantiating via parameterless constructor and then populating the properties isn't compatible - // with read-only types, where properties have no setters. Supporting those would be problematic given the that different - // connectors have completely different representations of the data coming back from the database, and which needs to be - // populated. - public TRecord CreateRecord() - { - Debug.Assert(typeof(TRecord) == this._recordType, "Type mismatch between record type and model type."); - - return (TRecord)this._recordFactory(); - } - - /// - /// Gets the vector property with the provided name if a name is provided, and falls back - /// to a vector property in the schema if not. - /// - /// The search options, which defines the vector property name. - /// The provided property name is not a valid text data property name.ORNo name was provided and there's more than one vector property. - public VectorPropertyModel GetVectorPropertyOrSingle(VectorSearchOptions searchOptions) - { - if (searchOptions.VectorProperty is not null) - { - return this.GetMatchingProperty(searchOptions.VectorProperty, data: false); - } - - // If vector property name is not provided, check if there is a single vector property, or throw if there are no vectors or more than one. - // TODO: Make a single switch expression + coalesce from the following - dotnet format fails on it for now - if (this._singleVectorProperty is null) - { - switch (this.VectorProperties) - { - case [var singleProperty]: - this._singleVectorProperty = singleProperty; - break; - - case { Count: 0 }: - throw new InvalidOperationException($"The '{this._recordType.Name}' type does not have any vector properties."); - - default: - throw new InvalidOperationException($"The '{this._recordType.Name}' type has multiple vector properties, please specify your chosen property via options."); - } - } - - return this._singleVectorProperty; - } - - /// - /// Gets the text data property with the provided name that has full text search indexing enabled, or falls back - /// to a text data property in the schema if no name is provided. - /// - /// The full text search property selector. - /// The provided property name is not a valid text data property name.ORNo name was provided and there's more than one text data property with full text search indexing enabled. - public DataPropertyModel GetFullTextDataPropertyOrSingle(Expression>? expression) - { - if (expression is not null) - { - var property = this.GetMatchingProperty(expression, data: true); - - return property.IsFullTextIndexed - ? property - : throw new InvalidOperationException($"The property '{property.ModelName}' on '{this._recordType.Name}' must have full text search indexing enabled."); - } - - if (this._singleFullTextSearchProperty is null) - { - // If text data property name is not provided, check if a single full text indexed text property exists or throw otherwise. - var fullTextStringProperties = this.DataProperties - .Where(l => l.Type == typeof(string) && l.IsFullTextIndexed) - .ToList(); - - // If text data property name is not provided, check if a single full text indexed text property exists or throw otherwise. - switch (fullTextStringProperties) - { - // If there is a single property, use it. - // If there are no properties, throw. - // If there are multiple properties, throw. - case [var singleProperty]: - this._singleFullTextSearchProperty = singleProperty; - break; - - case { Count: 0 }: - throw new InvalidOperationException($"The '{this._recordType.Name}' type does not have any text data properties that have full text indexing enabled."); - - default: - throw new InvalidOperationException($"The '{this._recordType.Name}' type has multiple text data properties that have full text indexing enabled, please specify your chosen property via options."); - } - } - - return this._singleFullTextSearchProperty; - } - - /// - /// Gets the data or key property selected by the provided expression. - /// - /// The property selector. - /// The provided property name is not a valid data or key property name. - public PropertyModel GetDataOrKeyProperty(Expression> expression) - => this.GetMatchingProperty(expression, data: true); - - private TProperty GetMatchingProperty(Expression> expression, bool data) - where TProperty : PropertyModel - { - var node = expression.Body; - - // First, unwrap any object convert node: r => (object)r.PropertyName becomes r => r.PropertyName - if (expression.Body is UnaryExpression { NodeType: ExpressionType.Convert } convert - && convert.Type == typeof(object)) - { - node = convert.Operand; - } - - var propertyName = node switch - { - // Simple member expression over the lambda parameter (r => r.PropertyName) - MemberExpression { Member: PropertyInfo clrProperty } member when member.Expression == expression.Parameters[0] - => clrProperty.Name, - - // Dictionary access over the lambda parameter, in dynamic mapping (r => r["PropertyName"]) - MethodCallExpression { Method.Name: "get_Item", Arguments: [var keyExpression] } methodCall - => keyExpression switch - { - ConstantExpression { Value: string text } => text, - MemberExpression field when TryGetCapturedValue(field, out object? capturedValue) && capturedValue is string text => text, - _ => throw new InvalidOperationException("Invalid dictionary key expression") - }, - - _ => throw new InvalidOperationException("Property selector lambda is invalid") - }; - - if (!this.PropertyMap.TryGetValue(propertyName, out var property)) - { - throw new InvalidOperationException($"Property '{propertyName}' could not be found."); - } - - return property is TProperty typedProperty - ? typedProperty - : throw new InvalidOperationException($"Property '{propertyName}' isn't of type '{typeof(TProperty).Name}'."); - - static bool TryGetCapturedValue(Expression expression, out object? capturedValue) - { - if (expression is MemberExpression { Expression: ConstantExpression constant, Member: FieldInfo fieldInfo } - && constant.Type.Attributes.HasFlag(TypeAttributes.NestedPrivate) - && Attribute.IsDefined(constant.Type, typeof(CompilerGeneratedAttribute), inherit: true)) - { - capturedValue = fieldInfo.GetValue(constant.Value); - return true; - } - - capturedValue = null; - return false; - } - } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModelBuilder.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModelBuilder.cs deleted file mode 100644 index 3b6f6ccc6b51..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModelBuilder.cs +++ /dev/null @@ -1,609 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Reflection; -using Microsoft.Extensions.AI; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -/// -/// Represents a builder for a . -/// This is an internal support type meant for use by connectors only and not by applications. -/// -/// This class is single-use only, and not thread-safe. -[Experimental("MEVD9001")] -public abstract class CollectionModelBuilder -{ - /// - /// Gets the options for building the model. - /// - protected CollectionModelBuildingOptions Options { get; } - - /// - /// Gets the key properties of the record. - /// - protected List KeyProperties { get; } = []; - - /// - /// Gets the data properties of the record. - /// - protected List DataProperties { get; } = []; - - /// - /// Gets the vector properties of the record. - /// - protected List VectorProperties { get; } = []; - - /// - /// Gets all properties of the record, of all types. - /// - protected IEnumerable Properties => this.PropertyMap.Values; - - /// - /// Gets all properties of the record, of all types, indexed by their model name. - /// - protected Dictionary PropertyMap { get; } = []; - - /// - /// Gets the default embedding generator to use for vector properties, when none is specified at the property or collection level. - /// - protected IEmbeddingGenerator? DefaultEmbeddingGenerator { get; private set; } - - /// - /// Gets the collection's generic key type parameter (TKey), if provided. - /// Used by to validate that TKey corresponds to the key property type on the model. - /// - protected Type? KeyType { get; private set; } - - /// - /// Constructs a new . - /// - protected CollectionModelBuilder(CollectionModelBuildingOptions options) - => this.Options = options; - - /// - /// Builds and returns an from the given and . - /// - /// The CLR type of the record. - /// The collection's generic key type parameter (TKey), used to validate correspondence with the key property type. - /// An optional record definition that overrides attribute-based configuration. - /// An optional default embedding generator for vector properties. - [RequiresDynamicCode("This model building variant is not compatible with NativeAOT. See BuildDynamic() for dynamic mapping, and a third variant accepting source-generated delegates will be introduced in the future.")] - [RequiresUnreferencedCode("This model building variant is not compatible with trimming. See BuildDynamic() for dynamic mapping, and a third variant accepting source-generated delegates will be introduced in the future.")] - public virtual CollectionModel Build(Type recordType, Type keyType, VectorStoreCollectionDefinition? definition, IEmbeddingGenerator? defaultEmbeddingGenerator) - { - this.KeyType = keyType; - - if (recordType == typeof(Dictionary)) - { - throw new ArgumentException("Dynamic mapping with Dictionary requires calling BuildDynamic()."); - } - - this.DefaultEmbeddingGenerator = definition?.EmbeddingGenerator ?? defaultEmbeddingGenerator; - - // Build a lookup of definition properties by name for matching with CLR properties. - Dictionary? definitionByName = null; - if (definition is not null) - { - definitionByName = []; - foreach (var p in definition.Properties) - { - definitionByName[p.Name] = p; - } - } - - // Process CLR properties, matching to definition properties where available. - // TODO: This traverses the CLR type's properties, making it incompatible with trimming (and NativeAOT). - // TODO: We could put [DynamicallyAccessedMembers] to preserve all properties, but that approach wouldn't - // TODO: work with hierarchical data models (#10957). - foreach (var clrProperty in recordType.GetProperties()) - { - VectorStoreProperty? definitionProperty = null; - _ = definitionByName?.TryGetValue(clrProperty.Name, out definitionProperty); - - this.ProcessProperty(clrProperty, definitionProperty, recordType); - } - - // Go over the properties, configure POCO accessors and validate type compatibility. - foreach (var property in this.Properties) - { - var clrProperty = recordType.GetProperty(property.ModelName) - ?? throw new InvalidOperationException($"Property '{property.ModelName}' not found on CLR type '{recordType.FullName}'."); - - var clrPropertyType = clrProperty.PropertyType; - if ((Nullable.GetUnderlyingType(clrPropertyType) ?? clrPropertyType) != (Nullable.GetUnderlyingType(property.Type) ?? property.Type)) - { - throw new InvalidOperationException( - $"Property '{property.ModelName}' has a different CLR type in the record definition ('{property.Type.Name}') and on the .NET property ('{clrProperty.PropertyType}')."); - } - - property.ConfigurePocoAccessors(clrProperty); - } - - this.Customize(); - this.Validate(recordType, definition); - - // Extra validation for non-dynamic mapping scenarios: ensure the type has a parameterless constructor. - if (!this.Options.UsesExternalSerializer && recordType.GetConstructor(Type.EmptyTypes) is null) - { - throw new NotSupportedException($"Type '{recordType.Name}' must have a parameterless constructor."); - } - - return new(recordType, () => Activator.CreateInstance(recordType)!, this.KeyProperties, this.DataProperties, this.VectorProperties, this.PropertyMap); - } - - /// - /// Builds and returns an for dynamic mapping scenarios from the given . - /// - /// The record definition describing the collection's schema. - /// An optional default embedding generator for vector properties. - public virtual CollectionModel BuildDynamic(VectorStoreCollectionDefinition definition, IEmbeddingGenerator? defaultEmbeddingGenerator) - { - if (definition is null) - { - throw new ArgumentException("Vector store record definition must be provided for dynamic mapping."); - } - - this.DefaultEmbeddingGenerator = defaultEmbeddingGenerator; - - foreach (var defProp in definition.Properties) - { - this.ProcessProperty(clrProperty: null, defProp, type: null); - } - - this.Customize(); - this.Validate(type: null, definition); - - foreach (var property in this.Properties) - { - property.ConfigureDynamicAccessors(); - } - - return new(typeof(Dictionary), static () => new Dictionary(), this.KeyProperties, this.DataProperties, this.VectorProperties, this.PropertyMap); - } - - /// - /// As part of building the model, this method processes a single property, accepting both a CLR - /// (from which attributes are read) and a from the user-provided record definition. - /// Either may be , but not both. - /// When both are provided, the record definition values override attribute-configured values. - /// - protected virtual void ProcessProperty(PropertyInfo? clrProperty, VectorStoreProperty? definitionProperty, Type? type) - { - Debug.Assert(clrProperty is not null || definitionProperty is not null); - - VectorStoreKeyAttribute? keyAttribute = null; - VectorStoreDataAttribute? dataAttribute = null; - VectorStoreVectorAttribute? vectorAttribute = null; - - if (clrProperty is not null) - { - // Read attributes from CLR property. - keyAttribute = clrProperty.GetCustomAttribute(); - dataAttribute = clrProperty.GetCustomAttribute(); - vectorAttribute = clrProperty.GetCustomAttribute(); - - // Validate that at most one mapping attribute is present. - if ((keyAttribute is not null ? 1 : 0) + (dataAttribute is not null ? 1 : 0) + (vectorAttribute is not null ? 1 : 0) > 1) - { - throw new InvalidOperationException( - $"Property '{type!.Name}.{clrProperty.Name}' has multiple of {nameof(VectorStoreKeyAttribute)}, {nameof(VectorStoreDataAttribute)} or {nameof(VectorStoreVectorAttribute)}. Only one of these attributes can be specified on a property."); - } - - // If no mapping attribute and no definition, skip this property. - if (keyAttribute is null && dataAttribute is null && vectorAttribute is null && definitionProperty is null) - { - return; - } - - // Validate kind compatibility between attribute and definition. - if (definitionProperty is not null - && ((keyAttribute is not null && definitionProperty is not VectorStoreKeyProperty) - || (dataAttribute is not null && definitionProperty is not VectorStoreDataProperty) - || (vectorAttribute is not null && definitionProperty is not VectorStoreVectorProperty))) - { - string definitionKind = definitionProperty switch - { - VectorStoreKeyProperty => "key", - VectorStoreDataProperty => "data", - VectorStoreVectorProperty => "vector", - _ => throw new ArgumentException($"Unknown type '{definitionProperty.GetType().FullName}' in vector store record definition.") - }; - - throw new InvalidOperationException( - $"Property '{clrProperty.Name}' is present in the {nameof(VectorStoreCollectionDefinition)} as a {definitionKind} property, but the .NET property on type '{type?.Name}' has an incompatible attribute."); - } - } - - string propertyName = clrProperty?.Name ?? definitionProperty!.Name; - Type propertyType = clrProperty?.PropertyType - ?? definitionProperty!.Type - ?? throw new InvalidOperationException(VectorDataStrings.MissingTypeOnPropertyDefinition(definitionProperty!)); - - PropertyModel property; - string? attributeStorageName = null; - - if (keyAttribute is not null || definitionProperty is VectorStoreKeyProperty) - { - var keyProperty = new KeyPropertyModel(propertyName, propertyType); - - if (keyAttribute is not null) - { - keyProperty.IsAutoGenerated = keyAttribute.IsAutoGeneratedNullable ?? this.SupportsKeyAutoGeneration(keyProperty.Type); - attributeStorageName = keyAttribute.StorageName; - } - - // Definition values override attribute values. - if (definitionProperty is VectorStoreKeyProperty defKey) - { - keyProperty.IsAutoGenerated = defKey.IsAutoGenerated ?? this.SupportsKeyAutoGeneration(keyProperty.Type); - } - - this.KeyProperties.Add(keyProperty); - property = keyProperty; - } - else if (dataAttribute is not null || definitionProperty is VectorStoreDataProperty) - { - var dataProperty = new DataPropertyModel(propertyName, propertyType); - - if (dataAttribute is not null) - { - dataProperty.IsIndexed = dataAttribute.IsIndexed; - dataProperty.IsFullTextIndexed = dataAttribute.IsFullTextIndexed; - attributeStorageName = dataAttribute.StorageName; - } - - // Definition values override attribute values. - if (definitionProperty is VectorStoreDataProperty defData) - { - dataProperty.IsIndexed = defData.IsIndexed; - dataProperty.IsFullTextIndexed = defData.IsFullTextIndexed; - } - - this.DataProperties.Add(dataProperty); - property = dataProperty; - } - else if (vectorAttribute is not null || definitionProperty is VectorStoreVectorProperty) - { - // If a definition exists, create via the definition to preserve generic type info (VectorStoreVectorProperty). - var vectorProperty = definitionProperty is VectorStoreVectorProperty defVec - ? defVec.CreatePropertyModel() - : new VectorPropertyModel(propertyName, propertyType); - - if (vectorAttribute is not null) - { - vectorProperty.Dimensions = vectorAttribute.Dimensions; - vectorProperty.IndexKind = vectorAttribute.IndexKind; - vectorProperty.DistanceFunction = vectorAttribute.DistanceFunction; - attributeStorageName = vectorAttribute.StorageName; - } - - // Definition values override attribute values. - if (definitionProperty is VectorStoreVectorProperty defVectorProp) - { - vectorProperty.Dimensions = defVectorProp.Dimensions; - - if (defVectorProp.IndexKind is not null) - { - vectorProperty.IndexKind = defVectorProp.IndexKind; - } - - if (defVectorProp.DistanceFunction is not null) - { - vectorProperty.DistanceFunction = defVectorProp.DistanceFunction; - } - } - - this.ConfigureVectorPropertyEmbedding( - vectorProperty, - (definitionProperty as VectorStoreVectorProperty)?.EmbeddingGenerator ?? this.DefaultEmbeddingGenerator, - (definitionProperty as VectorStoreVectorProperty)?.EmbeddingType); - - this.VectorProperties.Add(vectorProperty); - property = vectorProperty; - } - else - { - throw new UnreachableException(); - } - - // Apply storage name: attribute first, then definition (which takes precedence). - this.SetPropertyStorageName(property, attributeStorageName, type); - if (definitionProperty is not null) - { - this.SetPropertyStorageName(property, definitionProperty.StorageName, type); - } - - if (definitionProperty?.ProviderAnnotations is not null) - { - property.ProviderAnnotations = new Dictionary(definitionProperty.ProviderAnnotations); - } - - if (clrProperty is not null) - { - property.PropertyInfo = clrProperty; - } - - this.PropertyMap.Add(propertyName, property); - } - - private void SetPropertyStorageName(PropertyModel property, string? storageName, Type? type) - { - if (property is KeyPropertyModel && this.Options.ReservedKeyStorageName is not null) - { - // If we have ReservedKeyStorageName, there can only be a single key property (validated in the constructor) - property.StorageName = this.Options.ReservedKeyStorageName; - return; - } - - if (storageName is null) - { - return; - } - - // If a custom serializer is used (e.g. JsonSerializer), it would ignore our own attributes/config, and - // our model needs to be in sync with the serializer's behavior (for e.g. storage names in filters). - // So we ignore the config here as well. - // TODO: Consider throwing here instead of ignoring - if (this.Options.UsesExternalSerializer && type != null) - { - return; - } - - property.StorageName = storageName; - } - - /// - /// Gets the embedding types supported by this provider, in priority order. - /// The first type whose embedding generator is compatible with the input type will be used. - /// - /// - /// Override this property in connectors that support additional embedding types beyond of . - /// - protected virtual IReadOnlyList EmbeddingGenerationDispatchers { get; } - = [EmbeddingGenerationDispatcher.Create>()]; - - /// - /// Attempts to resolve the embedding type for the given vector property, iterating over in priority order. - /// - private (Type? EmbeddingType, EmbeddingGenerationDispatcher? Handler) ResolveEmbeddingType( - VectorPropertyModel vectorProperty, - IEmbeddingGenerator embeddingGenerator, - Type? userRequestedEmbeddingType) - { - foreach (var supported in this.EmbeddingGenerationDispatchers) - { - if (supported.ResolveEmbeddingType(vectorProperty, embeddingGenerator, userRequestedEmbeddingType) is { } resolved) - { - return (resolved, supported); - } - } - - return (null, null); - } - - /// - /// Resolves the embedding handler for a native vector property type, where embedding generation is only needed for search. - /// Since the property type is already a valid native type, we only check if the generator can produce the - /// embedding output type (regardless of input type, which is only known at search time). - /// - private EmbeddingGenerationDispatcher? ResolveSearchOnlyEmbeddingHandler(VectorPropertyModel vectorProperty, IEmbeddingGenerator embeddingGenerator) - { - foreach (var supported in this.EmbeddingGenerationDispatchers) - { - if (supported.CanGenerateEmbedding(vectorProperty, embeddingGenerator)) - { - return supported; - } - } - - return null; - } - - /// - /// Configures embedding generation for a vector property. Sets the embedding generator, resolves the embedding type, - /// and assigns the appropriate . - /// - /// - /// If the property's type is natively supported (e.g. of ), the embedding type - /// is set to the property's type; if a generator is also configured, a search-only dispatcher is resolved so that search can convert - /// arbitrary inputs (e.g. string) to embeddings. - /// Otherwise, if a generator is configured, the embedding type is resolved from it. If resolution fails, the embedding type remains - /// and an error is deferred to the validation phase. - /// - private void ConfigureVectorPropertyEmbedding( - VectorPropertyModel vectorProperty, - IEmbeddingGenerator? embeddingGenerator, - Type? userRequestedEmbeddingType) - { - vectorProperty.EmbeddingGenerator = embeddingGenerator; - - if (this.IsVectorPropertyTypeValid(vectorProperty.Type, out _)) - { - if (userRequestedEmbeddingType is not null && userRequestedEmbeddingType != vectorProperty.Type) - { - throw new InvalidOperationException(VectorDataStrings.DifferentEmbeddingTypeSpecifiedForNativelySupportedType(vectorProperty, userRequestedEmbeddingType)); - } - - vectorProperty.EmbeddingType = vectorProperty.Type; - - // Even for native types, if an embedding generator is configured, resolve the dispatcher - // so that search can convert arbitrary inputs (e.g. string) to embeddings. - if (embeddingGenerator is not null) - { - vectorProperty.EmbeddingGenerationDispatcher = this.ResolveSearchOnlyEmbeddingHandler(vectorProperty, embeddingGenerator); - } - } - else if (embeddingGenerator is not null) - { - // The property type isn't a valid embedding type, but an embedding generator is configured. - // Try to resolve the embedding type from it: if the configured generator supports translating the input type (e.g. string) to - // an output type supported by the provider, we set that as the embedding type. - // If this fails, EmbeddingType remains null and we defer the error to the validation phase. - var (embeddingType, handler) = this.ResolveEmbeddingType(vectorProperty, embeddingGenerator, userRequestedEmbeddingType); - vectorProperty.EmbeddingType = embeddingType; - vectorProperty.EmbeddingGenerationDispatcher = handler; - } - - // If the property type isn't valid and there's no embedding generator, that's an error. - // But we throw later, in validation, to allow for provider customization to correct this invalid state after this step. - } - - /// - /// Extension hook for connectors to be able to customize the model. - /// - protected virtual void Customize() - { - } - - /// - /// Validates the model after all properties have been processed. - /// - protected virtual void Validate(Type? type, VectorStoreCollectionDefinition? definition) - { - if (this.KeyProperties.Count > 1) - { - throw new NotSupportedException($"Multiple key properties found on {TypeMessage()}the provided {nameof(VectorStoreCollectionDefinition)} while only one is supported."); - } - - if (this.KeyProperties.Count == 0) - { - throw new NotSupportedException($"No key property found on {TypeMessage()}the provided {nameof(VectorStoreCollectionDefinition)} while at least one is required."); - } - - if (this.Options.RequiresAtLeastOneVector && this.VectorProperties.Count == 0) - { - throw new NotSupportedException($"No vector property found on {TypeMessage()}the provided {nameof(VectorStoreCollectionDefinition)} while at least one is required."); - } - - if (!this.Options.SupportsMultipleVectors && this.VectorProperties.Count > 1) - { - throw new NotSupportedException($"Multiple vector properties found on {TypeMessage()}the provided {nameof(VectorStoreCollectionDefinition)} while only one is supported."); - } - - var storageNameMap = new Dictionary(); - - foreach (var property in this.PropertyMap.Values) - { - this.ValidateProperty(property, definition); - - if (storageNameMap.TryGetValue(property.StorageName, out var otherproperty)) - { - throw new InvalidOperationException($"Property '{property.ModelName}' is being mapped to storage name '{property.StorageName}', but property '{otherproperty.ModelName}' is already mapped to the same storage name."); - } - - storageNameMap[property.StorageName] = property; - } - - string TypeMessage() => type is null ? "" : $"type '{type.Name}' or "; - } - - /// - /// Validates a single property, performing validation on it. - /// - protected virtual void ValidateProperty(PropertyModel propertyModel, VectorStoreCollectionDefinition? definition) - { - var type = propertyModel.Type; - - Debug.Assert(propertyModel.Type is not null); - - switch (propertyModel) - { - case KeyPropertyModel keyProperty: - if (keyProperty.IsAutoGenerated && !this.SupportsKeyAutoGeneration(keyProperty.Type)) - { - throw new NotSupportedException( - $"Property '{keyProperty.ModelName}' is configured for auto-generation, but key properties of type '{keyProperty.Type.Name}' do not support auto-generation."); - } - - this.ValidateKeyProperty(keyProperty); - break; - - case DataPropertyModel dataProperty: - if (!this.IsDataPropertyTypeValid(dataProperty.Type, out var supportedTypes)) - { - throw new NotSupportedException( - $"Property '{dataProperty.ModelName}' has unsupported type '{type.Name}'. Data properties must be one of the supported types: {supportedTypes}."); - } - break; - - case VectorPropertyModel vectorProperty: - if (vectorProperty.EmbeddingType is null) - { - if (this.IsVectorPropertyTypeValid(vectorProperty.Type, out string? supportedVectorTypes)) - { - throw new UnreachableException("EmbeddingType cannot be null when the property type is supported."); - } - - if (vectorProperty.EmbeddingGenerator is null) - { - throw new InvalidOperationException(VectorDataStrings.UnsupportedVectorPropertyWithoutEmbeddingGenerator(vectorProperty)); - } - - // If the user has configured a desired embedding type (done to use en embedding type other than the provider's default one), throw errors tailored to that. - // Throw errors related to that. - var userRequestedEmbeddingType = definition?.Properties.OfType().SingleOrDefault(p => p.Name == vectorProperty.ModelName)?.EmbeddingType; - if (userRequestedEmbeddingType is not null) - { - throw new InvalidOperationException(this.IsVectorPropertyTypeValid(userRequestedEmbeddingType, out _) - ? VectorDataStrings.ConfiguredEmbeddingTypeIsUnsupportedByTheGenerator(vectorProperty, userRequestedEmbeddingType, supportedVectorTypes) - : VectorDataStrings.ConfiguredEmbeddingTypeIsUnsupportedByTheProvider(vectorProperty, userRequestedEmbeddingType, supportedVectorTypes)); - } - - throw new InvalidOperationException(VectorDataStrings.IncompatibleEmbeddingGenerator(vectorProperty, vectorProperty.EmbeddingGenerator, supportedVectorTypes)); - } - - if (!this.IsVectorPropertyTypeValid(vectorProperty.EmbeddingType, out string? supportedVectorTypes2)) - { - // Should in principle never happen, only with incorrect provider customization. - throw new InvalidOperationException($"Property '{vectorProperty.ModelName}' has unsupported embedding type '{vectorProperty.EmbeddingType.Name}'. Vector properties must be one of the supported types: {supportedVectorTypes2}."); - } - - if (vectorProperty.Dimensions <= 0) - { - throw new InvalidOperationException($"Vector property '{propertyModel.ModelName}' must have a positive number of dimensions."); - } - - break; - - default: - throw new UnreachableException(); - } - } - - /// - /// Configures auto-generation for the given key property. - /// Defaults to configuring key properties as auto-generated, and throwing if auto-generation is requested for - /// any other type. - /// - protected virtual bool SupportsKeyAutoGeneration(Type keyPropertyType) - => keyPropertyType == typeof(Guid); - - /// - /// Validates the key property. The default implementation validates that the collection's generic key type () - /// corresponds to the key property type on the model, if was provided. - /// Provider overrides should call the base implementation. - /// - protected virtual void ValidateKeyProperty(KeyPropertyModel keyProperty) - { - if (this.KeyType is not null && this.KeyType != typeof(object) && this.KeyType != keyProperty.Type) - { - throw new InvalidOperationException( - $"The collection's generic key type is '{this.KeyType.Name}', but the key property '{keyProperty.ModelName}' has type '{keyProperty.Type.Name}'. The generic key type must match the key property type."); - } - } - - /// - /// Validates that the .NET type for a data property is supported by the provider. - /// - protected abstract bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes); - - /// - /// Validates that the .NET type for a vector property is supported by the provider. - /// - protected abstract bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes); -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModelBuildingOptions.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModelBuildingOptions.cs deleted file mode 100644 index 28ae1f026af0..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/CollectionModelBuildingOptions.cs +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Diagnostics.CodeAnalysis; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -/// -/// Contains options affecting model building; passed to . -/// This is an internal support type meant for use by connectors only and not by applications. -/// -[Experimental("MEVD9001")] -public sealed class CollectionModelBuildingOptions -{ - /// - /// Gets a value that indicates whether multiple vector properties are supported. - /// - public required bool SupportsMultipleVectors { get; init; } - - /// - /// Gets a value that indicates whether at least one vector property is required. - /// - public required bool RequiresAtLeastOneVector { get; init; } - - /// - /// Gets a value that indicates whether an external serializer will be used (for example, System.Text.Json). - /// - public bool UsesExternalSerializer { get; init; } - - /// - /// Gets the special, reserved name for the key property of the database. - /// When set, the model builder manages the key storage name, and users cannot customize it. - /// - public string? ReservedKeyStorageName { get; init; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/DataPropertyModel.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/DataPropertyModel.cs deleted file mode 100644 index b2d800245054..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/DataPropertyModel.cs +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Diagnostics.CodeAnalysis; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -/// -/// Represents a data property on a vector store record. -/// This is an internal support type meant for use by connectors only and not by applications. -/// -[Experimental("MEVD9001")] -public class DataPropertyModel(string modelName, Type type) : PropertyModel(modelName, type) -{ - /// - /// Gets or sets a value indicating whether this data property is indexed. - /// - /// - /// The default is . - /// - public bool IsIndexed { get; set; } - - /// - /// Gets or sets a value indicating whether this data property is indexed for full-text search. - /// - /// - /// The default is . - /// - public bool IsFullTextIndexed { get; set; } - - /// - public override string ToString() - => $"{this.ModelName} (Data, {this.Type.Name})"; -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/EmbeddingGenerationDispatcher.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/EmbeddingGenerationDispatcher.cs deleted file mode 100644 index b1825cda295a..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/EmbeddingGenerationDispatcher.cs +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.AI; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -/// -/// Represents a supported embedding type for a vector store provider. -/// This is an internal support type meant for use by connectors only and not by applications. -/// -/// -/// Each instance encapsulates both build-time embedding type resolution and runtime embedding generation -/// for a specific subtype. -/// -[Experimental("MEVD9001")] -public abstract class EmbeddingGenerationDispatcher -{ - /// - /// Gets the type that this instance supports. - /// - public abstract Type EmbeddingType { get; } - - /// - /// Attempts to resolve the embedding type for the given , using the given . - /// - /// The resolved embedding type, or if the generator does not support this embedding type. - public abstract Type? ResolveEmbeddingType(VectorPropertyModel vectorProperty, IEmbeddingGenerator embeddingGenerator, Type? userRequestedEmbeddingType); - - /// - /// Generates embeddings of this type from the given , using the embedding generator configured on the . - /// - public abstract Task> GenerateEmbeddingsAsync(VectorPropertyModel vectorProperty, IEnumerable values, CancellationToken cancellationToken); - - /// - /// Generates a single embedding of this type from the given , using the embedding generator configured on the . - /// - public abstract Task GenerateEmbeddingAsync(VectorPropertyModel vectorProperty, object? value, CancellationToken cancellationToken); - - /// - /// Checks whether the given can produce embeddings of this type for any of the input types - /// supported by the given . - /// This is used for native vector property types (e.g., of ), where embedding generation - /// is only needed for search and the input type is not known at model-build time. - /// - public abstract bool CanGenerateEmbedding(VectorPropertyModel vectorProperty, IEmbeddingGenerator embeddingGenerator); - - /// - /// Creates a new for the given type. - /// - public static EmbeddingGenerationDispatcher Create() - where TEmbedding : Embedding - => new EmbeddingGenerationDispatcher(); -} - -/// -/// A implementation for a specific type. -/// This is an internal support type meant for use by connectors only and not by applications. -/// -[Experimental("MEVD9001")] -public sealed class EmbeddingGenerationDispatcher : EmbeddingGenerationDispatcher - where TEmbedding : Embedding -{ - /// - public override Type EmbeddingType => typeof(TEmbedding); - - /// - public override Type? ResolveEmbeddingType(VectorPropertyModel vectorProperty, IEmbeddingGenerator embeddingGenerator, Type? userRequestedEmbeddingType) - => vectorProperty.ResolveEmbeddingType(embeddingGenerator, userRequestedEmbeddingType); - - /// - public override bool CanGenerateEmbedding(VectorPropertyModel vectorProperty, IEmbeddingGenerator embeddingGenerator) - => vectorProperty.CanGenerateEmbedding(embeddingGenerator); - - /// - public override Task> GenerateEmbeddingsAsync(VectorPropertyModel vectorProperty, IEnumerable values, CancellationToken cancellationToken) - => vectorProperty.GenerateEmbeddingsCoreAsync(values, cancellationToken); - - /// - public override Task GenerateEmbeddingAsync(VectorPropertyModel vectorProperty, object? value, CancellationToken cancellationToken) - => vectorProperty.GenerateEmbeddingCoreAsync(value, cancellationToken); -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterPreprocessingOptions.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterPreprocessingOptions.cs deleted file mode 100644 index 4b0cc8f07e2b..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterPreprocessingOptions.cs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Diagnostics.CodeAnalysis; - -namespace Microsoft.Extensions.VectorData.ProviderServices.Filter; - -/// -/// Options for filter expression preprocessing. -/// This is an internal support type meant for use by connectors only and not by applications. -/// -[Experimental("MEVD9001")] -public class FilterPreprocessingOptions -{ - /// - /// Whether the connector supports parameterization. - /// - /// - /// If , the visitor will inline captured variables and constant member accesses as simple constant nodes. - /// If , these will instead be replaced with nodes. - /// - public bool SupportsParameterization { get; init; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterTranslatorBase.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterTranslatorBase.cs deleted file mode 100644 index ad847b34b1d1..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterTranslatorBase.cs +++ /dev/null @@ -1,403 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Linq.Expressions; -using System.Reflection; - -namespace Microsoft.Extensions.VectorData.ProviderServices.Filter; - -/// -/// Base class for filter translators used by vector data connectors. -/// Provides common functionality for preprocessing filter expressions and matching common patterns. -/// This is an internal support type meant for use by connectors only and not by applications. -/// -[Experimental("MEVD9001")] -public abstract class FilterTranslatorBase -{ - /// - /// The collection model for the current translation operation. - /// - protected CollectionModel Model { get; private set; } = null!; - - /// - /// The parameter expression representing the record in the filter lambda. - /// - protected ParameterExpression RecordParameter { get; private set; } = null!; - - /// - /// Preprocesses the filter expression before translation. - /// Sets and , runs the preprocessing visitor, - /// and returns the preprocessed expression. - /// - /// The filter lambda expression to preprocess. - /// The collection model containing property information. - /// Options controlling the preprocessing behavior. - /// The preprocessed expression ready for translation. - protected Expression PreprocessFilter(LambdaExpression lambdaExpression, CollectionModel model, FilterPreprocessingOptions options) - { - this.Model = model; - this.RecordParameter = lambdaExpression.Parameters[0]; - - var preprocessor = new FilterTranslationPreprocessor(options.SupportsParameterization); - return preprocessor.Preprocess(lambdaExpression.Body); - } - - /// - /// Tries to match a Contains method call expression and extract the source collection and item expressions. - /// - /// The method call expression to match. - /// When successful, the source collection expression. - /// When successful, the item expression being searched for. - /// if the expression is a recognized Contains pattern; otherwise, . - protected static bool TryMatchContains( - MethodCallExpression methodCall, - [NotNullWhen(true)] out Expression? source, - [NotNullWhen(true)] out Expression? item) - { - switch (methodCall) - { - // Enumerable.Contains() - case { Method.Name: nameof(Enumerable.Contains), Arguments: [var src, var itm] } - when methodCall.Method.DeclaringType == typeof(Enumerable): - source = src; - item = itm; - return true; - - // List.Contains() - case - { - Method: - { - Name: nameof(Enumerable.Contains), - DeclaringType: { IsGenericType: true } declaringType - }, - Object: Expression src, - Arguments: [var itm] - } when declaringType.GetGenericTypeDefinition() == typeof(List<>): - source = src; - item = itm; - return true; - - // C# 14 made changes to overload resolution to prefer Span-based overloads when those exist ("first-class spans"); - // this makes MemoryExtensions.Contains() be resolved rather than Enumerable.Contains() (see above). - // MemoryExtensions.Contains() also accepts a Span argument for the source, adding an implicit cast we need to remove. - // See https://github.com/dotnet/runtime/issues/109757 for more context. - // Note that MemoryExtensions.Contains has an optional 3rd ComparisonType parameter; we only match when - // it's null. - case { Method.Name: nameof(MemoryExtensions.Contains), Arguments: [var spanArg, var itm, ..] } - when methodCall.Method.DeclaringType == typeof(MemoryExtensions) - && (methodCall.Arguments.Count is 2 - || (methodCall.Arguments.Count is 3 && methodCall.Arguments[2] is ConstantExpression { Value: null })) - && TryUnwrapSpanImplicitCast(spanArg, out var src): - source = src; - item = itm; - return true; - - default: - source = null; - item = null; - return false; - } - } - - /// - /// Tries to bind an expression to a property in the collection model. - /// - /// The expression to bind. - /// When successful, the property model that was bound. - /// if the expression was successfully bound to a property; otherwise, . - protected virtual bool TryBindProperty(Expression expression, [NotNullWhen(true)] out PropertyModel? propertyModel) - { - var unwrappedExpression = expression; - while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) - { - unwrappedExpression = convert.Operand; - } - - var modelName = unwrappedExpression switch - { - // Regular member access for strongly-typed POCO binding (e.g. r => r.SomeInt == 8) - MemberExpression memberExpression when memberExpression.Expression == this.RecordParameter - => memberExpression.Member.Name, - - // Dictionary lookup for weakly-typed dynamic binding (e.g. r => r["SomeInt"] == 8) - MethodCallExpression - { - Method: { Name: "get_Item", DeclaringType: var declaringType }, - Arguments: [ConstantExpression { Value: string keyName }] - } methodCall when methodCall.Object == this.RecordParameter && declaringType == typeof(Dictionary) - => keyName, - - _ => null - }; - - if (modelName is null) - { - propertyModel = null; - return false; - } - - if (!this.Model.PropertyMap.TryGetValue(modelName, out propertyModel)) - { - throw new InvalidOperationException($"Property name '{modelName}' provided as part of the filter clause is not a valid property name."); - } - - // Now that we have the property, go over all wrapping Convert nodes again to ensure that they're compatible with the property type - var unwrappedPropertyType = Nullable.GetUnderlyingType(propertyModel.Type) ?? propertyModel.Type; - unwrappedExpression = expression; - while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) - { - var convertType = Nullable.GetUnderlyingType(convert.Type) ?? convert.Type; - if (convertType != unwrappedPropertyType && convertType != typeof(object)) - { - throw new InvalidCastException($"Property '{propertyModel.ModelName}' is being cast to type '{convert.Type.Name}', but its configured type is '{propertyModel.Type.Name}'."); - } - - unwrappedExpression = convert.Operand; - } - - return true; - } - - /// - /// Tries to unwrap an implicit cast to Span or ReadOnlySpan that may be present in expressions - /// when C# 14's first-class span support causes MemoryExtensions methods to be resolved. - /// - /// The expression to unwrap. - /// When successful, the unwrapped expression. - /// if a span implicit cast was unwrapped; otherwise, . - protected static bool TryUnwrapSpanImplicitCast(Expression expression, [NotNullWhen(true)] out Expression? result) - { - // Different versions of the compiler seem to generate slightly different expression tree representations for this - // implicit cast: - var (unwrapped, castDeclaringType) = expression switch - { - UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Operand: var operand - } => (operand, implicitCastDeclaringType), - - MethodCallExpression - { - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Arguments: [var firstArgument] - } => (firstArgument, implicitCastDeclaringType), - - // After the preprocessor runs, the Convert node may have Method: null because the visitor - // recreates the UnaryExpression with a different operand type (QueryParameterExpression). - // Handle this case by checking if the target type is Span or ReadOnlySpan. - UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: null, - Type: { IsGenericType: true } targetType, - Operand: var operand - } when targetType.GetGenericTypeDefinition() is var gtd - && (gtd == typeof(Span<>) || gtd == typeof(ReadOnlySpan<>)) - => (operand, targetType), - - _ => (null, null) - }; - - // For the dynamic case, there's a Convert node representing an up-cast to object[]; unwrap that too. - // Also handle cases where the preprocessor adds a Convert node back to the array type. - while (unwrapped is UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: null, - Operand: var innerOperand - }) - { - unwrapped = innerOperand; - } - - if (unwrapped is not null - && castDeclaringType?.GetGenericTypeDefinition() is var genericTypeDefinition - && (genericTypeDefinition == typeof(Span<>) || genericTypeDefinition == typeof(ReadOnlySpan<>))) - { - result = unwrapped; - return true; - } - - result = null; - return false; - } - - #region FilterTranslationPreprocessor - - /// - /// A processor for user-provided filter expressions which performs various common transformations before actual translation takes place. - /// - private sealed class FilterTranslationPreprocessor : ExpressionVisitor - { - private readonly bool _supportsParameterization; - private List? _parameterNames; - - internal FilterTranslationPreprocessor(bool supportsParameterization) - { - this._supportsParameterization = supportsParameterization; - } - - internal Expression Preprocess(Expression node) - { - if (this._supportsParameterization) - { - this._parameterNames = []; - } - - return this.Visit(node); - } - - /// - protected override Expression VisitMember(MemberExpression node) - { - var visited = (MemberExpression)base.VisitMember(node); - - // This identifies field and property access over constants, which can be evaluated immediately. - // This covers captured variables, since those are actually member accesses over compiled-generated closure types: - // var x = 8; - // _ = await collection.SearchAsync(vector, top: 3, new() { Filter = r => r.Int == x }); - // - // This also covers member variables: - // _ = await collection.SearchAsync(vector, top: 3, new() { Filter = r => r.Int == this._x }); - // ... as "this" here is represented by a ConstantExpression node in the tree. - // - // Some databases - mostly relational ones - support out-of-band parameters which can be referenced via placeholders - // from the query itself. For those databases, we transform the member access to QueryParameterExpression (this simplifies things for those - // connectors, and centralizes the pattern matching in a single centralized place). - // For databases which don't support parameters, we simply inline the evaluated member access as a constant in the tree, so that translators don't - // even need to be aware of it. - - // Evaluate the MemberExpression to get the actual value, either for instance members (expression is a ConstantExpression) or for - // static members (expression is null). - object? baseValue; - switch (visited.Expression) - { - // Member access over constant (i.e. instance members) - case ConstantExpression { Value: var v }: - baseValue = v; - break; - - // Member constant over null (i.e. static members) - case null: - baseValue = null; - break; - - // Member constant over something that has already been parameterized (i.e. nested member access, e.g. r=> r.Int == this.SomeWrapper.Something) - case QueryParameterExpression p: - baseValue = p.Value; - - // The previous parameter is getting replaced by the new one we're creating here, so remove its name from the list of parameter names. - this._parameterNames!.Remove(p.Name); - break; - - default: - return visited; - } - - object? evaluatedValue; - - var memberInfo = visited.Member; - - switch (memberInfo) - { - case FieldInfo fieldInfo: - evaluatedValue = fieldInfo.GetValue(baseValue); - break; - - case PropertyInfo { GetMethod.IsStatic: false } propertyInfo when baseValue is null: - throw new InvalidOperationException($"Cannot access member '{propertyInfo.Name}' on null object."); - - case PropertyInfo propertyInfo: - evaluatedValue = propertyInfo.GetValue(baseValue); - break; - default: - return visited; - } - - // Inline the evaluated value (if the connector doesn't support parameterization, or if the field is readonly), - if (!this._supportsParameterization) - { - return Expression.Constant(evaluatedValue, visited.Type); - } - - // Otherwise, transform the node to a QueryParameterExpression which the connector will then translate to a parameter (e.g. SqlParameter). - - // TODO: Share the same parameter when it references the same captured value - - // Make sure parameter names are unique. - var origName = memberInfo.Name; - var name = origName; - for (var i = 0; this._parameterNames!.Contains(name); i++) - { - name = $"{origName}_{i}"; - } - this._parameterNames.Add(name); - - return new QueryParameterExpression(name, evaluatedValue, visited.Type); - } - - /// - protected override Expression VisitNew(NewExpression node) - { - var visited = (NewExpression)base.VisitNew(node); - - // Recognize certain well-known constructors where we can evaluate immediately, converting the NewExpression to a ConstantExpression. - // This is particularly useful for converting inline instantiation of DateTime, DateTimeOffset, DateOnly, and TimeOnly to constants, which can then be easily translated. - switch (visited.Constructor) - { - case ConstructorInfo constructor when constructor.DeclaringType == typeof(DateTimeOffset) || constructor.DeclaringType == typeof(DateTime) -#if NET - || constructor.DeclaringType == typeof(DateOnly) || constructor.DeclaringType == typeof(TimeOnly) -#endif - : - var constantArguments = new object?[visited.Arguments.Count]; - - // We first do a fast path to check if all arguments are constants; this catches the common case of e.g. new DateTime(2023, 10, 1). - // If an argument isn't a constant (e.g. new DateTimeOffset(..., TimeSpan.FromHours(2))), we fall back to trying the LINQ interpreter - // as a general-purpose expression evaluator - but note that this is considerably slower. - for (var i = 0; i < visited.Arguments.Count; i++) - { - if (visited.Arguments[i] is ConstantExpression constantArgument) - { - constantArguments[i] = constantArgument.Value; - } - else - { - // There's a non-constant argument - try the LINQ interpreter. -#pragma warning disable CA1031 // Do not catch general exception types - try - { - var evaluated = Expression.Lambda>(Expression.Convert(visited, typeof(object))) -#if NET - .Compile(preferInterpretation: true) -#else - .Compile() -#endif - .Invoke(); - - return Expression.Constant(evaluated, constructor.DeclaringType); - } - catch - { - return visited; - } -#pragma warning restore CA1031 - } - } - - var constantValue = constructor.Invoke(constantArguments); - return Expression.Constant(constantValue, constructor.DeclaringType); - } - - return visited; - } - } - - #endregion FilterTranslationPreprocessor -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/QueryParameterExpression.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/QueryParameterExpression.cs deleted file mode 100644 index 0c37722a3bcc..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/QueryParameterExpression.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Diagnostics.CodeAnalysis; -using System.Linq.Expressions; - -namespace Microsoft.Extensions.VectorData.ProviderServices.Filter; - -/// -/// An expression representation a query parameter (captured variable) in the filter expression. -/// -[Experimental("MEVD9001")] -public class QueryParameterExpression(string name, object? value, Type type) : Expression -{ - /// - /// The name of the parameter. - /// - public string Name { get; } = name; - - /// - /// The value of the parameter. - /// - public object? Value { get; } = value; - - /// - public override ExpressionType NodeType => ExpressionType.Extension; - - /// - public override Type Type => type; - - /// - protected override Expression VisitChildren(ExpressionVisitor visitor) => this; -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/IRecordCreator.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/IRecordCreator.cs deleted file mode 100644 index 079659ce5bcb..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/IRecordCreator.cs +++ /dev/null @@ -1,8 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -internal interface IRecordCreator -{ - TRecord Create(); -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/KeyPropertyModel.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/KeyPropertyModel.cs deleted file mode 100644 index 2b31e56a3d9d..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/KeyPropertyModel.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Diagnostics.CodeAnalysis; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -/// -/// Represents a key property on a vector store record. -/// This is an internal support type meant for use by connectors only and not by applications. -/// -[Experimental("MEVD9001")] -public class KeyPropertyModel(string modelName, Type type) : PropertyModel(modelName, type) -{ - /// - /// Gets or sets whether this key property's value is auto-generated or not. - /// - public bool IsAutoGenerated { get; set; } - - /// - /// Gets or sets the name that the JSON serializer will produce for this key property. - /// This is needed for connectors that use an external JSON serializer combined with a reserved key storage name - /// (e.g. CosmosDB NoSQL uses "id"): the serializer produces a JSON object with the policy-transformed name, and - /// the connector needs to find and replace it with the reserved storage name. - /// - public string? SerializedKeyName { get; set; } - - /// - public override string ToString() - => $"{this.ModelName} (Key, {this.Type.Name}{(this.IsAutoGenerated ? ", auto-generated" : "")})"; -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/PropertyModel.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/PropertyModel.cs deleted file mode 100644 index 7b3635911e95..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/PropertyModel.cs +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Reflection; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -/// -/// Represents a property on a vector store record. -/// This is an internal support type meant for use by connectors only and not by applications. -/// -[Experimental("MEVD9001")] -public abstract class PropertyModel(string modelName, Type type) -{ - private string? _storageName; - private Func? _getter; - private Action? _setter; - - /// - /// Gets or sets the model name of the property. If the property corresponds to a .NET property, this name is the name of that property. - /// - public string ModelName { get; set; } = modelName; - - /// - /// Gets or sets the storage name of the property. This is the name to which the property is mapped in the vector store. - /// - public string StorageName - { - get => this._storageName ?? this.ModelName; - set => this._storageName = value; - } - - /// - /// Gets or sets the CLR type of the property. - /// - public Type Type { get; set; } = type; - - /// - /// Gets or sets the reflection for the .NET property. - /// - /// - /// The reflection for the .NET property. - /// when using dynamic mapping. - /// - public PropertyInfo? PropertyInfo { get; set; } - - /// - /// Gets or sets a dictionary of provider-specific annotations for this property. - /// - /// - /// This allows setting database-specific configuration options that aren't universal across all vector stores. - /// - public Dictionary? ProviderAnnotations { get; set; } - - /// - /// Gets whether the property type is nullable. For value types, this is when the type is - /// . For reference types on .NET 6+, this uses NRT annotations via - /// NullabilityInfoContext when a is available - /// (i.e., POCO mapping); otherwise, reference types are assumed nullable. - /// - public bool IsNullable - { - get - { - // Value types: nullable only if Nullable - if (this.Type.IsValueType) - { - return Nullable.GetUnderlyingType(this.Type) is not null; - } - - // Reference types: check NRT annotation via NullabilityInfoContext when available -#if NET - if (this.PropertyInfo is { } propertyInfo) - { - var nullabilityInfo = new NullabilityInfoContext().Create(propertyInfo); - return nullabilityInfo.ReadState != NullabilityState.NotNull; - } -#endif - - // Dynamic mapping or old framework: assume nullable for reference types - return true; - } - } - - /// - /// Configures the property accessors using a CLR for POCO mapping. - /// - // TODO: Implement compiled delegates for better performance, #11122 - // TODO: Implement source-generated accessors for NativeAOT, #10256 - internal void ConfigurePocoAccessors(PropertyInfo propertyInfo) - { - this.PropertyInfo = propertyInfo; - this._getter = propertyInfo.GetValue; - this._setter = (record, value) => - { - // If the value is null, no need to set the property (it's the CLR default) - if (value is not null) - { - propertyInfo.SetValue(record, value); - } - }; - } - - /// - /// Configures the property accessors for dynamic mapping using . - /// - internal void ConfigureDynamicAccessors() - { - var modelName = this.ModelName; - var propertyType = this.Type; - - this._getter = record => - { - var dictionary = (Dictionary)record; - var value = dictionary.TryGetValue(modelName, out var tempValue) ? tempValue : null; - - if (value is not null && value.GetType() != (Nullable.GetUnderlyingType(propertyType) ?? propertyType)) - { - throw new InvalidCastException($"Property '{modelName}' has a value of type '{value.GetType().Name}', but its configured type is '{propertyType.Name}'."); - } - - return value; - }; - - this._setter = (record, value) => ((Dictionary)record)[modelName] = value; - } - - /// - /// Reads the property from the given , returning the value as an . - /// - public object? GetValueAsObject(object record) - { - Debug.Assert(this._getter is not null, "Property accessors have not been configured."); - return this._getter!(record); - } - - /// - /// Writes the property from the given , accepting the value to write as an . - /// - public void SetValueAsObject(object record, object? value) - { - Debug.Assert(this._setter is not null, "Property accessors have not been configured."); - this._setter!(record, value); - } - - /// - /// Reads the property from the given . - /// - // TODO: actually implement the generic accessors to avoid boxing, and make use of them in connectors - public T GetValue(object record) - => (T)(object)this.GetValueAsObject(record)!; - - /// - /// Writes the property from the given . - /// - // TODO: actually implement the generic accessors to avoid boxing, and make use of them in connectors - public void SetValue(object record, T value) - => this.SetValueAsObject(record, value); -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorDataStrings.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorDataStrings.cs deleted file mode 100644 index 22a65a732f8b..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorDataStrings.cs +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using Microsoft.Extensions.AI; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member - -/// -/// Exposes methods for constructing strings that should be used by providers when throwing exceptions. -/// -[Experimental("MEVD9001")] -public static class VectorDataStrings -{ - public static string ConfiguredEmbeddingTypeIsUnsupportedByTheGenerator(VectorPropertyModel vectorProperty, Type userRequestedEmbeddingType, string supportedVectorTypes) - => $"Vector property '{vectorProperty.ModelName}' has embedding type '{TypeName(userRequestedEmbeddingType)}' configured, but that type isn't supported by your embedding generator."; - - public static string ConfiguredEmbeddingTypeIsUnsupportedByTheProvider(VectorPropertyModel vectorProperty, Type userRequestedEmbeddingType, string supportedVectorTypes) - => $"Vector property '{vectorProperty.ModelName}' has embedding type '{TypeName(userRequestedEmbeddingType)}' configured, but that type isn't supported by your provider. Supported types are {supportedVectorTypes}."; - - public static string EmbeddingGeneratorWithInvalidEmbeddingType(VectorPropertyModel vectorProperty) - => $"An embedding generator was configured on property '{vectorProperty.ModelName}', but output embedding type '{vectorProperty.EmbeddingType.Name}' isn't supported by the connector."; - - public static string EmbeddingPropertyTypeIncompatibleWithEmbeddingGenerator(VectorPropertyModel vectorProperty) - => $"Property '{vectorProperty.ModelName}' has embedding type '{TypeName(vectorProperty.Type)}', but an embedding generator is configured on the property. Remove the embedding generator or change the property's .NET type to a non-embedding input type to the generator (e.g. string)."; - - public static string DifferentEmbeddingTypeSpecifiedForNativelySupportedType(VectorPropertyModel vectorProperty, Type embeddingType) - => $"Property '{vectorProperty.ModelName}' has {nameof(VectorStoreVectorProperty.EmbeddingType)} configured to '{TypeName(embeddingType)}', but the property already has natively supported '{TypeName(vectorProperty.Type)}'. {nameof(VectorStoreVectorProperty.EmbeddingType)} only needs to be specified for properties that require embedding generation."; - - public static string GetCollectionWithDictionaryNotSupported - => "Dynamic mapping via Dictionary is not supported via this method, call GetDynamicCollection() instead."; - - public static string IncludeVectorsNotSupportedWithEmbeddingGeneration - => "When an embedding generator is configured, `Include Vectors` cannot be enabled."; - - public static string IncompatibleEmbeddingGenerator(VectorPropertyModel vectorProperty, IEmbeddingGenerator embeddingGenerator, string supportedOutputTypes) - => $"Embedding generator '{TypeName(embeddingGenerator.GetType())}' on vector property '{vectorProperty.ModelName}' cannot convert the input type '{TypeName(vectorProperty.Type)}' to a supported vector type (one of: {supportedOutputTypes})."; - - public static string IncompatibleEmbeddingGeneratorWasConfiguredForInputType(Type inputType, Type embeddingGeneratorType) - => $"An input of type '{TypeName(inputType)}' was provided, but an incompatible embedding generator of type '{TypeName(embeddingGeneratorType)}' was configured."; - - public static string InvalidSearchInputAndNoEmbeddingGeneratorWasConfigured(Type inputType, string supportedVectorTypes) - => $"A value of type '{TypeName(inputType)}' was passed to 'SearchAsync', but that isn't a supported vector type by your provider and no embedding generator was configured. The supported vector types are: {supportedVectorTypes}."; - - public static string MissingTypeOnPropertyDefinition(VectorStoreProperty property) - => $"Property '{property.Name}' has no type specified in its definition, and does not have a corresponding .NET property. Specify the type on the definition."; - - public static string UnsupportedVectorPropertyWithoutEmbeddingGenerator(VectorPropertyModel vectorProperty) - => $"Vector property '{vectorProperty.ModelName}' has type '{TypeName(vectorProperty.Type)}' which isn't supported by your provider, and no embedding generator is configured. Configure a generator that supports converting '{TypeName(vectorProperty.Type)}' to vector type supported by your provider."; - - public static string NonDynamicCollectionWithDictionaryNotSupported(Type dynamicCollectionType) - => $"Dynamic mapping via Dictionary is not supported via this class, use '{TypeName(dynamicCollectionType)}' instead."; - - private static string TypeName(this Type type) - { - var i = type.Name.IndexOf('`'); - if (i == -1) - { - return type.Name switch - { - "Int32" => "int", - "Int64" => "long", - "Boolean" => "bool", - "Double" => "double", - "Single" => "float", - "String" => "string", - - _ => type.Name - }; - } - - var genericTypeName = type.Name.Substring(0, i); - var genericArgs = string.Join(", ", type.GetGenericArguments().Select(t => t.TypeName())); - return $"{genericTypeName}<{genericArgs}>"; - } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorPropertyModel.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorPropertyModel.cs deleted file mode 100644 index 75d3a1057ff9..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorPropertyModel.cs +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.AI; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -/// -/// Represents a vector property on a vector store record. -/// This is an internal support type meant for use by connectors only and not by applications. -/// -[Experimental("MEVD9001")] -public class VectorPropertyModel(string modelName, Type type) : PropertyModel(modelName, type) -{ - private int _dimensions; - - /// - /// Gets or sets the number of dimensions that the vector has. - /// - /// - /// This property is required when creating collections, but can be omitted if not using that functionality. - /// If not provided when trying to create a collection, create will fail. - /// - public int Dimensions - { - get => this._dimensions; - - set - { - if (value <= 0) - { - throw new ArgumentOutOfRangeException(nameof(value), "Dimensions must be greater than zero."); - } - - this._dimensions = value; - } - } - - /// - /// Gets or sets the kind of index to use. - /// - /// - /// The default varies by database type. For more information, see the documentation of your chosen database connector. - /// - /// - public string? IndexKind { get; set; } - - /// - /// Gets or sets the distance function to use when comparing vectors. - /// - /// - /// The default varies by database type. For more information, see the documentation of your chosen database connector. - /// - /// - public string? DistanceFunction { get; set; } - - /// - /// Gets or sets the type representing the embedding stored in the database if is set. - /// Otherwise, this property is identical to . - /// - /// - /// This property may be during model building while the embedding type is being resolved, - /// but is guaranteed to be non-null after building completes (validation ensures this). - /// - [AllowNull] - public Type EmbeddingType { get; set; } = null!; - - /// - /// Gets or sets the embedding generator to use for this property. - /// - public IEmbeddingGenerator? EmbeddingGenerator { get; set; } - - /// - /// Gets or sets the that was resolved for this property during model building. - /// This handler is used for runtime embedding generation dispatch. - /// - /// - /// This is for vector properties whose type is natively supported by the provider - /// (e.g., of , [], ), - /// since no embedding generation is needed. - /// - public EmbeddingGenerationDispatcher? EmbeddingGenerationDispatcher { get; set; } - - /// - /// Checks whether the given can produce embeddings of type - /// for any input type known to this property model. The base implementation checks for and ; - /// also checks for TInput. - /// - /// This is used for native vector property types, where the input type isn't known at model-build time. - public virtual bool CanGenerateEmbedding(IEmbeddingGenerator embeddingGenerator) - where TEmbedding : Embedding - => embeddingGenerator is IEmbeddingGenerator - || embeddingGenerator is IEmbeddingGenerator; - - /// - /// Checks whether the configured on this property supports the given embedding type. - /// The implementation on this non-generic checks for - /// and as input types for . - /// - public virtual Type? ResolveEmbeddingType(IEmbeddingGenerator embeddingGenerator, Type? userRequestedEmbeddingType) - where TEmbedding : Embedding - => embeddingGenerator switch - { - // On the TInput side, this out-of-the-box/simple implementation supports string and DataContent only - // (users who want arbitrary TInput types need to use the generic subclass of this type). - // The TEmbedding side is provided by the connector via the generic type parameter to this method, as the connector controls/knows which embedding types are supported. - // Note that if the user has manually specified an embedding type (e.g. to choose Embedding rather than the default Embedding), - // that's provided via the userRequestedEmbeddingType argument; we use that as a filter. - IEmbeddingGenerator when this.Type == typeof(string) && (userRequestedEmbeddingType is null || userRequestedEmbeddingType == typeof(TEmbedding)) - => typeof(TEmbedding), - IEmbeddingGenerator when this.Type == typeof(DataContent) && (userRequestedEmbeddingType is null || userRequestedEmbeddingType == typeof(TEmbedding)) - => typeof(TEmbedding), - - null => throw new ArgumentNullException(nameof(embeddingGenerator), "This method should only be called when an embedding generator is configured."), - _ => null - }; - - /// - /// Generates embeddings for the given , using the configured . - /// - /// Thrown if no is configured on this property. - public Task> GenerateEmbeddingsAsync(IEnumerable values, CancellationToken cancellationToken) - => this.EmbeddingGenerationDispatcher is not { } dispatcher - ? throw new InvalidOperationException($"No embedding generation is configured for property '{this.ModelName}'.") - : dispatcher.GenerateEmbeddingsAsync(this, values, cancellationToken); - - /// - /// Generates a single embedding for the given , using the configured . - /// - /// Thrown if no is configured on this property. - public Task GenerateEmbeddingAsync(object? value, CancellationToken cancellationToken) - => this.EmbeddingGenerationDispatcher is not { } dispatcher - ? throw new InvalidOperationException($"No embedding generation is configured for property '{this.ModelName}'.") - : dispatcher.GenerateEmbeddingAsync(this, value, cancellationToken); - - /// - /// Core method to generate a batch of embeddings. Called by with the correct type parameter. - /// - internal virtual async Task> GenerateEmbeddingsCoreAsync(IEnumerable values, CancellationToken cancellationToken) - where TEmbedding : Embedding - => this.EmbeddingGenerator switch - { - IEmbeddingGenerator generator when this.EmbeddingType == typeof(TEmbedding) - => await generator.GenerateAsync( - values.Select(v => v is string s - ? s - : throw new InvalidOperationException($"Property '{this.ModelName}' was configured with an embedding generator accepting a string, but {v?.GetType().Name ?? "null"} was provided.")), - options: null, - cancellationToken).ConfigureAwait(false), - - IEmbeddingGenerator generator when this.EmbeddingType == typeof(TEmbedding) - => await generator.GenerateAsync( - values.Select(v => v is DataContent c - ? c - : throw new InvalidOperationException($"Property '{this.ModelName}' was configured with an embedding generator accepting a {nameof(DataContent)}, but {v?.GetType().Name ?? "null"} was provided.")), - options: null, - cancellationToken).ConfigureAwait(false), - - null => throw new UnreachableException("This method should only be called when an embedding generator is configured."), - - _ => throw new InvalidOperationException( - $"The embedding generator configured on property '{this.ModelName}' cannot produce an embedding of type '{typeof(TEmbedding).Name}' for the given input type."), - }; - - /// - /// Core method to generate a single embedding. Called by with the correct type parameter. - /// - internal virtual async Task GenerateEmbeddingCoreAsync(object? value, CancellationToken cancellationToken) - where TEmbedding : Embedding - => this.EmbeddingGenerator switch - { - IEmbeddingGenerator generator when value is string s - => await generator.GenerateAsync(s, options: null, cancellationToken).ConfigureAwait(false), - - IEmbeddingGenerator generator when value is DataContent c - => await generator.GenerateAsync(c, options: null, cancellationToken).ConfigureAwait(false), - - null => throw new UnreachableException("This method should only be called when an embedding generator is configured."), - - _ => throw new InvalidOperationException( - VectorDataStrings.IncompatibleEmbeddingGeneratorWasConfiguredForInputType(value?.GetType() ?? typeof(object), this.EmbeddingGenerator!.GetType())), - }; - - /// - /// Returns the types of input that this property model supports. - /// - public virtual Type[] GetSupportedInputTypes() => [typeof(string), typeof(DataContent)]; - - /// - public override string ToString() - => $"{this.ModelName} (Vector, {this.Type.Name})"; -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorPropertyModel{TInput}.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorPropertyModel{TInput}.cs deleted file mode 100644 index e7f51d2c58e1..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/VectorPropertyModel{TInput}.cs +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.AI; - -namespace Microsoft.Extensions.VectorData.ProviderServices; - -/// -[Experimental("MEVD9001")] -public sealed class VectorPropertyModel(string modelName) : VectorPropertyModel(modelName, typeof(TInput)) -{ - /// - public override bool CanGenerateEmbedding(IEmbeddingGenerator embeddingGenerator) - => embeddingGenerator is IEmbeddingGenerator - || base.CanGenerateEmbedding(embeddingGenerator); - - /// - public override Type? ResolveEmbeddingType(IEmbeddingGenerator embeddingGenerator, Type? userRequestedEmbeddingType) - => embeddingGenerator switch - { - IEmbeddingGenerator when this.Type == typeof(TInput) && (userRequestedEmbeddingType is null || userRequestedEmbeddingType == typeof(TEmbedding)) - => typeof(TEmbedding), - - null => throw new ArgumentNullException(nameof(embeddingGenerator), "This method should only be called when an embedding generator is configured."), - _ => null - }; - - /// - internal override async Task> GenerateEmbeddingsCoreAsync(IEnumerable values, CancellationToken cancellationToken) - { - switch (this.EmbeddingGenerator) - { - case IEmbeddingGenerator generator when this.EmbeddingType == typeof(TEmbedding): - return await generator.GenerateAsync( - values.Select(v => v is TInput s - ? s - : throw new InvalidOperationException($"Property '{this.ModelName}' was configured with an embedding generator accepting a {typeof(TInput).Name}, but {v?.GetType().Name ?? "null"} was provided.")), - options: null, - cancellationToken).ConfigureAwait(false); - - case null: - throw new UnreachableException("This method should only be called when an embedding generator is configured."); - - default: - throw new InvalidOperationException( - $"The embedding generator configured on property '{this.ModelName}' cannot produce an embedding of type '{typeof(TEmbedding).Name}' for the given input type."); - } - } - - /// - internal override async Task GenerateEmbeddingCoreAsync(object? value, CancellationToken cancellationToken) - { - if (this.EmbeddingGenerator is IEmbeddingGenerator generator && value is TInput t) - { - return await generator.GenerateAsync(t, options: null, cancellationToken).ConfigureAwait(false); - } - - // Fall through to base class which checks for string and DataContent input types. - return await base.GenerateEmbeddingCoreAsync(value, cancellationToken).ConfigureAwait(false); - } - - /// - public override Type[] GetSupportedInputTypes() => [typeof(TInput)]; -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/README.md b/dotnet/src/VectorData/VectorData.Abstractions/README.md new file mode 100644 index 000000000000..0e5c7c8e55ac --- /dev/null +++ b/dotnet/src/VectorData/VectorData.Abstractions/README.md @@ -0,0 +1 @@ +The code for Microsoft.Extensions.VectorData.Abstractions can now be found in the [dotnet/extensions repository](https://github.com/dotnet/extensions/tree/main/src/Libraries/Microsoft.Extensions.VectorData.Abstractions) diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreDataAttribute.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreDataAttribute.cs deleted file mode 100644 index d326bb068760..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreDataAttribute.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines an attribute to mark a property on a record class as 'data'. -/// -/// -/// Marking a property as 'data' means that the property is not a key and not a vector. But optionally, -/// this property can have an associated vector field containing an embedding for this data. -/// The characteristics defined here influence how the property is treated by the vector store. -/// -[AttributeUsage(AttributeTargets.Property, AllowMultiple = false)] -public sealed class VectorStoreDataAttribute : Attribute -{ - /// - /// Gets or sets a value indicating whether this data property is indexed. - /// - /// - /// The default is . - /// - public bool IsIndexed { get; init; } - - /// - /// Gets or sets a value indicating whether this data property is indexed for full-text search. - /// - /// - /// The default is . - /// - public bool IsFullTextIndexed { get; init; } - - /// - /// Gets or sets an optional name to use for the property in storage, if different from the property name. - /// - /// - /// For example, the property name might be "MyProperty" and the storage name might be "my_property". - /// - public string? StorageName { get; init; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreKeyAttribute.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreKeyAttribute.cs deleted file mode 100644 index 5e4001a447d3..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreKeyAttribute.cs +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines an attribute to mark a property on a record class as the key under which the record is stored in a vector store. -/// -/// -/// The characteristics defined here influence how the property is treated by the vector store. -/// -[AttributeUsage(AttributeTargets.Property, AllowMultiple = false)] -public sealed class VectorStoreKeyAttribute : Attribute -{ - /// - /// Gets or sets an optional name to use for the property in storage, if different from the property name. - /// - /// - /// For example, the property name might be "MyProperty" and the storage name might be "my_property". - /// - public string? StorageName { get; init; } - - /// - /// Gets or sets whether this key property's value is auto-generated or not. - /// - /// - /// The availability of auto-generated properties - as well as the .NET types supported for them - varies across provider implementations. - /// The getter returns when the value has not been explicitly set; use to distinguish - /// between "explicitly set to false" and "not set". - /// The getter does not throw, even though it cannot distinguish "explicitly set to false" and "not set"; this is a workaround for a C# compiler - /// limitation that does not allow to be used as an attribute argument. - /// - public bool IsAutoGenerated - { - // The getter returns GetValueOrDefault() rather than throwing, as a workaround for a C# compiler limitation: - // Nullable cannot be used as a compile-time attribute argument, so the public property must be bool. - get => this.IsAutoGeneratedNullable.GetValueOrDefault(); - set => this.IsAutoGeneratedNullable = value; - } - - /// - /// Gets whether this key property's value is auto-generated or not, or if not set. - /// - internal bool? IsAutoGeneratedNullable { get; private set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreVectorAttribute.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreVectorAttribute.cs deleted file mode 100644 index 4f898116c69f..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordAttributes/VectorStoreVectorAttribute.cs +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines an attribute to mark a property on a record class as a vector. -/// -/// -/// The characteristics defined here influence how the property is treated by the vector store. -/// -[AttributeUsage(AttributeTargets.Property, AllowMultiple = false)] -public sealed class VectorStoreVectorAttribute : Attribute -{ - /// - /// Initializes a new instance of the class. - /// - /// The number of dimensions that the vector has. - public VectorStoreVectorAttribute(int Dimensions) - { - if (Dimensions <= 0) - { - throw new ArgumentOutOfRangeException(nameof(Dimensions), "Dimensions must be greater than zero."); - } - - this.Dimensions = Dimensions; - } - - /// - /// Gets the number of dimensions that the vector has. - /// - /// - /// This property is required when creating collections, but can be omitted if not using that functionality. - /// If not provided when trying to create a collection, create will fail. - /// - public int Dimensions { get; private set; } - - /// - /// Gets or sets the kind of index to use. - /// - /// - /// The default value varies by database type. See the documentation of your chosen database connector for more information. - /// - /// -#pragma warning disable CA1019 // Define accessors for attribute arguments: The constructor overload that contains this property is obsolete. - public string? IndexKind { get; init; } -#pragma warning restore CA1019 - - /// - /// Gets or sets the distance function to use when comparing vectors. - /// - /// - /// The default value varies by database type. See the documentation of your chosen database connector for more information. - /// - /// -#pragma warning disable CA1019 // Define accessors for attribute arguments: The constructor overload that contains this property is obsolete. - public string? DistanceFunction { get; init; } -#pragma warning restore CA1019 - - /// - /// Gets or sets an optional name to use for the property in storage, if different from the property name. - /// - /// - /// For example, the property name might be "MyProperty" and the storage name might be "my_property". - /// - public string? StorageName { get; init; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/DistanceFunction.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/DistanceFunction.cs deleted file mode 100644 index 068caed55807..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/DistanceFunction.cs +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines a list of well-known distance functions that can be used to compare vectors. -/// -/// -/// Not all Vector Store connectors support all distance functions, and some connectors might -/// support additional distance functions that aren't defined here. -/// For more information on what's supported, see the documentation for each connector. -/// -public static class DistanceFunction -{ - /// - /// Specifies the function that measures the cosine (angular) similarity between two vectors. - /// - /// - /// Cosine similarity measures only the angle between the two vectors, without taking into account the length of the vectors. - /// ConsineSimilarity = 1 - CosineDistance. - /// -1 means vectors are opposite. - /// 0 means vectors are orthogonal. - /// 1 means vectors are identical. - /// - public const string CosineSimilarity = nameof(CosineSimilarity); - - /// - /// Specifies the function that measures the cosine (angular) distance between two vectors. - /// - /// - /// CosineDistance = 1 - CosineSimilarity. - /// 2 means vectors are opposite. - /// 1 means vectors are orthogonal. - /// 0 means vectors are identical. - /// - public const string CosineDistance = nameof(CosineDistance); - - /// - /// Specifies the dot product similarity function, which measures both the length and angle between two vectors. - /// - /// - /// The higher the value, the more similar the vectors. - /// - public const string DotProductSimilarity = nameof(DotProductSimilarity); - - /// - /// Specifies the negative dot product similarity function, which measures both the length and angle between two vectors. - /// - /// - /// The value of NegativeDotProduct = -1 * DotProductSimilarity. - /// The higher the value, the greater the distance between the vectors and the less similar the vectors. - /// - public const string NegativeDotProductSimilarity = nameof(NegativeDotProductSimilarity); - - /// - /// Specifies the function that measures the Euclidean distance between two vectors. - /// - /// - /// Also known as l2-norm. - /// - public const string EuclideanDistance = nameof(EuclideanDistance); - - /// - /// Specifies the function that measures the Euclidean squared distance between two vectors. - /// - /// - /// Also known as l2-squared. - /// - public const string EuclideanSquaredDistance = nameof(EuclideanSquaredDistance); - - /// - /// Specifies the function that measures the number of differences between vectors at each dimension. - /// - public const string HammingDistance = nameof(HammingDistance); - - /// - /// Specifies the function that measures the Manhattan distance between two vectors. - /// - public const string ManhattanDistance = nameof(ManhattanDistance); -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/IndexKind.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/IndexKind.cs deleted file mode 100644 index 372f451e4c05..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/IndexKind.cs +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines a list of well-known index types that can be used to index vectors. -/// -/// -/// Not all Vector Store connectors support all index types, and some connectors might -/// support additional index types that aren't defined here. For more information on what's -/// supported, see the documentation for each connector. -/// -public static class IndexKind -{ - /// - /// Specifies the Hierarchical Navigable Small World, which performs an approximate nearest neighbor (ANN) search. - /// - /// - /// This search has lower accuracy than exhaustive k nearest neighbor, but is faster and more efficient. - /// - public const string Hnsw = nameof(Hnsw); - - /// - /// Specifies the brute force search to find the nearest neighbors. - /// - /// - /// This search calculates the distances between all pairs of data points, so it has a linear time complexity that grows directly proportional to the number of points. - /// It's also referred to as "exhaustive k nearest neighbor" in some databases. - /// This search has high recall accuracy, but is slower and more expensive than HNSW. - /// It works better with smaller datasets. - /// - public const string Flat = nameof(Flat); - - /// - /// Specifies an Inverted File with Flat Compression. - /// - /// - /// This search is designed to enhance search efficiency by narrowing the search area through the use of neighbor partitions or clusters. - /// Also referred to as approximate nearest neighbor (ANN) search. - /// - public const string IvfFlat = nameof(IvfFlat); - - /// - /// Specifies the Disk-based Approximate Nearest Neighbor algorithm, which is designed for efficiently searching for approximate nearest neighbors (ANN) in high-dimensional spaces. - /// - /// - /// The primary focus of DiskANN is to handle large-scale datasets that can't fit entirely into memory, leveraging disk storage to store the data while maintaining fast search times. - /// - public const string DiskAnn = nameof(DiskAnn); - - /// - /// Specifies an index that compresses vectors using DiskANN-based quantization methods for better efficiency in the kNN search. - /// - public const string QuantizedFlat = nameof(QuantizedFlat); - - /// - /// Specifies a dynamic index that switches automatically from to indexes. - /// - public const string Dynamic = nameof(Dynamic); -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreCollectionDefinition.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreCollectionDefinition.cs deleted file mode 100644 index b7ba514b0adc..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreCollectionDefinition.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using Microsoft.Extensions.AI; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Describes the properties of a record in a vector store collection. -/// -/// -/// Each property contains additional information about how the property will be treated by the vector store. -/// -public sealed class VectorStoreCollectionDefinition -{ - private IList? _properties; - - /// - /// Gets or sets the list of properties that are stored in the record. - /// - [AllowNull] - public IList Properties - { - get => this._properties ??= []; - set => this._properties = value; - } - - /// - /// Gets or sets the default embedding generator for vector properties in this collection. - /// - public IEmbeddingGenerator? EmbeddingGenerator { get; set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreDataProperty.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreDataProperty.cs deleted file mode 100644 index 0f1a7d764bde..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreDataProperty.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines a data property on a vector store record. -/// -/// -/// The characteristics defined here influence how the property is treated by the vector store. -/// -public sealed class VectorStoreDataProperty : VectorStoreProperty -{ - /// - /// Initializes a new instance of the class. - /// - /// The name of the property on the data model. If the record is mapped to a .NET type, this corresponds to the .NET property name on that type. - /// The type of the property. Required when using a record type of Dictionary<string, object?> (dynamic mapping), but can be omitted when mapping any other .NET type. - public VectorStoreDataProperty(string name, Type? type = null) - : base(name, type) - { - } - - /// - /// Gets or sets a value indicating whether this data property is indexed. - /// - /// - /// The default is . - /// - public bool IsIndexed { get; set; } - - /// - /// Gets or sets a value indicating whether this data property is indexed for full-text search. - /// - /// - /// The default is . - /// - public bool IsFullTextIndexed { get; set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreKeyProperty.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreKeyProperty.cs deleted file mode 100644 index 2c2a6f6262f0..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreKeyProperty.cs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines a key property on a vector store record. -/// -/// -/// The characteristics defined here influence how the property is treated by the vector store. -/// -public sealed class VectorStoreKeyProperty : VectorStoreProperty -{ - /// - /// Initializes a new instance of the class. - /// - /// The name of the property on the data model. If the record is mapped to a .NET type, this corresponds to the .NET property name on that type. - /// The type of the property. Required when using a record type of Dictionary<string, object?> (dynamic mapping), but can be omitted when mapping any other .NET type. - public VectorStoreKeyProperty(string name, Type? type = null) - : base(name, type) - { - } - - /// - /// Gets or sets whether this key property's value is auto-generated or not. - /// - /// - /// The availability of auto-generated properties - as well as the .NET types supported for them - varies across provider implementations. - /// - public bool? IsAutoGenerated { get; set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreProperty.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreProperty.cs deleted file mode 100644 index d3f2b8faced9..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreProperty.cs +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines a base property class for properties on a vector store record. -/// -/// -/// The characteristics defined here influence how the property is treated by the vector store. -/// -public abstract class VectorStoreProperty -{ - /// - /// Initializes a new instance of the class. - /// - /// The name of the property on the data model. If the record is mapped to a .NET type, this corresponds to the .NET property name on that type. - /// The type of the property. - private protected VectorStoreProperty(string name, Type? type) - { - if (string.IsNullOrWhiteSpace(name)) - { - throw new ArgumentException("Value cannot be null or whitespace.", nameof(name)); - } - - this.Name = name; - this.Type = type; - } - - private protected VectorStoreProperty(VectorStoreProperty source) - { - this.Name = source.Name; - this.StorageName = source.StorageName; - this.Type = source.Type; - this.ProviderAnnotations = source.ProviderAnnotations is not null - ? new Dictionary(source.ProviderAnnotations) - : null; - } - - /// - /// Gets or sets the name of the property on the data model. - /// - public string Name { get; set; } - - /// - /// Gets or sets an optional name to use for the property in storage, if different from the property name. - /// - /// - /// For example, the property name might be "MyProperty" and the storage name might be "my_property". - /// This property is only respected by implementations that don't support a well-known - /// serialization mechanism like JSON, in which case the attributes used by that serialization system will - /// be used. - /// - public string? StorageName { get; set; } - - /// - /// Gets or sets the type of the property. - /// - public Type? Type { get; set; } - - /// - /// Gets or sets a dictionary of provider-specific annotations for this property. - /// - /// - /// This allows setting database-specific configuration options that aren't universal across all vector stores. - /// Use provider-specific extension methods to set and get values in a strongly-typed manner. - /// - public Dictionary? ProviderAnnotations { get; set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreVectorProperty.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreVectorProperty.cs deleted file mode 100644 index 54762c7ebe81..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreVectorProperty.cs +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.Extensions.AI; -using Microsoft.Extensions.VectorData.ProviderServices; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines a vector property on a vector store record. -/// -/// -/// The characteristics defined here influence how the property is treated by the vector store. -/// -public class VectorStoreVectorProperty : VectorStoreProperty -{ - private int _dimensions; - - /// - /// Initializes a new instance of the class. - /// - /// The name of the property on the data model. If the record is mapped to a .NET type, this corresponds to the .NET property name on that type. - /// The number of dimensions that the vector has. - public VectorStoreVectorProperty(string name, int dimensions) - : base(name, type: null) - { - this.Dimensions = dimensions; - } - - /// - /// Initializes a new instance of the class. - /// - /// The name of the property on the data model. If the record is mapped to a .NET type, this corresponds to the .NET property name on that type. - /// The type of the property. - /// The number of dimensions that the vector has. - public VectorStoreVectorProperty(string name, Type type, int dimensions) - : base(name, type) - { - this.Dimensions = dimensions; - } - - /// - /// Gets or sets the default embedding generator to use for this property. - /// - /// - /// If not set, embedding generation will be performed in the database, if supported by your connector. - /// Otherwise, if your database does not support embedding generation, only pregenerated embeddings can be used (for example, ReadOnlyMemory<float>). - /// - public IEmbeddingGenerator? EmbeddingGenerator { get; set; } - - /// - /// Gets or sets the number of dimensions that the vector has. - /// - /// - /// This property is required when creating collections, but can be omitted if not using that functionality. - /// If not provided when trying to create a collection, create will fail. - /// - public int Dimensions - { - get => this._dimensions; - - set - { - if (value <= 0) - { - throw new ArgumentOutOfRangeException(nameof(value), "Dimensions must be greater than zero."); - } - - this._dimensions = value; - } - } - - /// - /// Gets or sets the kind of index to use. - /// - /// - /// The default varies by database type. See the documentation of your chosen database connector for more information. - /// - /// - public string? IndexKind { get; set; } - - /// - /// Gets or sets the distance function to use when comparing vectors. - /// - /// - /// The default varies by database type. See the documentation of your chosen database connector for more information. - /// - /// - public string? DistanceFunction { get; set; } - - /// - /// Gets or sets the desired embedding type (for example, Embedding<Half>) for cases where the default (typically Embedding<float>) isn't suitable. - /// - public Type? EmbeddingType { get; set; } - - internal virtual VectorPropertyModel CreatePropertyModel() - => new(this.Name, this.Type ?? throw new InvalidOperationException(VectorDataStrings.MissingTypeOnPropertyDefinition(this))) - { - Dimensions = this.Dimensions, - IndexKind = this.IndexKind, - DistanceFunction = this.DistanceFunction, - EmbeddingGenerator = this.EmbeddingGenerator, - EmbeddingType = this.EmbeddingType! - }; -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreVectorProperty{TInput}.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreVectorProperty{TInput}.cs deleted file mode 100644 index 23a63b27604c..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordDefinition/VectorStoreVectorProperty{TInput}.cs +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.AI; -using Microsoft.Extensions.VectorData.ProviderServices; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines a vector property on a vector store record. -/// -/// -/// -/// The characteristics defined here influence how the property is treated by the vector store. -/// -/// -/// This generic version of only needs to be used when an is -/// configured on the property, and a custom .NET type is used as input (any type other than or ). -/// -/// -public class VectorStoreVectorProperty : VectorStoreVectorProperty -{ - /// - public VectorStoreVectorProperty(string propertyName, int dimensions) - : base(propertyName, typeof(TInput), dimensions) - { - } - - internal override VectorPropertyModel CreatePropertyModel() - => new VectorPropertyModel(this.Name) - { - Dimensions = this.Dimensions, - IndexKind = this.IndexKind, - DistanceFunction = this.DistanceFunction, - EmbeddingGenerator = this.EmbeddingGenerator - }; -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordOptions/FilteredRecordRetrievalOptions.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordOptions/FilteredRecordRetrievalOptions.cs deleted file mode 100644 index 0ef217b8b4de..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordOptions/FilteredRecordRetrievalOptions.cs +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq.Expressions; -using System.Threading; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines options for calling . -/// -/// The type of the record. -public sealed class FilteredRecordRetrievalOptions -{ - private int _skip = 0; - - /// - /// Gets or sets the number of results to skip before returning results, that is, the index of the first result to return. - /// - /// The value is less than 0. - public int Skip - { - get => this._skip; - set - { - if (value < 0) - { - throw new ArgumentOutOfRangeException(nameof(value), "Skip must be greater than or equal to 0."); - } - - this._skip = value; - } - } - - /// - /// Gets or sets the data property to order by. - /// - /// - /// If not provided, the order of returned results is non-deterministic. - /// - public Func? OrderBy { get; set; } - - /// - /// Gets or sets a value indicating whether to include vectors in the retrieval result. - /// - public bool IncludeVectors { get; set; } - - /// - /// Represents a builder for sorting. - /// - // This type does not derive any collection in order to avoid Intellisense suggesting LINQ methods. - public sealed class OrderByDefinition - { - private readonly List _values = []; - - /// - /// Gets the expressions to sort by. - /// - /// This property is intended to be consumed by the connectors to retrieve the configuration. - public IReadOnlyList Values => this._values; - - /// - /// Creates an ascending sort. - /// - public OrderByDefinition Ascending(Expression> propertySelector) - { - if (propertySelector is null) - { - throw new ArgumentNullException(nameof(propertySelector)); - } - - this._values.Add(new(propertySelector, true)); - return this; - } - - /// - /// Creates a descending sort. - /// - public OrderByDefinition Descending(Expression> propertySelector) - { - if (propertySelector is null) - { - throw new ArgumentNullException(nameof(propertySelector)); - } - - this._values.Add(new(propertySelector, false)); - return this; - } - - /// - /// Provides a way to define property ordering. - /// - /// This class is intended to be consumed by the connectors to retrieve the configuration. - public sealed class SortInfo - { - internal SortInfo(Expression> propertySelector, bool isAscending) - { - this.PropertySelector = propertySelector; - this.Ascending = isAscending; - } - - /// - /// Gets the expression to select the property to sort by. - /// - public Expression> PropertySelector { get; } - - /// - /// Gets a value that indicates whether the sort is ascending; otherwise, false. - /// - /// - /// if the sort is ascending; otherwise, . - /// - public bool Ascending { get; } - } - } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/RecordOptions/RecordRetrievalOptions.cs b/dotnet/src/VectorData/VectorData.Abstractions/RecordOptions/RecordRetrievalOptions.cs deleted file mode 100644 index ac17b697136e..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/RecordOptions/RecordRetrievalOptions.cs +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Threading; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines options for calling -/// or . -/// -public class RecordRetrievalOptions -{ - /// - /// Gets or sets a value indicating whether to include vectors in the retrieval result. - /// - public bool IncludeVectors { get; set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/Throw.cs b/dotnet/src/VectorData/VectorData.Abstractions/Throw.cs deleted file mode 100644 index 42682c708155..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/Throw.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -internal static class Throw -{ - /// Throws an exception indicating that a required service is not available. - public static InvalidOperationException CreateMissingServiceException(Type serviceType, object? serviceKey) => - new(serviceKey is null ? - $"No service of type '{serviceType}' is available." : - $"No service of type '{serviceType}' for the key '{serviceKey}' is available."); -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorData.Abstractions.csproj b/dotnet/src/VectorData/VectorData.Abstractions/VectorData.Abstractions.csproj deleted file mode 100644 index ea63dd6f4065..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorData.Abstractions.csproj +++ /dev/null @@ -1,71 +0,0 @@ - - - - Microsoft.Extensions.VectorData.Abstractions - Microsoft.Extensions.VectorData - net10.0;net8.0;netstandard2.0;net462 - true - - false - - - - - - 10.1.0 - 10.0.0.0 - - 10.0.1 - Microsoft.Extensions.VectorData.Abstractions - $(AssemblyName) - Abstractions for vector database access. - -Commonly Used Types: -Microsoft.Extensions.VectorData.IVectorStore -Microsoft.Extensions.VectorData.IVectorStoreRecordCollection<TKey, TRecord> - neticon.png - neticon.png - PACKAGE.md - - Vector, Database, SDK - $(PackageDescription) - https://dot.net/ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/HybridSearchOptions.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/HybridSearchOptions.cs deleted file mode 100644 index e3ca2cc0b430..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/HybridSearchOptions.cs +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Linq.Expressions; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines options for hybrid search when using a dense vector and string keywords to do the search. -/// -public class HybridSearchOptions -{ - private int _skip = 0; - - /// - /// Gets or sets a search filter to use before doing the hybrid search. - /// - public Expression>? Filter { get; set; } - - /// - /// Gets or sets the target dense vector property to search on. - /// Only needs to be set when the collection has multiple vector properties. - /// - /// - /// If this property isn't set, checks if there is a vector property to use by default, and - /// throws if either none or multiple exist. - /// - public Expression>? VectorProperty { get; set; } - - /// - /// Gets or sets the additional target property to do the text or keyword search on. - /// The property must have full text indexing enabled. - /// - /// - /// If this property isn't set, checks if there is a text property with full text indexing enabled, and - /// throws an exception if either none or multiple exist. - /// - public Expression>? AdditionalProperty { get; set; } - - /// - /// Gets or sets the number of results to skip before returning results, that is, the index of the first result to return. - /// - /// The value is less than 0. - public int Skip - { - get => this._skip; - set - { - if (value < 0) - { - throw new ArgumentOutOfRangeException(nameof(value), "Skip must be greater than or equal to 0."); - } - - this._skip = value; - } - } - - /// - /// Gets or sets a value indicating whether to include vectors in the retrieval result. - /// - public bool IncludeVectors { get; set; } - - /// - /// Gets or sets the score threshold to filter results. - /// - /// - /// - /// The meaning of the score is a combination of the distance function configured for and the text - /// relevance score for the full-text search on . - /// - /// - /// The range of scores also depends on the distance function; for example, cosine similarity/distance scores - /// fall within 0 to 1, while Euclidean distance is unbounded. Scores can also differ between vector databases. - /// - /// - public double? ScoreThreshold { get; set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/IKeywordHybridSearchable.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/IKeywordHybridSearchable.cs deleted file mode 100644 index d0bb9ee95c00..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/IKeywordHybridSearchable.cs +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections; -using System.Collections.Generic; -using System.Threading; -using Microsoft.Extensions.AI; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Contains a method for performing a hybrid search using a vector and keywords. -/// -/// The record data model to use for retrieving data from the store. -public interface IKeywordHybridSearchable -{ - /// - /// Performs a hybrid search for records that match the given embedding and keywords, after applying the provided filters. - /// - /// The type of the input value on which to perform the vector similarity search. - /// The value on which to perform the similarity search. See the remarks section for more details. - /// A collection of keywords to search the store with. - /// The maximum number of results to return. - /// The options that control the behavior of the search. - /// The to monitor for cancellation requests. The default is . - /// The records found by the hybrid search, including their result scores. - /// - /// The types supported for the vary based on the provider being used and the embedding generation configured: - /// - /// - /// - /// A or (for images, sound...) if an appropriate has been configured that accepts that type as input. - /// For example, register an that accepts as input in your dependency injection container, and then pass in a - /// argument to this method; the argument will be automatically passed to the to generate the embedding and perform the search. - /// Some databases support generating embeddings at the database side. In this case, you can pass in a or without configuring an - /// with Microsoft.Extensions.VectorData. The provider will simply send your argument to the database as-is for embedding generation. - /// - /// - /// Arbitrary .NET types can also be passed in as long as an appropriate has been configured; for example, you can create your own - /// that accepts your own custom types as input, and uses another to generate embedding from multiple properties. For .NET types beyond and - /// , you must use the generic in your record definition. - /// - /// - /// To work with embeddings directly, pass in a or a .NET array of the appropriate type. Most providers support at least ReadOnlyMemory<float> and float[], - /// but some support other types (for example, ReadOnlyMemory<Half>, ). Some providers might also support their own custom types as well, for example, to represent sparse embeddings. - /// Consult your provider's documentation for supported types. - /// - /// - /// If you're using directly in your code, that type returns an (for example, Embedding{float}), - /// which can also be passed in directly, as long as the provider supports the specific embedding type. However, consider registering your with the provider - /// instead and pass in the input type (for example, ). - /// - /// - /// - IAsyncEnumerable> HybridSearchAsync( - TInput searchValue, - ICollection keywords, - int top, - HybridSearchOptions? options = default, - CancellationToken cancellationToken = default) - where TInput : notnull; - - /// Asks the for an object of the specified type . - /// The type of object being requested. - /// An optional key that can be used to help identify the target service. - /// The found object, otherwise . - /// is . - /// - /// The purpose of this method is to allow for the retrieval of strongly typed services that might be provided by the , - /// including itself or any services it might be wrapping. For example, to access the for the instance, - /// can be used to request it. - /// - object? GetService(Type serviceType, object? serviceKey = null); -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/IVectorSearchable.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/IVectorSearchable.cs deleted file mode 100644 index 8d6dc76e8daf..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/IVectorSearchable.cs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections; -using System.Collections.Generic; -using System.Threading; -using Microsoft.Extensions.AI; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines an interface for performing vector searches on a vector store. -/// -/// The record data model to use for retrieving data from the store. -public interface IVectorSearchable -{ - /// - /// Searches the vector store for records that are similar to the given value. - /// - /// The type of the input value on which to perform the similarity search. - /// The value on which to perform the similarity search. See the remarks section for more details. - /// The maximum number of results to return. - /// The options that control the behavior of the search. - /// The to monitor for cancellation requests. The default is . - /// The records found by the vector search, including their result scores. - /// - /// The types supported for the vary based on the provider being used and the embedding generation configured: - /// - /// - /// - /// A or (for images, sound...) if an appropriate has been configured that accepts that type as input. - /// For example, register an that accepts as input in your dependency injection container, and then pass in a - /// argument to this method; the argument will be automatically passed to the to generate the embedding and perform the search. - /// Some databases support generating embeddings at the database side. In this case, you can pass in a or without configuring an - /// with Microsoft.Extensions.VectorData. The provider will simply send your argument to the database as-is for embedding generation. - /// - /// - /// Arbitrary .NET types can also be passed in as long as an appropriate has been configured; for example, you can create your own - /// that accepts your own custom types as input, and uses another to generate embedding from multiple properties. For .NET types beyond and - /// , you must use the generic in your record definition. - /// - /// - /// To work with embeddings directly, pass in a or a .NET array of the appropriate type. Most providers support at least ReadOnlyMemory<float> and float[], - /// but some support other types (for example, ReadOnlyMemory<Half>, ). Some providers might also support their own custom types as well, for example, to represent sparse embeddings. - /// Consult your provider's documentation for supported types. - /// - /// - /// If you're using directly in your code, that type returns an (for example, Embedding{float}), - /// which can also be passed in directly, as long as the provider supports the specific embedding type. However, consider registering your with the provider - /// instead and pass in the input type (for example, ). - /// - /// - /// - IAsyncEnumerable> SearchAsync( - TInput searchValue, - int top, - VectorSearchOptions? options = default, - CancellationToken cancellationToken = default) - where TInput : notnull; - - /// Asks the for an object of the specified type . - /// The type of object being requested. - /// An optional key that can be used to help identify the target service. - /// The found object, otherwise . - /// is . - /// - /// The purpose of this method is to allow for the retrieval of strongly typed services that might be provided by the , - /// including itself or any services it might be wrapping. For example, to access the for the instance, - /// can be used to request it. - /// - object? GetService(Type serviceType, object? serviceKey = null); -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/KeywordHybridSearchExtensions.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/KeywordHybridSearchExtensions.cs deleted file mode 100644 index dea184745fb9..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/KeywordHybridSearchExtensions.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// Provides a collection of static methods for extending instances. -public static class KeywordHybridSearchExtensions -{ - /// - /// Asks the for an object of the specified type - /// and throw an exception if one isn't available. - /// - /// The record data model to use for retrieving data from the store. - /// The keyword hybrid search. - /// The type of object being requested. - /// An optional key that can be used to help identify the target service. - /// The found object. - /// is . - /// is . - /// No service of the requested type for the specified key is available. - public static object GetRequiredService(this IKeywordHybridSearchable keywordHybridSearch, Type serviceType, object? serviceKey = null) - { - if (keywordHybridSearch is null) { throw new ArgumentNullException(nameof(keywordHybridSearch)); } - if (serviceType is null) { throw new ArgumentNullException(nameof(serviceType)); } - - return - keywordHybridSearch.GetService(serviceType, serviceKey) ?? - throw Throw.CreateMissingServiceException(serviceType, serviceKey); - } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/RecordSearchOptions.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/RecordSearchOptions.cs deleted file mode 100644 index 207034bd95c9..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/RecordSearchOptions.cs +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Linq.Expressions; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines options for vector search via . -/// -public class VectorSearchOptions -{ - private int _skip = 0; - - /// - /// Gets or sets a search filter to use before doing the vector search. - /// - public Expression>? Filter { get; set; } - - /// - /// Gets or sets the vector property to search on. - /// Only needs to be set when the collection has multiple vector properties. - /// - /// - /// If this property isn't set provided, checks if there is a vector property to use by default, and - /// throws an exception if either none or multiple exist. - /// - public Expression>? VectorProperty { get; set; } - - /// - /// Gets or sets the number of results to skip before returning results, that is, the index of the first result to return. - /// - /// The value is less than 0. - public int Skip - { - get => this._skip; - set - { - if (value < 0) - { - throw new ArgumentOutOfRangeException(nameof(value), "Skip must be greater than or equal to 0."); - } - - this._skip = value; - } - } - - /// - /// Gets or sets a value indicating whether to include vectors in the retrieval result. - /// - public bool IncludeVectors { get; set; } - - /// - /// Gets or sets the score threshold to filter results. - /// - /// - /// - /// The meaning of the score depends on the distance function configured for the vector property. - /// For similarity functions (e.g. , ), - /// higher scores indicate more similar results, and results with scores lower than the threshold will be filtered out. - /// For distance functions (e.g. , ), - /// lower scores indicate more similar results, and results with scores higher than the threshold will be filtered out. - /// - /// - /// The range of scores also depends on the distance function; for example, cosine similarity/distance scores - /// fall within 0 to 1, while Euclidean distance is unbounded. Scores can also differ between vector databases. - /// - /// - public double? ScoreThreshold { get; set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/VectorSearchExtensions.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/VectorSearchExtensions.cs deleted file mode 100644 index 65e931c99d4e..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/VectorSearchExtensions.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// Provides a collection of static methods for extending instances. -public static class VectorSearchExtensions -{ - /// - /// Asks the for an object of the specified type - /// and throws an exception if one isn't available. - /// - /// The record data model to use for retrieving data from the store. - /// The vector search. - /// The type of object being requested. - /// An optional key that can be used to help identify the target service. - /// The found object. - /// is . - /// is . - /// No service of the requested type for the specified key is available. - public static object GetRequiredService(this IVectorSearchable vectorSearch, Type serviceType, object? serviceKey = null) - { - if (vectorSearch is null) { throw new ArgumentNullException(nameof(vectorSearch)); } - if (serviceType is null) { throw new ArgumentNullException(nameof(serviceType)); } - - return - vectorSearch.GetService(serviceType, serviceKey) ?? - throw Throw.CreateMissingServiceException(serviceType, serviceKey); - } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/VectorSearchResult.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/VectorSearchResult.cs deleted file mode 100644 index f5793844d674..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorSearch/VectorSearchResult.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.Extensions.VectorData; - -/// -/// Represents a single search result from a vector search. -/// -/// The record data model to use for retrieving data from the store. -public sealed class VectorSearchResult -{ - /// - /// Initializes a new instance of the class. - /// - /// The record that was found by the search. - /// The score of this result in relation to the search query. - public VectorSearchResult(TRecord record, double? score) - { - this.Record = record; - this.Score = score; - } - - /// - /// Gets the record that was found by the search. - /// - public TRecord Record { get; } - - /// - /// Gets the score of this result in relation to the search query. - /// - public double? Score { get; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStore.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStore.cs deleted file mode 100644 index ab284796fd6b..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStore.cs +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Threading; -using System.Threading.Tasks; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Represents a vector store that contains collections of records. -/// -/// -/// This type can be used with collections of any schema type, but requires you to provide schema information when getting a collection. -/// Unless otherwise documented, implementations of this abstract base class can be expected to be thread-safe, and can be used concurrently from multiple threads. -/// -public abstract class VectorStore : IDisposable -{ - /// - /// Gets a collection from the vector store. - /// - /// The data type of the record key. - /// The record data model to use for adding, updating, and retrieving data from the collection. - /// The name of the collection. - /// The schema of the record type. - /// A new instance for managing the records in the collection. - /// - /// To successfully request a collection, either must be annotated with attributes that define the schema of - /// the record type, or must be provided. - /// - /// - /// - /// - [RequiresDynamicCode("This API is not compatible with NativeAOT. For dynamic mapping via Dictionary, use GetCollectionDynamic() instead.")] - [RequiresUnreferencedCode("This API is not compatible with trimming. For dynamic mapping via Dictionary, use GetCollectionDynamic() instead.")] - public abstract VectorStoreCollection GetCollection(string name, VectorStoreCollectionDefinition? definition = null) - where TKey : notnull - where TRecord : class; - - /// - /// Gets a collection from the vector store, using dynamic mapping; the record type is represented as a . - /// - /// The name of the collection. - /// The schema of the record type. - /// A new instance for managing the records in the collection. - public abstract VectorStoreCollection> GetDynamicCollection(string name, VectorStoreCollectionDefinition definition); - - /// - /// Retrieves the names of all the collections in the vector store. - /// - /// The to monitor for cancellation requests. The default is . - /// The list of names of all the collections in the vector store. - public abstract IAsyncEnumerable ListCollectionNamesAsync(CancellationToken cancellationToken = default); - - /// - /// Checks if the collection exists in the vector store. - /// - /// The name of the collection. - /// The to monitor for cancellation requests. The default is . - /// if the collection exists, otherwise. - public abstract Task CollectionExistsAsync(string name, CancellationToken cancellationToken = default); - - /// - /// Deletes the collection from the vector store. - /// - /// The name of the collection to delete. - /// The to monitor for cancellation requests. The default is . - /// A that completes when the collection has been deleted. - public abstract Task EnsureCollectionDeletedAsync(string name, CancellationToken cancellationToken = default); - - /// Asks the for an object of the specified type . - /// The type of object being requested. - /// An optional key that can be used to help identify the target service. - /// The found object, otherwise . - /// is . - /// - /// The purpose of this method is to allow for the retrieval of strongly typed services that might be provided by the , - /// including itself or any services it might be wrapping. For example, to access the for the instance, - /// can be used to request it. - /// - public abstract object? GetService(Type serviceType, object? serviceKey = null); - - /// - /// Disposes the and releases any resources it holds. - /// - /// if called from ; if called from a finalizer. - protected virtual void Dispose(bool disposing) - { - } - - /// - public void Dispose() - { - this.Dispose(disposing: true); - GC.SuppressFinalize(this); - } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollection.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollection.cs deleted file mode 100644 index cf5636215712..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollection.cs +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq.Expressions; -using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Represents a named collection of records in a vector store, and can be used to search and manage records, and to create or delete the collection itself. -/// -/// The data type of the record key. -/// The record data model to use for adding, updating, and retrieving data from the store. -/// -/// Unless otherwise documented, implementations of this abstract base class can be expected to be thread-safe, and can be used concurrently from multiple threads. -/// -#pragma warning disable CA1711 // Identifiers should not have incorrect suffix (Collection) -public abstract class VectorStoreCollection : IVectorSearchable, IDisposable -#pragma warning restore CA1711 - where TKey : notnull - where TRecord : class -{ - /// - /// Gets the name of the collection. - /// - public abstract string Name { get; } - - /// - /// Checks if the collection exists in the vector store. - /// - /// The to monitor for cancellation requests. The default is . - /// if the collection exists, otherwise. - public abstract Task CollectionExistsAsync(CancellationToken cancellationToken = default); - - /// - /// Creates this collection in the vector store if it doesn't already exist. - /// - /// The to monitor for cancellation requests. The default is . - /// A that completes when the collection has been created. - public abstract Task EnsureCollectionExistsAsync(CancellationToken cancellationToken = default); - - /// - /// Deletes the collection from the vector store if it exists. - /// - /// The to monitor for cancellation requests. The default is . - /// A that completes when the collection has been deleted. - public abstract Task EnsureCollectionDeletedAsync(CancellationToken cancellationToken = default); - - /// - /// Gets a record from the vector store. Does not guarantee that the collection exists. - /// Returns null if the record is not found. - /// - /// The unique ID associated with the record to get. - /// Optional options for retrieving the record. - /// The to monitor for cancellation requests. The default is . - /// The record if found, otherwise null. - /// The command fails to execute for any reason. - public abstract Task GetAsync(TKey key, RecordRetrievalOptions? options = default, CancellationToken cancellationToken = default); - - /// - /// Gets a batch of records from the vector store. Does not guarantee that the collection exists. - /// - /// The unique IDs associated with the record to get. - /// Optional options for retrieving the records. - /// The to monitor for cancellation requests. The default is . - /// The records associated with the specified unique keys. - /// - /// - /// The exact method of retrieval is implementation-specific and can vary based on database support. - /// The default implementation of this method retrieves the records one after the other, but implementations which supporting batching can override to provide a more efficient implementation. - /// - /// - /// Only found records are returned, so the result set might be smaller than the requested keys. - /// - /// - /// This method throws for any issues other than records not being found. - /// - /// - /// The command fails to execute for any reason. - public virtual async IAsyncEnumerable GetAsync(IEnumerable keys, RecordRetrievalOptions? options = default, [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - Verify.NotNull(keys); - - foreach (var key in keys) - { - var record = await this.GetAsync(key, options, cancellationToken).ConfigureAwait(false); - - if (record is not null) - { - yield return record; - } - } - } - - /// - /// Deletes a record from the vector store. Does not guarantee that the collection exists. - /// - /// The unique ID associated with the record to remove. - /// The to monitor for cancellation requests. The default is . - /// The unique identifier for the record. - /// The command fails to execute for any reason other than that the record does not exist. - public abstract Task DeleteAsync(TKey key, CancellationToken cancellationToken = default); - - /// - /// Deletes a batch of records from the vector store. Does not guarantee that the collection exists. - /// - /// The unique IDs associated with the records to remove. - /// The to monitor for cancellation requests. The default is . - /// A that completes when the records have been deleted. - /// - /// - /// The exact method of deleting is implementation-specific and can vary based on database support. - /// The default implementation of this method deletes the records one after the other, but implementations which supporting batching can override to provide a more efficient implementation. - /// - /// - /// If a record isn't found, it is ignored and the batch succeeds. - /// If any record can't be deleted for any other reason, the operation throws. Some records might have already been deleted while others might not have, so the entire operation should be retried. - /// - /// - /// The command fails to execute for any reason other than that a record does not exist. - public virtual async Task DeleteAsync(IEnumerable keys, CancellationToken cancellationToken = default) - { - Verify.NotNull(keys); - - foreach (var key in keys) - { - await this.DeleteAsync(key, cancellationToken).ConfigureAwait(false); - } - } - - /// - /// Upserts a record into the vector store. Does not guarantee that the collection exists. - /// If the record already exists, it is updated. - /// If the record does not exist, it is created. - /// - /// The record to upsert. - /// The to monitor for cancellation requests. The default is . - /// The command fails to execute for any reason. - public abstract Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default); - - /// - /// Upserts a batch of records into the vector store. Does not guarantee that the collection exists. - /// If the record already exists, it is updated. - /// If the record does not exist, it is created. - /// - /// The records to upsert. - /// The to monitor for cancellation requests. The default is . - /// - /// - /// The exact method of upserting the batch is implementation-specific and can vary based on database support. - /// - /// - /// Similarly, the error behavior can vary across databases: where possible, the batch should be upserted atomically, so that any errors cause the entire batch to be rolled - /// back. Where not supported, some records might be upserted while others are not. If key properties are set by the user, then the entire upsert operation is idempotent, - /// and can simply be retried again if an error occurs. However, if store-generated keys are in use, the upsert operation is no longer idempotent; in that case, if the - /// database doesn't guarantee atomicity, retrying could cause duplicate records to be created. - /// - /// - /// Implementations of should implement this method in a way which performs embedding generation once for the batch, rather than - /// generating an embedding for each record separately. This is why a default implementation that calls is not provided. - /// - /// - /// The command fails to execute for any reason. - public abstract Task UpsertAsync(IEnumerable records, CancellationToken cancellationToken = default); - - /// - /// Gets matching records from the vector store. Does not guarantee that the collection exists. - /// - /// The predicate to filter the records. - /// The maximum number of results to return. - /// Options for retrieving the records. - /// The to monitor for cancellation requests. The default is . - /// The records that match the given predicate. - /// The command fails to execute for any reason. - public abstract IAsyncEnumerable GetAsync(Expression> filter, int top, FilteredRecordRetrievalOptions? options = null, CancellationToken cancellationToken = default); - - /// - public abstract IAsyncEnumerable> SearchAsync(TInput searchValue, int top, VectorSearchOptions? options = null, CancellationToken cancellationToken = default) - where TInput : notnull; - - /// - public abstract object? GetService(Type serviceType, object? serviceKey = null); - - /// - /// Disposes the and releases any resources it holds. - /// - /// if called from ; if called from a finalizer. - protected virtual void Dispose(bool disposing) - { - } - - /// - public void Dispose() - { - this.Dispose(disposing: true); - GC.SuppressFinalize(this); - } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollectionMetadata.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollectionMetadata.cs deleted file mode 100644 index f36fe294ce63..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollectionMetadata.cs +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.Extensions.VectorData; - -/// Provides metadata about an . -public class VectorStoreCollectionMetadata -{ - /// Gets or sets the name of the vector store system. - /// - /// Where possible, this value maps to the "db.system.name" attribute defined in the - /// OpenTelemetry Semantic Conventions for database calls and systems; see . - /// Example: redis, sqlite, mysql. - /// - public string? VectorStoreSystemName { get; init; } - - /// - /// Gets or sets the name of the vector store (database). - /// - public string? VectorStoreName { get; init; } - - /// - /// Gets or sets the name of a collection (table, container) within the vector store (database). - /// - public string? CollectionName { get; init; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollectionOptions.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollectionOptions.cs deleted file mode 100644 index 04f67aae662b..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreCollectionOptions.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Microsoft.Extensions.AI; - -namespace Microsoft.Extensions.VectorData; - -/// Defines an abstract base class for options passed to a collection. -public abstract class VectorStoreCollectionOptions -{ - /// - /// Initializes a new instance of the class. - /// - protected VectorStoreCollectionOptions() - { - } - - /// - /// Initializes a new instance of the class. - /// - protected VectorStoreCollectionOptions(VectorStoreCollectionOptions? source) - { - this.Definition = source?.Definition; - this.EmbeddingGenerator = source?.EmbeddingGenerator; - } - - /// - /// Gets or sets an optional record definition that defines the schema of the record type. - /// - /// - /// If not provided, the schema will be inferred from the record model class using reflection. - /// In this case, the record model properties must be annotated with the appropriate attributes to indicate their usage. - /// See , , and . - /// - public VectorStoreCollectionDefinition? Definition { get; set; } - - /// - /// Gets or sets the default embedding generator to use when generating vectors embeddings with this collection. - /// - public IEmbeddingGenerator? EmbeddingGenerator { get; set; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreException.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreException.cs deleted file mode 100644 index a90599b34142..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreException.cs +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// -/// Defines a base exception type for any type of failure when using vector stores. -/// -public class VectorStoreException : Exception -{ - /// - /// Initializes a new instance of the class. - /// - public VectorStoreException() - { - } - - /// - /// Initializes a new instance of the class with a specified error message. - /// - /// The error message that explains the reason for the exception. - public VectorStoreException(string? message) : base(message) - { - } - - /// - /// Initializes a new instance of the class with a specified error message and a reference to the inner exception that's the cause of this exception. - /// - /// The error message that explains the reason for the exception. - /// The exception that's the cause of the current exception, or a null reference if no inner exception is specified. - public VectorStoreException(string? message, Exception? innerException) : base(message, innerException) - { - } - - /// Gets or sets the name of the vector store system. - /// - /// Where possible, this value maps to the "db.system.name" attribute defined in the - /// OpenTelemetry Semantic Conventions for database calls and systems; see . - /// Example: redis, sqlite, mysql. - /// - public string? VectorStoreSystemName { get; init; } - - /// - /// Gets or sets the name of the vector store (database). - /// - public string? VectorStoreName { get; init; } - - /// - /// Gets or sets the name of the vector store collection that the failing operation was performed on. - /// - public string? CollectionName { get; init; } - - /// - /// Gets or sets the name of the vector store operation that failed. - /// - public string? OperationName { get; init; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreExtensions.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreExtensions.cs deleted file mode 100644 index ceeecb8166bb..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreExtensions.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; - -namespace Microsoft.Extensions.VectorData; - -/// Provides a collection of static methods for extending instances. -public static class VectorStoreExtensions -{ - /// - /// Asks the for an object of the specified type - /// and throws an exception if one isn't available. - /// - /// The record data model to use for retrieving data from the store. - /// The vector store. - /// The type of object being requested. - /// An optional key that can be used to help identify the target service. - /// The found object. - /// is . - /// is . - /// No service of the requested type for the specified key is available. - public static object GetRequiredService(this VectorStore vectorStore, Type serviceType, object? serviceKey = null) - { - if (vectorStore is null) { throw new ArgumentNullException(nameof(vectorStore)); } - if (serviceType is null) { throw new ArgumentNullException(nameof(serviceType)); } - - return - vectorStore.GetService(serviceType, serviceKey) ?? - throw Throw.CreateMissingServiceException(serviceType, serviceKey); - } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreMetadata.cs b/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreMetadata.cs deleted file mode 100644 index 97571a0691bf..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/VectorStorage/VectorStoreMetadata.cs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.Extensions.VectorData; - -/// Provides metadata about an . -public class VectorStoreMetadata -{ - /// Gets or sets the name of the vector store system. - /// - /// Where possible, this value maps to the "db.system.name" attribute defined in the - /// OpenTelemetry Semantic Conventions for database calls and systems; see . - /// Example: redis, sqlite, mysql. - /// - public string? VectorStoreSystemName { get; init; } - - /// - /// Gets or sets the name of the vector store (database). - /// - public string? VectorStoreName { get; init; } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/neticon.png b/dotnet/src/VectorData/VectorData.Abstractions/neticon.png deleted file mode 100644 index a0f1fdbf4d5eae0e561018cccee74f6a454cdb9c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7006 zcmeHMXH-+`n%)#eMU;C)kZw7O2nvFLpcE@A^-u+AN(mh$UH*JD5Jjm{4}uUR zs5C(zdURn*zrcHqdVxK)P)7322TAMVbNR4HRzo3_~zdgjvf?Ot98@H{LHdy zK*)TM=g&B9f}+9IKfm=aF5e3_{PQJ$ zY4?9DHvtd+Y14o8TQs=)&+P)Wjb3|LIT@*NDqyYm#gu^q*EFSow<%yKVx`_Ka)!0 z2YAaQr%LYyQ%n$Rjx)e%JeM5_ov70FUMveJTS(J+%C4(L)~h*MQ8!wJtf_X{`Ol?k z;{27%#**2uiR&R6-eaRK1Mdgl2xHQ=uS(~VqsTVrsUnQhc zRIK5>@(05w3gHYdsI0;;sOO66pUEl)DGyD(D4>$7drUDFZ|uxx;-nWj7d|rj=u+D@ z-HU+mLOInrsXdSL1Z6nVB&D z@>f4!yq=_B+16+qw5k=4o#*tf;6Oe*F;`&L!)bT{U7Wc3YmG2;NRxb%woCt~*Yr2E zfwiUdS=7SK&5>df-aqY8lp~SEUG*ziXGvHMLp_#vgvVMQ*&{+d@(a>v4;7p_%Jte0Ga5zNbUI28WAgY5f?FX^;q`1WTw2~t|P54N&e^@=nFqDj}W#o z_-kZBWDQ%($YJH43Y7YrbjfsUrAEjla>?j0;YLdXxjK}P@xDGc%r&c)6`t?XW=*{r z%Z^p)?6*7obKU_;NZK_ejh9n&?qzO0#(}Uo+KSm|e}q1+f$wM!G8>lLvKK1UK^uz5 zDk&5(DuUnzQy{aQ8%b~*_4Ri`TOj}Dd{0OCls}^VD8=qDC%Q9tSSt5LZoxd!|ai3oGtf&cOy(`^W9zMNR;bII|OS+Pe(-9=f!m6}w zV>f(mH^BYE-=Wl=)Q2s2TF*j&tRkN0KOu3-(VN?4?-v|?W^Xj)@u4^bNB%bN+f|D= z?r1ey$UbahYv!qISaxV8>+1Mnz!M&S1o+~titx|65MA`iQMjscL!+LOGjZ?p>}x6d z4`FiZV9i-E6F8c|Fq37-TTTtJOdIZ9<*YrJU86UuQr6dipNC%AxT?lXa9U=`iq+2= zOT!CFUlJM1&INj~InR!=@x@{Z8BnvgL~_>nN)y@!r<0$uGCJ<0B-q!vZn@~#5^Ig8B}}g&dYBee=x50Wv$R^^f%aTE~g_a7&8Y(5L>! zkYgCl@1ZVqFSwkH(ns-EtYbOFLrarf#r6W9#x8rO<<_6h33faYV{<&_gBahO#ga9j z$|}=ea)vEm|Hb`E%L9Gn#Osxg( z&sxXz7lsse+_i@<_LUl@8$916h*m6!R?~zr_ZQU^H3F(aC1is#I$VP$GO(s!pT&Y# z85JYcwQqu6Ja6sje&x*)nOdx;bt1hNMTSwSikFeKE)+MRrW?mg=8mp^AR_kz{C%e* z32H_>c600^d$9)ob+$yzpyxHa+k0Sz7GG41I0A59bKJf?X}E6mX$pU~Wc%_?$2w1s zZEbk$svZ4U+WH;XPEb^-IqhGQX1U|z8KWp8&jVlWFPP+7Um6;oMy?>TFU`cMT5bYx z;7_~MfZ(sumPQHg++U)9PT=+=zxu+qmP==xJ&oI%XgD8=YZo%*rGq2U_J^D4d%7H`}jau-;<_^n?THcf9*rKD^J#%p%l zA8DILPr+wPY^MpxQbxGXG2f0xcjxSw;wjl53EsXe0poYHgfc(T;v5J;H$neUhElxe zrX0NdQ4e#4L4e-JmsN$%C+#BKX8TYA1YlhN`|QyqnlH{Igil*i0?NrD9qi2Fw_&~eMSk3UGyWzcay4oPaWE~nJ{R}-u+%oE z^4pk7G%~M66x6$a(@21!KD)Us1JG?!Xn4Zb;NYOn2SGc%JK!@mQv*PGMGxMb{#a4F z_#t!~GhhJR9)$w;fi20azFx86@7j4yB zpC7-bK<170rK@aOPg zDv69Iy;oMY0yq-ORy`~=Y8>ZQ_}+6m=ElBFD(BO@q9)h-K%)s9-^rh(;7T`vu={0p zCzf*G!~Iex?wWwWS?rOOYx{i!_Lh~OXJ7gYPR(bWfke`)l(GCjjtT06t7+0hHGHhh zA9y}JSM5#_xw|dqtlV?PVqZwGRm*pM)dvDj|LAzkF?4x}RLkCA#>G3V21ZLIt^gG< zQI&0O8}Rf;Def0;ZbweV+|x(R-?(Vnj5F9~eOT)4!nDr7Yq-5!y1bz1t;HjQSLn-A zt1qf%FzvKZ`+#!ufUYj;;FE!eL$>Pcse)qp0BW@>*U{2zo_CWHpgvHpnGofD&KYKY z+!}avbdRD^hZQf zU#$@f{W=^JvL7g)bcEZ<)O9tw4?Dxp&lksZ;$I_{?{l;o=>&}=tF-5MU&27^*rhJT zcd0DiLPxBSPJ<5cx}JGQAds^*(&j4-nHoTwx>dVUGJHkMM7w*nPbN5n_W)JJ zoSF~F)URWm1xS-QkhpAB(#}xq`0?;AQ=#^xj8iv{-*?l`8a;)kpuatAQXeVT+=;#A zT0rvGu`_`{>KMvxzgLkb$EeCy`RyvAx+nC!D381cssru;3nBjt{S>AGvQAs(kxLO{ zIp*xXImIAQJ>kiL&b~R(P_(nAu2z<~Dc*-_c3=C`sjCz@AZVOwgE5s@G#uy{iQNJ} z*pY1bjnx4K{yik#93ftw2}MI#Dt>w>)q5vp~-G zX7!=BUrYpB-3#04(mvmC$-Y!WY8${8gcraWB}q}i z(|PAS*SoXp)9`8tTYTuy7`=#uWFoR#J2(AVcxr-9uF+7kB$GxNkA$Vfoz}l40*Ydo zXReR;i`X4$Te~{&2?RE~^39WlS?>E>my@CS3|paiTe-zGjS$iwI*YbAHOwW*PD@wI z=Nl-L-*Y(4b+hX{-tb98arKb!Q^EK+RA0Lfp4`cv&x7o<`~ghNZ#@Z$`B6O*2R6%R z+kg>9tGG(TtYgVXWD_X)ySeq_3Tq2*GEPMlF@o;BBxfbxC%!xOuwUa+?wXac%Dce> z+d&$P_VsrSw*$bMY#z8~U%K$AIc8vOosw2D4`XdBe5NKVuc+s10x-cw)v;&2Yd`@# z6UL-Y1G;FY$G$?{@cwL6zaRL5p_lTzugeI5PB@eSk^x^LJ=N!qHsScr*=1fnx>1;L zY5eqB8dlecz6GSs<7{=#sl?FWEY66Ejk>f}1odw~P?}i0yH&4d%vKKZ@hTi7-IW8%;{(vI`&L;i z@`wN4O!SHFV&u%JzXt*g%E%4J$^z@6FOtA7Yc(*Rz2%_90Exxp+}r^Vb|pF?C;F8w zu&f+_Jsvg^Wp?I6!+uV$Bi#fzohClm^T{PdQzz%Nn}GENT0zaz{xqo+NWJ!QdLYKf zBHdX|LMnBh5jXZ;>OoAWv*rOX&O8Sbzjyl*y-%<2V2oE_*lEG(1GlpzBZ6aoOp%y8 ze&=uJp63A7*h}C9j-sY70bc4bHQr`@q#!@&!5LxUu`)c;-&WVK?$9+vP%D`7v^_`5 zrOcY7w(+sWUl!hkCI>q|qg_*OZ$os^0Fsg`di5ki_Tzr$8gh}#WNKHtX|hlAupfW6 zk_ZWVB&Hjb9ZbLk!Ie1lMyGd?qhgq8>{#iC>Kg^*taLx^YuW+VQG;}IK{6+Y@0i7& z6iRAQBlI8*LwK}P>x0;cL*en^{8^OvUg%KTXIa~~>xA%u_2)y{h_+YQ?tpDgX9rIe zOo3t5%oVK)PzXFaqN#F2^qJbgB3HzT`{nJcFO`#ATLWNBXfYU5CYHs&PnH^f*Wl6k z?<0KM*e@M?auAvtBi}A#6V#ej{yvSOE8v?4^Jb8y4~i{ zSIC{Kc9#!&HhKqJI9L>s*NbwiwWXI+w-X6TM}&3$PlPOE+G8HP8Hi(#UMtyKy= zLo(ZOb7qTQ^r{NHBg^h=C`gbboZigk0*;z5+XW@P;EzUwQZv5|SZ6W0tBbATVDt$& z4th!!{t_tBc>V9qZE^8&@=VbaMh;!ivCF~IC28PzN2Z{@`)H;y3+{?j%eQl6gP|I9 z-agi;Y>P($m>0yG48Z>=AC0W_h5((46THSuk)X||?u=A_N-{J)`M9Q^WnUMh84VTQ zIvQlFtG4Z5X~3!o0K!K+^E@{TZ;5W3XkNzy z*j?DZB4J)s(LK@K0K1T4u&xvPHDTX zs$=NfQalJo9RXF+0@j1~t~aK@*DAWgsI@Sl{8AP8%T`P`Vu~Tv_%ZmbJz^#V>NJZl-TbST^RMK5DlNOs$kegkbICLYRJk-}g{l-Wn^Vya`SL3T1tiIw^Z zm~h)cx+UimpKrqQ=$a*_BCrvMGi%5Nr5qU)hq|P1Tjp!gLgpIqRRIs`qsDGjcel*OH-c~&6W812bsUI z>umkx8_8Ottu&n?L`^t@;63h8!Nb19V4*G1v2?3e;$WrvvX7%#JaxH?R) zN@KLmgq3q$NONDrj=7c`8~kK5VTf>xS$Q2C8@T{(7ygTX1N^6hZ&3*F7Z@!5FaMz+ n@b3Qu^xx$8Uk}h2jH{d|uJ4jrSC|P(2)ca1@;v^m$K8JeR7TPQ diff --git a/dotnet/src/VectorData/Weaviate/Weaviate.csproj b/dotnet/src/VectorData/Weaviate/Weaviate.csproj index 26b59e293cb9..4e98297f7d08 100644 --- a/dotnet/src/VectorData/Weaviate/Weaviate.csproj +++ b/dotnet/src/VectorData/Weaviate/Weaviate.csproj @@ -27,6 +27,7 @@ + @@ -34,10 +35,6 @@ - - - - diff --git a/dotnet/test/VectorData/VectorData.ConformanceTests/VectorData.ConformanceTests.csproj b/dotnet/test/VectorData/VectorData.ConformanceTests/VectorData.ConformanceTests.csproj index dfdf5e4c0da0..eedce5d9664e 100644 --- a/dotnet/test/VectorData/VectorData.ConformanceTests/VectorData.ConformanceTests.csproj +++ b/dotnet/test/VectorData/VectorData.ConformanceTests/VectorData.ConformanceTests.csproj @@ -15,13 +15,10 @@ + - - - - diff --git a/dotnet/test/VectorData/VectorData.UnitTests/CollectionModelBuilderTests.cs b/dotnet/test/VectorData/VectorData.UnitTests/CollectionModelBuilderTests.cs deleted file mode 100644 index fe588e1100d1..000000000000 --- a/dotnet/test/VectorData/VectorData.UnitTests/CollectionModelBuilderTests.cs +++ /dev/null @@ -1,525 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.AI; -using Microsoft.Extensions.VectorData; -using Microsoft.Extensions.VectorData.ProviderServices; -using Xunit; - -namespace VectorData.UnitTests; - -#pragma warning disable CA2000 // Dispose objects before losing scope - -public class CollectionModelBuilderTests -{ - [Fact] - public void Default_embedding_generator_without_record_definition() - { - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - var model = new CustomModelBuilder().Build(typeof(RecordWithStringVectorProperty), typeof(int), definition: null, embeddingGenerator); - - // The embedding's .NET type (Embedding) is inferred from the embedding generator. - Assert.Same(embeddingGenerator, model.VectorProperty.EmbeddingGenerator); - Assert.Same(typeof(string), model.VectorProperty.Type); - Assert.Same(typeof(Embedding), model.VectorProperty.EmbeddingType); - } - - [Fact] - public void Default_embedding_generator_with_clr_type_and_record_definition() - { - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - - var recordDefinition = new VectorStoreCollectionDefinition - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)), - new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)), - new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(string), dimensions: 3) - { - // The following configures the property to be Embedding (non-default embedding type for this connector) - EmbeddingType = typeof(Embedding) - } - ] - }; - - var model = new CustomModelBuilder().Build(typeof(RecordWithStringVectorProperty), typeof(int), recordDefinition, embeddingGenerator); - - // The embedding's .NET type (Embedding) is inferred from the embedding generator. - Assert.Same(embeddingGenerator, model.VectorProperty.EmbeddingGenerator); - Assert.Same(typeof(string), model.VectorProperty.Type); - Assert.Same(typeof(Embedding), model.VectorProperty.EmbeddingType); - } - - [Fact] - public void Default_embedding_generator_with_dynamic() - { - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - - var recordDefinition = new VectorStoreCollectionDefinition - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)), - new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)), - new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(string), dimensions: 3) - ] - }; - - var model = new CustomModelBuilder().BuildDynamic(recordDefinition, embeddingGenerator); - - // The embedding's .NET type (Embedding) is inferred from the embedding generator. - Assert.Same(embeddingGenerator, model.VectorProperty.EmbeddingGenerator); - Assert.Same(typeof(string), model.VectorProperty.Type); - Assert.Same(typeof(Embedding), model.VectorProperty.EmbeddingType); - } - - [Fact] - public void Default_embedding_generator_with_dynamic_and_non_default_EmbeddingType() - { - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - - var recordDefinition = new VectorStoreCollectionDefinition - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)), - new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)), - new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(string), dimensions: 3) - { - EmbeddingType = typeof(Embedding) - } - ] - }; - - var model = new CustomModelBuilder().BuildDynamic(recordDefinition, embeddingGenerator); - - Assert.Same(embeddingGenerator, model.VectorProperty.EmbeddingGenerator); - Assert.Same(typeof(string), model.VectorProperty.Type); - Assert.Same(typeof(Embedding), model.VectorProperty.EmbeddingType); - } - - [Fact] - public void Property_embedding_generator_takes_precedence_over_default_generator() - { - using var propertyEmbeddingGenerator = new FakeEmbeddingGenerator>(); - using var defaultEmbeddingGenerator = new FakeEmbeddingGenerator>(); - - var recordDefinition = new VectorStoreCollectionDefinition - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)), - new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)), - new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(string), dimensions: 3) - { - EmbeddingGenerator = propertyEmbeddingGenerator - } - ] - }; - - var model = new CustomModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator); - - Assert.Same(propertyEmbeddingGenerator, model.VectorProperty.EmbeddingGenerator); - } - - [Theory] - [InlineData(true)] - [InlineData(false)] - public void Embedding_property_type_with_default_embedding_generator(bool dynamic) - { - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - - var model = dynamic - ? new CustomModelBuilder().BuildDynamic( - new VectorStoreCollectionDefinition - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)), - new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)), - new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(ReadOnlyMemory), dimensions: 3) - ] - }, - embeddingGenerator) - : new CustomModelBuilder().Build(typeof(RecordWithEmbeddingVectorProperty), typeof(int), definition: null, embeddingGenerator); - - var vectorProperty = model.VectorProperty; - Assert.Same(embeddingGenerator, vectorProperty.EmbeddingGenerator); - Assert.Same(typeof(ReadOnlyMemory), vectorProperty.Type); - } - - [Fact] - public void Embedding_property_type_with_property_embedding_generator() - { - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - - var model = new CustomModelBuilder().Build( - typeof(RecordWithEmbeddingVectorProperty), - typeof(int), - new VectorStoreCollectionDefinition - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)), - new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)), - new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(ReadOnlyMemory), dimensions: 3) - { - EmbeddingGenerator = embeddingGenerator - } - ] - }, - embeddingGenerator); - - var vectorProperty = model.VectorProperty; - Assert.Same(embeddingGenerator, vectorProperty.EmbeddingGenerator); - Assert.Same(typeof(ReadOnlyMemory), vectorProperty.EmbeddingType); - } - - [Theory] - [InlineData(true)] - [InlineData(false)] - public void Custom_input_type(bool dynamic) - { - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - - // TODO: Allow custom input type without a record definition (i.e. generic attribute) - var recordDefinition = new VectorStoreCollectionDefinition - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)), - new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)), - new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), dimensions: 3) - ] - }; - - var model = dynamic - ? new CustomModelBuilder().BuildDynamic(recordDefinition, embeddingGenerator) - : new CustomModelBuilder().Build(typeof(RecordWithCustomerVectorProperty), typeof(int), recordDefinition, embeddingGenerator); - - var vectorProperty = model.VectorProperty; - - Assert.Same(embeddingGenerator, vectorProperty.EmbeddingGenerator); - Assert.Same(typeof(Customer), vectorProperty.Type); - Assert.Same(typeof(Embedding), vectorProperty.EmbeddingType); - } - - [Fact] - public void Incompatible_embedding_on_embedding_generator_throws() - { - // Embedding is not a supported embedding type by the connector - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - - var exception = Assert.Throws(() => - new CustomModelBuilder().Build(typeof(RecordWithStringVectorProperty), typeof(int), definition: null, embeddingGenerator)); - - Assert.Equal($"Embedding generator 'FakeEmbeddingGenerator>' on vector property '{nameof(RecordWithStringVectorProperty.Embedding)}' cannot convert the input type 'string' to a supported vector type (one of: ReadOnlyMemory, Embedding, float[], ReadOnlyMemory, Embedding, Half[]).", exception.Message); - } - - [Fact] - public void Incompatible_input_on_embedding_generator_throws() - { - // int is not a supported input type for the embedding generator - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - - var exception = Assert.Throws(() => - new CustomModelBuilder().Build(typeof(RecordWithStringVectorProperty), typeof(int), definition: null, embeddingGenerator)); - - Assert.Equal($"Embedding generator 'FakeEmbeddingGenerator>' on vector property '{nameof(RecordWithStringVectorProperty.Embedding)}' cannot convert the input type 'string' to a supported vector type (one of: ReadOnlyMemory, Embedding, float[], ReadOnlyMemory, Embedding, Half[]).", exception.Message); - } - - [Fact] - public void Non_embedding_vector_property_without_embedding_generator_throws() - { - var exception = Assert.Throws(() => - new CustomModelBuilder().Build(typeof(RecordWithStringVectorProperty), typeof(int), definition: null, defaultEmbeddingGenerator: null)); - - Assert.Equal($"Vector property '{nameof(RecordWithStringVectorProperty.Embedding)}' has type 'string' which isn't supported by your provider, and no embedding generator is configured. Configure a generator that supports converting 'string' to vector type supported by your provider.", exception.Message); - } - - [Fact] - public void EmbeddingType_not_supported_by_provider() - { - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - - var recordDefinition = new VectorStoreCollectionDefinition - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)), - new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)), - new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(string), dimensions: 3) - { - EmbeddingType = typeof(Embedding) // The provider supports float/Half only, not byte - } - ] - }; - - var exception = Assert.Throws(() => - new CustomModelBuilder().Build(typeof(RecordWithStringVectorProperty), typeof(int), recordDefinition, embeddingGenerator)); - - Assert.Equal("Vector property 'Embedding' has embedding type 'Embedding' configured, but that type isn't supported by your provider. Supported types are ReadOnlyMemory, Embedding, float[], ReadOnlyMemory, Embedding, Half[].", exception.Message); - } - - [Fact] - public void EmbeddingType_not_supported_by_generator() - { - using var embeddingGenerator = new FakeEmbeddingGenerator>(); - - var recordDefinition = new VectorStoreCollectionDefinition - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)), - new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)), - new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(string), dimensions: 3) - { - EmbeddingType = typeof(Embedding) // The generator (instantiated above) supports only Embedding - } - ] - }; - - var exception = Assert.Throws(() => - new CustomModelBuilder().Build(typeof(RecordWithStringVectorProperty), typeof(int), recordDefinition, embeddingGenerator)); - - Assert.Equal("Vector property 'Embedding' has embedding type 'Embedding' configured, but that type isn't supported by your embedding generator.", exception.Message); - } - - [Fact] - public void Missing_Type_on_property_definition() - { - var recordDefinition = new VectorStoreCollectionDefinition - { - Properties = - [ - new VectorStoreKeyProperty(nameof(RecordWithEmbeddingVectorProperty.Id), typeof(int)), - new VectorStoreDataProperty(nameof(RecordWithEmbeddingVectorProperty.Name), typeof(string)), - new VectorStoreVectorProperty(nameof(RecordWithEmbeddingVectorProperty.Embedding), typeof(ReadOnlyMemory), dimensions: 3) - ] - }; - - // Key - recordDefinition.Properties[0].Type = null; - var exception = Assert.Throws(() => new CustomModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null)); - Assert.Equal($"Property '{nameof(RecordWithEmbeddingVectorProperty.Id)}' has no type specified in its definition, and does not have a corresponding .NET property. Specify the type on the definition.", exception.Message); - - // Data - recordDefinition.Properties[0].Type = typeof(int); - recordDefinition.Properties[1].Type = null; - exception = Assert.Throws(() => new CustomModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null)); - Assert.Equal($"Property '{nameof(RecordWithEmbeddingVectorProperty.Name)}' has no type specified in its definition, and does not have a corresponding .NET property. Specify the type on the definition.", exception.Message); - - // Vector - recordDefinition.Properties[1].Type = typeof(string); - recordDefinition.Properties[2].Type = null; - exception = Assert.Throws(() => new CustomModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null)); - Assert.Equal($"Property '{nameof(RecordWithEmbeddingVectorProperty.Embedding)}' has no type specified in its definition, and does not have a corresponding .NET property. Specify the type on the definition.", exception.Message); - } - - public class RecordWithStringVectorProperty - { - [VectorStoreKey] - public int Id { get; set; } - - [VectorStoreData] - public string Name { get; set; } - - [VectorStoreVector(Dimensions: 3)] - public string Embedding { get; set; } - } - - public class RecordWithEmbeddingVectorProperty - { - [VectorStoreKey] - public int Id { get; set; } - - [VectorStoreData] - public string Name { get; set; } - - [VectorStoreVector(Dimensions: 3)] - public ReadOnlyMemory Embedding { get; set; } - } - - public class RecordWithCustomerVectorProperty - { - [VectorStoreKey] - public int Id { get; set; } - - [VectorStoreData] - public string Name { get; set; } - - [VectorStoreVector(Dimensions: 3)] - public Customer Embedding { get; set; } - } - - public class Customer - { - public string FirstName { get; set; } - public string LastName { get; set; } - } - - private sealed class CustomModelBuilder(CollectionModelBuildingOptions? options = null) - : CollectionModelBuilder(options ?? s_defaultOptions) - { - private static readonly CollectionModelBuildingOptions s_defaultOptions = new() - { - SupportsMultipleVectors = true, - RequiresAtLeastOneVector = false - }; - - protected override void ValidateKeyProperty(KeyPropertyModel keyProperty) - { - var type = keyProperty.Type; - - if (type != typeof(string) && type != typeof(int)) - { - throw new NotSupportedException( - $"Property '{keyProperty.ModelName}' has unsupported type '{type.Name}'. Key properties must be one of the supported types: string, int."); - } - } - - protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) - { - supportedTypes = "string, int"; - - if (Nullable.GetUnderlyingType(type) is Type underlyingType) - { - type = underlyingType; - } - - return type == typeof(string) || type == typeof(int); - } - - protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) - => IsVectorPropertyTypeValidCore(type, out supportedTypes); - - internal static bool IsVectorPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes) - { - supportedTypes = "ReadOnlyMemory, Embedding, float[], ReadOnlyMemory, Embedding, Half[]"; - - if (Nullable.GetUnderlyingType(type) is Type underlyingType) - { - type = underlyingType; - } - - return type == typeof(ReadOnlyMemory) - || type == typeof(Embedding) - || type == typeof(float[]) - || type == typeof(ReadOnlyMemory) - || type == typeof(Embedding) - || type == typeof(Half[]); - } - - protected override IReadOnlyList EmbeddingGenerationDispatchers { get; } = - [ - EmbeddingGenerationDispatcher.Create>(), - EmbeddingGenerationDispatcher.Create>() - ]; - } - - [Fact] - public void IsAutoGenerated_attribute_true_overrides_default() - { - // Guid key with explicit IsAutoGenerated = true; the attribute should override SupportsKeyAutoGeneration. - var model = new GuidKeyModelBuilder().Build( - typeof(RecordWithGuidKeyAutoGeneratedTrue), typeof(Guid), definition: null, defaultEmbeddingGenerator: null); - - Assert.True(model.KeyProperty.IsAutoGenerated); - } - - [Fact] - public void IsAutoGenerated_attribute_false_overrides_default() - { - // Guid key with explicit IsAutoGenerated = false; the attribute should override SupportsKeyAutoGeneration, - // which would otherwise return true for Guid. - var model = new GuidKeyModelBuilder().Build( - typeof(RecordWithGuidKeyAutoGeneratedFalse), typeof(Guid), definition: null, defaultEmbeddingGenerator: null); - - Assert.False(model.KeyProperty.IsAutoGenerated); - } - - [Fact] - public void IsAutoGenerated_omitted_falls_back_to_SupportsKeyAutoGeneration_true() - { - // Guid key with no IsAutoGenerated attribute set; should fall back to SupportsKeyAutoGeneration, which returns true for Guid. - var model = new GuidKeyModelBuilder().Build( - typeof(RecordWithGuidKeyNoIsAutoGenerated), typeof(Guid), definition: null, defaultEmbeddingGenerator: null); - - Assert.True(model.KeyProperty.IsAutoGenerated); - } - - [Fact] - public void IsAutoGenerated_omitted_falls_back_to_SupportsKeyAutoGeneration_false() - { - // int key with no IsAutoGenerated attribute set; should fall back to SupportsKeyAutoGeneration, which returns false for non-Guid. - var model = new GuidKeyModelBuilder().Build( - typeof(RecordWithIntKeyNoIsAutoGenerated), typeof(int), definition: null, defaultEmbeddingGenerator: null); - - Assert.False(model.KeyProperty.IsAutoGenerated); - } - - public class RecordWithGuidKeyAutoGeneratedTrue - { - [VectorStoreKey(IsAutoGenerated = true)] - public Guid Id { get; set; } - } - - public class RecordWithGuidKeyAutoGeneratedFalse - { - [VectorStoreKey(IsAutoGenerated = false)] - public Guid Id { get; set; } - } - - public class RecordWithGuidKeyNoIsAutoGenerated - { - [VectorStoreKey] - public Guid Id { get; set; } - } - - public class RecordWithIntKeyNoIsAutoGenerated - { - [VectorStoreKey] - public int Id { get; set; } - } - - private sealed class GuidKeyModelBuilder() - : CollectionModelBuilder(new CollectionModelBuildingOptions - { - SupportsMultipleVectors = true, - RequiresAtLeastOneVector = false - }) - { - protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) - { - supportedTypes = null; - return true; - } - - protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) - { - supportedTypes = null; - return true; - } - } - - private sealed class FakeEmbeddingGenerator : IEmbeddingGenerator - where TEmbedding : Embedding - { - public Task> GenerateAsync( - IEnumerable values, - EmbeddingGenerationOptions? options = null, - CancellationToken cancellationToken = default) - => throw new UnreachableException(); - - public object? GetService(Type serviceType, object? serviceKey = null) - => throw new UnreachableException(); - - public void Dispose() { } - } -} diff --git a/dotnet/test/VectorData/VectorData.UnitTests/PropertyModelTests.cs b/dotnet/test/VectorData/VectorData.UnitTests/PropertyModelTests.cs deleted file mode 100644 index aad7b64beee3..000000000000 --- a/dotnet/test/VectorData/VectorData.UnitTests/PropertyModelTests.cs +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.Extensions.VectorData.ProviderServices; -using Xunit; - -namespace VectorData.UnitTests; - -public class PropertyModelTests -{ - #region Value type nullability - - [Fact] - public void IsNullable_NonNullableValueType_ReturnsFalse() - { - var property = new DataPropertyModel("test", typeof(int)); - Assert.False(property.IsNullable); - } - - [Fact] - public void IsNullable_NullableValueType_ReturnsTrue() - { - var property = new DataPropertyModel("test", typeof(int?)); - Assert.True(property.IsNullable); - } - - [Fact] - public void IsNullable_Guid_ReturnsFalse() - { - var property = new DataPropertyModel("test", typeof(Guid)); - Assert.False(property.IsNullable); - } - - [Fact] - public void IsNullable_NullableGuid_ReturnsTrue() - { - var property = new DataPropertyModel("test", typeof(Guid?)); - Assert.True(property.IsNullable); - } - - [Fact] - public void IsNullable_ReadOnlyMemoryFloat_ReturnsFalse() - { - var property = new DataPropertyModel("test", typeof(ReadOnlyMemory)); - Assert.False(property.IsNullable); - } - - [Fact] - public void IsNullable_NullableReadOnlyMemoryFloat_ReturnsTrue() - { - var property = new DataPropertyModel("test", typeof(ReadOnlyMemory?)); - Assert.True(property.IsNullable); - } - - #endregion - - #region Reference type nullability (dynamic mapping, no PropertyInfo) - - [Fact] - public void IsNullable_String_WithoutPropertyInfo_ReturnsTrue() - { - // Without PropertyInfo (dynamic mapping), reference types are assumed nullable - var property = new DataPropertyModel("test", typeof(string)); - Assert.True(property.IsNullable); - } - - [Fact] - public void IsNullable_ByteArray_WithoutPropertyInfo_ReturnsTrue() - { - var property = new DataPropertyModel("test", typeof(byte[])); - Assert.True(property.IsNullable); - } - - #endregion - -#if NET - #region NRT detection via NullabilityInfoContext (POCO mapping with PropertyInfo) - - [Fact] - public void IsNullable_NonNullableString_WithPropertyInfo_ReturnsFalse() - { - var propertyInfo = typeof(NrtTestRecord).GetProperty(nameof(NrtTestRecord.NonNullableString))!; - var property = new DataPropertyModel("test", typeof(string)) { PropertyInfo = propertyInfo }; - Assert.False(property.IsNullable); - } - - [Fact] - public void IsNullable_NullableString_WithPropertyInfo_ReturnsTrue() - { - var propertyInfo = typeof(NrtTestRecord).GetProperty(nameof(NrtTestRecord.NullableString))!; - var property = new DataPropertyModel("test", typeof(string)) { PropertyInfo = propertyInfo }; - Assert.True(property.IsNullable); - } - - [Fact] - public void IsNullable_NonNullableByteArray_WithPropertyInfo_ReturnsFalse() - { - var propertyInfo = typeof(NrtTestRecord).GetProperty(nameof(NrtTestRecord.NonNullableByteArray))!; - var property = new DataPropertyModel("test", typeof(byte[])) { PropertyInfo = propertyInfo }; - Assert.False(property.IsNullable); - } - - [Fact] - public void IsNullable_NullableByteArray_WithPropertyInfo_ReturnsTrue() - { - var propertyInfo = typeof(NrtTestRecord).GetProperty(nameof(NrtTestRecord.NullableByteArray))!; - var property = new DataPropertyModel("test", typeof(byte[])) { PropertyInfo = propertyInfo }; - Assert.True(property.IsNullable); - } - - [Fact] - public void IsNullable_ValueType_WithPropertyInfo_StillUsesTypeCheck() - { - var propertyInfo = typeof(NrtTestRecord).GetProperty(nameof(NrtTestRecord.NonNullableInt))!; - var property = new DataPropertyModel("test", typeof(int)) { PropertyInfo = propertyInfo }; - Assert.False(property.IsNullable); - - var nullablePropertyInfo = typeof(NrtTestRecord).GetProperty(nameof(NrtTestRecord.NullableInt))!; - var nullableProperty = new DataPropertyModel("test", typeof(int?)) { PropertyInfo = nullablePropertyInfo }; - Assert.True(nullableProperty.IsNullable); - } - -#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor -#pragma warning disable CA1812 // Class is used via reflection - private sealed class NrtTestRecord - { - public string NonNullableString { get; set; } - public string? NullableString { get; set; } - public byte[] NonNullableByteArray { get; set; } - public byte[]? NullableByteArray { get; set; } - public int NonNullableInt { get; set; } - public int? NullableInt { get; set; } - } -#pragma warning restore CA1812 -#pragma warning restore CS8618 - - #endregion -#endif -} diff --git a/dotnet/test/VectorData/VectorData.UnitTests/VectorData.UnitTests.csproj b/dotnet/test/VectorData/VectorData.UnitTests/VectorData.UnitTests.csproj deleted file mode 100644 index cc7bf6d4e2bc..000000000000 --- a/dotnet/test/VectorData/VectorData.UnitTests/VectorData.UnitTests.csproj +++ /dev/null @@ -1,43 +0,0 @@ - - - - VectorData.UnitTests - VectorData.UnitTests - net10.0 - true - enable - disable - false - $(NoWarn);VSTHRD111,CA2007,CS8618 - $(NoWarn);MEVD9001 - $(NoWarn);CA1515 - $(NoWarn);CA1707 - $(NoWarn);CA1716 - $(NoWarn);CA1720 - $(NoWarn);CA1721 - $(NoWarn);CA1861 - $(NoWarn);CA1863 - $(NoWarn);CA2007;VSTHRD111 - $(NoWarn);CS1591 - $(NoWarn);IDE1006 - - - - - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - - - - - - From f6ca2b5c5c5c522507527c8a6c92099246bb2c3b Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Wed, 24 Jun 2026 17:16:02 +0100 Subject: [PATCH 6/6] Exclude MEVD and related tests from build. --- .github/workflows/dotnet-build-and-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dotnet-build-and-test.yml b/.github/workflows/dotnet-build-and-test.yml index 8987c2f48e42..d7277dcdb76d 100644 --- a/.github/workflows/dotnet-build-and-test.yml +++ b/.github/workflows/dotnet-build-and-test.yml @@ -81,7 +81,7 @@ jobs: - name: Build dotnet solutions shell: bash run: | - export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | tr '\n' ' ') + export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | grep -v "MEVD.slnx" | tr '\n' ' ') for solution in $SOLUTIONS; do dotnet build $solution -c ${{ matrix.configuration }} --warnaserror done @@ -89,7 +89,7 @@ jobs: - name: Package install check shell: bash run: | - export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | tr '\n' ' ') + export SOLUTIONS=$(find ./dotnet/ -type f -name "*.slnx" | grep -v "MEVD.slnx" | tr '\n' ' ') for solution in $SOLUTIONS; do dotnet pack $solution -c ${{ matrix.configuration }} --no-build --no-restore --output ./artifacts done @@ -115,7 +115,7 @@ jobs: - name: Run Unit Tests shell: bash run: | - export UT_PROJECTS=$(find ./dotnet -type f -name "*.UnitTests.csproj" | grep -v -E "(Experimental.Orchestration.Flow.UnitTests.csproj|Experimental.Assistants.UnitTests.csproj)" | tr '\n' ' ') + export UT_PROJECTS=$(find ./dotnet -type f -name "*.UnitTests.csproj" | grep -v -E "(Experimental.Orchestration.Flow.UnitTests.csproj|Experimental.Assistants.UnitTests.csproj)" | grep -v "test/VectorData/" | tr '\n' ' ') for project in $UT_PROJECTS; do dotnet test -c ${{ matrix.configuration }} $project --no-build -v Normal --logger trx --collect:"XPlat Code Coverage" --results-directory:"TestResults/Coverage/" -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.ExcludeByAttribute=GeneratedCodeAttribute,CompilerGeneratedAttribute,ExcludeFromCodeCoverageAttribute done