diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java index feb43c99..6cb83041 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java @@ -2,9 +2,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; import java.util.List; @@ -16,7 +16,7 @@ public class Hotel { @VectorStoreRecordDataAttribute private final int code; @JsonProperty("summary") - @VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "descriptionEmbedding") + @VectorStoreRecordDataAttribute() private final String description; @JsonProperty("summaryEmbedding") @VectorStoreRecordVectorAttribute(dimensions = 3) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java index 8bee5a76..cb1bb2de 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -5,7 +5,7 @@ import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.tests.connectors.memory.Hotel; import com.mysql.cj.jdbc.MysqlDataSource; import org.junit.jupiter.params.ParameterizedTest; diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index 300f45b4..24945df6 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -2,12 +2,12 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.tests.connectors.memory.Hotel; import com.redis.testcontainers.RedisContainer; import org.junit.jupiter.api.BeforeAll; @@ -67,8 +67,6 @@ static void setup() { .withName("description") .withStorageName("summary") .withFieldType(String.class) - .withHasEmbedding(true) - .withEmbeddingFieldName("descriptionEmbedding") .build()); fields.add(VectorStoreRecordVectorField.builder() .withName("descriptionEmbedding") diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index 74e9ba23..9d833217 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -2,12 +2,12 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.tests.connectors.memory.Hotel; import com.redis.testcontainers.RedisContainer; import org.junit.jupiter.api.BeforeAll; @@ -68,8 +68,6 @@ static void setup() { .withName("description") .withStorageName("summary") .withFieldType(String.class) - .withHasEmbedding(true) - .withEmbeddingFieldName("descriptionEmbedding") .build()); fields.add(VectorStoreRecordVectorField.builder() .withName("descriptionEmbedding") diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java index 43fead85..fd63b9e5 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java @@ -5,12 +5,9 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisStorageType; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStore; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreOptions; -import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.tests.connectors.memory.Hotel; import com.redis.testcontainers.RedisContainer; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.testcontainers.junit.jupiter.Container; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/RunAll.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/RunAll.java index a5b19fe8..1a86b95d 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/RunAll.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/RunAll.java @@ -17,7 +17,7 @@ import com.microsoft.semantickernel.samples.syntaxexamples.functions.Example59_OpenAIFunctionCalling; import com.microsoft.semantickernel.samples.syntaxexamples.functions.Example60_AdvancedMethodFunctions; import com.microsoft.semantickernel.samples.syntaxexamples.java.KernelFunctionYaml_Example; -import com.microsoft.semantickernel.samples.syntaxexamples.memory.AzureAISearch_DataStorage; +import com.microsoft.semantickernel.samples.syntaxexamples.memory.AzureAISearchVectorStore; import com.microsoft.semantickernel.samples.syntaxexamples.plugins.Example10_DescribeAllPluginsAndFunctions; import com.microsoft.semantickernel.samples.syntaxexamples.plugins.Example13_ConversationSummaryPlugin; import com.microsoft.semantickernel.samples.syntaxexamples.template.Example06_TemplateLanguage; @@ -38,7 +38,7 @@ public class RunAll { public static void main(String[] args) { List mains = Arrays.asList( - AzureAISearch_DataStorage::main, + AzureAISearchVectorStore::main, Example01_NativeFunctions::main, Example03_Arguments::main, Example05_InlineFunctionDefinition::main, diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java similarity index 80% rename from samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java rename to samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java index 86666dc2..2a1a49ab 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java @@ -10,16 +10,15 @@ import com.azure.core.util.TracingOptions; import com.azure.search.documents.indexes.SearchIndexAsyncClient; import com.azure.search.documents.indexes.SearchIndexClientBuilder; -import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; -import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStore; import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreOptions; +import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Base64; @@ -27,10 +26,11 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; + import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -public class AzureAISearch_DataStorage { +public class AzureAISearchVectorStore { private static final String CLIENT_KEY = System.getenv("CLIENT_KEY"); private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY"); @@ -51,7 +51,7 @@ static class GitHubFile { @JsonProperty("fileId") // Set a different name for the storage field if needed @VectorStoreRecordKeyAttribute() private final String id; - @VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "embedding") + @VectorStoreRecordDataAttribute() private final String description; @VectorStoreRecordDataAttribute private final String link; @@ -118,13 +118,13 @@ public static void dataStorageWithAzureAISearch( OpenAITextEmbeddingGenerationService embeddingGeneration) { // Create a new Azure AI Search vector store - var azureAISearchVectorStore = AzureAISearchVectorStore.builder() - .withClient(searchClient) + var azureAISearchVectorStore = com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStore.builder() + .withSearchIndexAsyncClient(searchClient) .withOptions(new AzureAISearchVectorStoreOptions()) .build(); String collectionName = "skgithubfiles"; - var collection = azureAISearchVectorStore.getCollection( + var collection = (AzureAISearchVectorStoreRecordCollection) azureAISearchVectorStore.getCollection( collectionName, AzureAISearchVectorStoreRecordCollectionOptions.builder() .withRecordClass(GitHubFile.class) @@ -136,18 +136,31 @@ public static void dataStorageWithAzureAISearch( .then(storeData(collection, embeddingGeneration, sampleData())) .block(); - // Query the Azure AI Search client for results - // This might take a few seconds to return the best result - var result = searchClient.getSearchAsyncClient(collectionName) - .search("How to get started with the Semantic Kernel?") - .blockFirst(); + // Search for results + // Might need to wait for the data to be indexed + var results = search("How to get started", collection, embeddingGeneration).block(); + + if (results == null || results.isEmpty()) { + System.out.println("No search results found."); + return; + } + var searchResult = results.get(0); + System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", + searchResult.getScore(), searchResult.getRecord().link, searchResult.getRecord().description); + } + + + private static Mono>> search( + String searchText, + AzureAISearchVectorStoreRecordCollection recordCollection, + OpenAITextEmbeddingGenerationService embeddingGeneration) { - GitHubFile gitHubFile = result.getDocument(GitHubFile.class); - System.out.println("Best result: " + gitHubFile.description + ". Link: " + gitHubFile.link); + return embeddingGeneration.generateEmbeddingsAsync(Collections.singletonList(searchText)) + .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); } private static Mono> storeData( - VectorStoreRecordCollection recordStore, + AzureAISearchVectorStoreRecordCollection recordCollection, OpenAITextEmbeddingGenerationService embeddingGeneration, Map data) { @@ -163,7 +176,7 @@ private static Mono> storeData( entry.getValue(), entry.getKey(), embeddings.get(0).getVector()); - return recordStore.upsertAsync(gitHubFile, null); + return recordCollection.upsertAsync(gitHubFile, null); }); }) .collectList(); diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java index 80e6d490..70c7297d 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java @@ -6,12 +6,12 @@ import com.azure.core.credential.AzureKeyCredential; import com.azure.core.credential.KeyCredential; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.VolatileVectorStore; import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -37,7 +37,7 @@ static class GitHubFile { @VectorStoreRecordKeyAttribute() private final String id; - @VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "embedding") + @VectorStoreRecordDataAttribute() private final String description; @VectorStoreRecordDataAttribute private final String link; @@ -64,7 +64,7 @@ public String getDescription() { } static String encodeId(String realId) { - return AzureAISearch_DataStorage.GitHubFile.encodeId(realId); + return AzureAISearchVectorStore.GitHubFile.encodeId(realId); } } diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java index 4a9c6d10..9e2e4ea4 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java @@ -10,10 +10,10 @@ import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; import com.mysql.cj.jdbc.MysqlDataSource; import java.nio.charset.StandardCharsets; import java.sql.SQLException; @@ -45,7 +45,7 @@ static class GitHubFile { @VectorStoreRecordKeyAttribute() private final String id; - @VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "embedding") + @VectorStoreRecordDataAttribute() private final String description; @VectorStoreRecordDataAttribute private final String link; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java index 10674365..90acae83 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java @@ -12,10 +12,10 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStore; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreOptions; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; import java.util.Arrays; import java.util.Collections; @@ -47,7 +47,7 @@ public static class GitHubFile { @VectorStoreRecordKeyAttribute() private final String id; - @VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "embedding") + @VectorStoreRecordDataAttribute() private final String description; @VectorStoreRecordDataAttribute private final String link; @@ -78,7 +78,7 @@ public String getDescription() { } static String encodeId(String realId) { - return AzureAISearch_DataStorage.GitHubFile.encodeId(realId); + return AzureAISearchVectorStore.GitHubFile.encodeId(realId); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchEqualityFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchEqualityFilterClause.java new file mode 100644 index 00000000..98da48d1 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchEqualityFilterClause.java @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.azureaisearch; + +import com.microsoft.semantickernel.data.filtering.EqualityFilterClause; +import com.microsoft.semantickernel.exceptions.SKException; + +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; + +public class AzureAISearchEqualityFilterClause extends EqualityFilterClause { + + /** + * Initializes a new instance of the AzureAISearchEqualityFilterClause class. + * + * @param fieldName The field name to filter on. + * @param value The value. + */ + public AzureAISearchEqualityFilterClause(String fieldName, Object value) { + super(fieldName, value); + } + + /** + * Gets the filter string. + * + * @return The filter string. + */ + @Override + public String getFilter() { + String fieldName = getFieldName(); + Object value = getValue(); + + if (value instanceof String) { + return String.format("%s eq '%s'", fieldName, value); + } else if (value instanceof Boolean) { + return String.format("%s eq %s", fieldName, + value.toString().toLowerCase()); + } else if (value instanceof Integer) { + return String.format("%s eq %d", fieldName, (Integer) value); + } else if (value instanceof Long) { + return String.format("%s eq %d", fieldName, (Long) value); + } else if (value instanceof Float) { + return String.format("%s eq %f", fieldName, (Float) value); + } else if (value instanceof Double) { + return String.format("%s eq %f", fieldName, (Double) value); + } else if (value instanceof OffsetDateTime) { + return String.format("%s eq %s", fieldName, ((OffsetDateTime) value) + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + } else if (value == null) { + return String.format("%s eq null", fieldName); + } else { + throw new SKException("Unsupported filter value type '" + + value.getClass().getSimpleName() + "'."); + } + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchTagListContainsFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchTagListContainsFilterClause.java new file mode 100644 index 00000000..8e2ba0bf --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchTagListContainsFilterClause.java @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.azureaisearch; + +import com.microsoft.semantickernel.data.filtering.TagListContainsFilterClause; + +public class AzureAISearchTagListContainsFilterClause extends TagListContainsFilterClause { + + /** + * Initializes a new instance of the AzureAISearchTagListContainsFilterClause class. + * + * @param fieldName The field name to filter on. + * @param value The value. + */ + public AzureAISearchTagListContainsFilterClause(String fieldName, Object value) { + super(fieldName, value); + } + + /** + * Gets the filter string. + * + * @return The filter string. + */ + @Override + public String getFilter() { + return String.format("%s/any(t: t eq '%s')", getFieldName(), getValue()); + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java index 5c1ec120..e0e763c1 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java @@ -3,10 +3,9 @@ import com.azure.search.documents.indexes.SearchIndexAsyncClient; import com.azure.search.documents.indexes.models.SearchIndex; -import com.microsoft.semantickernel.data.VectorStore; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.VectorStore; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.List; @@ -16,19 +15,19 @@ public class AzureAISearchVectorStore implements VectorStore { - private final SearchIndexAsyncClient client; + private final SearchIndexAsyncClient searchIndexAsyncClient; private final AzureAISearchVectorStoreOptions options; /** * Creates a new instance of {@link AzureAISearchVectorStore}. * - * @param client The Azure AI Search client. + * @param searchIndexAsyncClient The Azure AI Search client. * @param options The options for the vector store. */ @SuppressFBWarnings("EI_EXPOSE_REP2") - public AzureAISearchVectorStore(@Nonnull SearchIndexAsyncClient client, + public AzureAISearchVectorStore(@Nonnull SearchIndexAsyncClient searchIndexAsyncClient, @Nonnull AzureAISearchVectorStoreOptions options) { - this.client = client; + this.searchIndexAsyncClient = searchIndexAsyncClient; this.options = options; } @@ -54,14 +53,14 @@ public final VectorStoreRecordCollection getCollectio return (VectorStoreRecordCollection) this.options .getVectorStoreRecordCollectionFactory() .createVectorStoreRecordCollection( - client, + searchIndexAsyncClient, collectionName, options.getRecordClass(), options.getRecordDefinition()); } return (VectorStoreRecordCollection) new AzureAISearchVectorStoreRecordCollection<>( - client, + searchIndexAsyncClient, collectionName, (AzureAISearchVectorStoreRecordCollectionOptions) options); } @@ -73,7 +72,7 @@ public final VectorStoreRecordCollection getCollectio */ @Override public Mono> getCollectionNamesAsync() { - return client.listIndexes().map(SearchIndex::getName).collectList(); + return searchIndexAsyncClient.listIndexes().map(SearchIndex::getName).collectList(); } /** @@ -91,19 +90,20 @@ public static Builder builder() { public static class Builder { @Nullable - private SearchIndexAsyncClient client; + private SearchIndexAsyncClient searchIndexAsyncClient; @Nullable private AzureAISearchVectorStoreOptions options; /** - * Sets the Azure AI Search client. + * Sets the Azure AI Search searchIndexClient. * - * @param client The Azure AI Search client. + * @param searchIndexAsyncClient The Azure AI Search searchIndexClient. * @return The updated builder instance. */ @SuppressFBWarnings("EI_EXPOSE_REP2") - public Builder withClient(@Nonnull SearchIndexAsyncClient client) { - this.client = client; + public Builder withSearchIndexAsyncClient( + @Nonnull SearchIndexAsyncClient searchIndexAsyncClient) { + this.searchIndexAsyncClient = searchIndexAsyncClient; return this; } @@ -125,14 +125,14 @@ public Builder withOptions( * @return The Azure AI Search vector store. */ public AzureAISearchVectorStore build() { - if (client == null) { - throw new IllegalStateException("client is required"); + if (searchIndexAsyncClient == null) { + throw new SKException("searchIndexAsyncClient is required"); } if (options == null) { - throw new IllegalStateException("options is required"); + throw new SKException("options is required"); } - return new AzureAISearchVectorStore(client, options); + return new AzureAISearchVectorStore(searchIndexAsyncClient, options); } } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java index 7723e652..724fe185 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java @@ -10,16 +10,16 @@ import com.azure.search.documents.indexes.models.VectorSearchAlgorithmConfiguration; import com.azure.search.documents.indexes.models.VectorSearchAlgorithmMetric; import com.azure.search.documents.indexes.models.VectorSearchProfile; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.exceptions.SKException; import java.time.OffsetDateTime; import java.util.List; import javax.annotation.Nonnull; -public class AzureAISearchVectorStoreCollectionCreateMapping { +class AzureAISearchVectorStoreCollectionCreateMapping { private static String getVectorSearchProfileName(VectorStoreRecordVectorField vectorField) { return vectorField.getEffectiveStorageName() + "Profile"; @@ -83,7 +83,8 @@ public static SearchField mapDataField(VectorStoreRecordDataField dataField) { return new SearchField(dataField.getEffectiveStorageName(), getSearchFieldDataType(dataField.getFieldType())) - .setFilterable(dataField.isFilterable()); + .setFilterable(dataField.isFilterable()) + .setSearchable(dataField.isFullTextSearchable()); } public static SearchField mapVectorField(VectorStoreRecordVectorField vectorField) { @@ -103,8 +104,8 @@ public static void updateVectorSearchParameters( } algorithms.add(getAlgorithmConfig(vectorField)); - profiles.add(new VectorSearchProfile(getVectorSearchProfileName(vectorField), - getAlgorithmConfigName(vectorField))); + profiles.add(new VectorSearchProfile( + getVectorSearchProfileName(vectorField), getAlgorithmConfigName(vectorField))); } public static SearchFieldDataType getSearchFieldDataType(Class fieldType) { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java new file mode 100644 index 00000000..016568f6 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.azureaisearch; + +import com.microsoft.semantickernel.data.filtering.BasicVectorSearchFilter; +import com.microsoft.semantickernel.exceptions.SKException; + +import java.util.Map; +import java.util.stream.Collectors; + +class AzureAISearchVectorStoreCollectionSearchMapping { + public static String buildFilterString(BasicVectorSearchFilter basicVectorSearchFilter, + Map storageNames) { + if (basicVectorSearchFilter == null + || basicVectorSearchFilter.getFilterClauses().isEmpty()) { + return ""; + } + + return String.join(" and ", + basicVectorSearchFilter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof AzureAISearchEqualityFilterClause) { + AzureAISearchEqualityFilterClause azureFilterClause = (AzureAISearchEqualityFilterClause) filterClause; + // Create new instance with the storage name of the field + return new AzureAISearchEqualityFilterClause( + storageNames.get(azureFilterClause.getFieldName()), + azureFilterClause.getValue()).getFilter(); + } else if (filterClause instanceof AzureAISearchTagListContainsFilterClause) { + AzureAISearchTagListContainsFilterClause azureFilterClause = (AzureAISearchTagListContainsFilterClause) filterClause; + // Create new instance with the storage name of the field + return new AzureAISearchTagListContainsFilterClause( + storageNames.get(azureFilterClause.getFieldName()), + azureFilterClause.getValue()).getFilter(); + } else { + throw new SKException("Unsupported filter clause type '" + + filterClause.getClass().getSimpleName() + "'."); + } + }) + .collect(Collectors.toList())); + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index 84a91f2b..e4632a36 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -6,21 +6,32 @@ import com.azure.search.documents.indexes.SearchIndexAsyncClient; import com.azure.search.documents.indexes.models.SearchField; import com.azure.search.documents.indexes.models.SearchIndex; -import com.azure.search.documents.indexes.models.VectorSearch; import com.azure.search.documents.indexes.models.VectorSearchAlgorithmConfiguration; import com.azure.search.documents.indexes.models.VectorSearchProfile; import com.azure.search.documents.models.IndexDocumentsResult; import com.azure.search.documents.models.IndexingResult; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; -import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.azure.search.documents.models.SearchOptions; +import com.azure.search.documents.models.VectorQuery; +import com.azure.search.documents.models.VectorizableTextQuery; +import com.azure.search.documents.models.VectorizedQuery; +import com.microsoft.semantickernel.data.vectorsearch.VectorizableSearch; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearch; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.VectorizedSearch; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizableTextSearchQuery; +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizedSearchQuery; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.time.OffsetDateTime; @@ -32,14 +43,15 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import javax.annotation.Nonnull; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; public class AzureAISearchVectorStoreRecordCollection implements - VectorStoreRecordCollection { + VectorStoreRecordCollection, + VectorizedSearch, + VectorizableSearch { private static final HashSet> supportedKeyTypes = new HashSet<>( Collections.singletonList( @@ -65,22 +77,25 @@ public class AzureAISearchVectorStoreRecordCollection implements List.class, Collection.class)); - private final SearchIndexAsyncClient client; + private final SearchIndexAsyncClient searchIndexAsyncClient; + private final SearchAsyncClient searchAsyncClient; private final String collectionName; - private final Map clientsByIndex = new ConcurrentHashMap<>(); private final AzureAISearchVectorStoreRecordCollectionOptions options; private final VectorStoreRecordDefinition recordDefinition; // List of non-vector fields. Used to fetch only non-vector fields when vectors are not requested private final List nonVectorFields = new ArrayList<>(); + private final Map storageNames; + private final String firstVectorFieldName; @SuppressFBWarnings("EI_EXPOSE_REP2") public AzureAISearchVectorStoreRecordCollection( - @Nonnull SearchIndexAsyncClient client, + @Nonnull SearchIndexAsyncClient searchIndexAsyncClient, @Nonnull String collectionName, @Nonnull AzureAISearchVectorStoreRecordCollectionOptions options) { - this.client = client; + this.searchIndexAsyncClient = searchIndexAsyncClient; this.collectionName = collectionName; + this.searchAsyncClient = searchIndexAsyncClient.getSearchAsyncClient(collectionName); this.options = options; // If record definition is not provided, create one from the record class @@ -104,6 +119,10 @@ public AzureAISearchVectorStoreRecordCollection( nonVectorFields.addAll(this.recordDefinition.getDataFields().stream() .map(VectorStoreRecordDataField::getEffectiveStorageName) .collect(Collectors.toList())); + + storageNames = recordDefinition.getStorageNames(); + firstVectorFieldName = recordDefinition.getVectorFields().isEmpty() ? null + : recordDefinition.getVectorFields().get(0).getName(); } @Override @@ -112,7 +131,8 @@ public String getCollectionName() { } private Mono> getIndexesAsync() { - return client.listIndexes().map(SearchIndex::getName).collect(Collectors.toList()); + return searchIndexAsyncClient.listIndexes().map(SearchIndex::getName) + .collect(Collectors.toList()); } @Override @@ -145,11 +165,11 @@ public Mono> createCollectionAsync() SearchIndex newIndex = new SearchIndex(collectionName) .setFields(searchFields) - .setVectorSearch(new VectorSearch() + .setVectorSearch(new com.azure.search.documents.indexes.models.VectorSearch() .setAlgorithms(algorithms) .setProfiles(profiles)); - return client.createIndex(newIndex).then(Mono.just(this)); + return searchIndexAsyncClient.createIndex(newIndex).then(Mono.just(this)); } @Override @@ -166,14 +186,12 @@ public Mono> createCollectionIfNotEx @Override public Mono deleteCollectionAsync() { - return client.deleteIndex(this.collectionName).then(); + return searchIndexAsyncClient.deleteIndex(this.collectionName).then(); } @Override public Mono getAsync( @Nonnull String key, GetRecordOptions options) { - SearchAsyncClient client = this.getSearchClient(this.collectionName); - // If vectors are not requested, only fetch non-vector fields List selectedFields = null; if (options != null && !options.includeVectors()) { @@ -185,16 +203,22 @@ public Mono getAsync( // Use custom mapper if available if (mapper != null && mapper.getStorageModelToRecordMapper() != null) { - return client.getDocument(key, SearchDocument.class) + return searchAsyncClient.getDocument(key, SearchDocument.class) .map(this.options.getVectorStoreRecordMapper()::mapStorageModeltoRecord); } - return client.getDocumentWithResponse(key, this.options.getRecordClass(), selectedFields) + return searchAsyncClient + .getDocumentWithResponse(key, this.options.getRecordClass(), selectedFields) .flatMap(response -> { + int statusCode = response.getStatusCode(); + if (statusCode >= 200 && statusCode < 300) { + return Mono.just(response.getValue()); + } if (response.getStatusCode() == 404) { return Mono.error(new SKException("Record not found: " + key)); } - return Mono.just(response.getValue()); + return Mono.error(new SKException("Failed to get record: " + key + ". Status code: " + + statusCode)); }); } @@ -221,7 +245,6 @@ public Mono> upsertBatchAsync( if (records.isEmpty()) { return Mono.just(Collections.emptyList()); } - SearchAsyncClient client = this.getSearchClient(this.collectionName); VectorStoreRecordMapper mapper = this.options .getVectorStoreRecordMapper(); @@ -236,7 +259,7 @@ public Mono> upsertBatchAsync( documents = records; } - return client.uploadDocuments(documents) + return searchAsyncClient.uploadDocuments(documents) .map(IndexDocumentsResult::getResults) .map( results -> results.stream() @@ -251,24 +274,112 @@ public Mono deleteAsync(String key, DeleteRecordOptions options) { @Override public Mono deleteBatchAsync(List keys, DeleteRecordOptions options) { - SearchAsyncClient client = this.getSearchClient(this.collectionName); - - return client.deleteDocuments(keys.stream().map(key -> { + return searchAsyncClient.deleteDocuments(keys.stream().map(key -> { SearchDocument document = new SearchDocument(); document.put(this.recordDefinition.getKeyField().getEffectiveStorageName(), key); return document; }).collect(Collectors.toList())).then(); } + private Mono>> searchAndMapAsync(SearchOptions searchOptions) { + VectorStoreRecordMapper mapper = this.options + .getVectorStoreRecordMapper(); + + return this.searchAsyncClient.search(null, searchOptions) + .flatMap(response -> { + Record record; + + // Use custom mapper if available + if (mapper != null && mapper.getStorageModelToRecordMapper() != null) { + record = mapper + .mapStorageModeltoRecord(response.getDocument(SearchDocument.class)); + } else { + record = response.getDocument(this.options.getRecordClass()); + } + + return Mono.just(new VectorSearchResult<>(record, response.getScore())); + }).collectList(); + } + /** - * Get a search client for the index specified. Note: the index might not exist, but we avoid - * checking everytime and the extra latency. + * Search the vector store for records that match the given embedding and filter. * - * @param indexName Index name - * @return Search client ready to read/write + * @param query The vector search query. + * @return A list of search results. */ - protected SearchAsyncClient getSearchClient(@Nonnull String indexName) { - return clientsByIndex.computeIfAbsent( - indexName, client::getSearchAsyncClient); + @Override + public Mono>> searchAsync(VectorSearchQuery query) { + if (firstVectorFieldName == null) { + throw new SKException("No vector fields defined. Cannot perform vector search"); + } + + VectorSearchOptions options = (VectorSearchOptions) query.getSearchOptions(); + if (options == null) { + options = VectorSearchOptions.createDefault(firstVectorFieldName); + } + + List vectorQueries = new ArrayList<>(); + + if (query instanceof VectorizedSearchQuery) { + vectorQueries.add(new VectorizedQuery(((VectorizedSearchQuery) query).getVector()) + .setFields( + storageNames + .get(options.getVectorFieldName() != null ? options.getVectorFieldName() + : firstVectorFieldName)) + .setKNearestNeighborsCount(options.getLimit())); + } else if (query instanceof VectorizableTextSearchQuery) { + vectorQueries + .add(new VectorizableTextQuery(((VectorizableTextSearchQuery) query).getQueryText()) + .setFields( + storageNames + .get(options.getVectorFieldName() != null ? options.getVectorFieldName() + : firstVectorFieldName)) + .setKNearestNeighborsCount(options.getLimit())); + } else { + throw new SKException("Unsupported query type: " + query.getQueryType()); + } + + String filter = AzureAISearchVectorStoreCollectionSearchMapping + .buildFilterString(options.getBasicVectorSearchFilter(), storageNames); + + SearchOptions searchOptions = new SearchOptions() + .setFilter(filter) + .setTop(options.getLimit()) + .setSkip(options.getOffset()) + .setScoringParameters() + .setVectorSearchOptions(new com.azure.search.documents.models.VectorSearchOptions() + .setQueries(vectorQueries)); + + if (!options.isIncludeVectors()) { + searchOptions.setSelect(nonVectorFields.toArray(new String[0])); + } + + return searchAndMapAsync(searchOptions); + } + + /** + * Vectorizable text search. This method searches for records that are similar to the given text. + * + * @param searchText The text to search with. + * @param options The options to use for the search. + * @return A list of search results. + */ + @Override + public Mono>> searchAsync(String searchText, + VectorSearchOptions options) { + return searchAsync(VectorSearchQuery.createQuery(searchText, options)); + } + + /** + * Vectorized search. This method searches for records that are similar to the given vector. + * + * @param vector The vector to search with. + * @param options The options to use for the search. + * @return A list of search results. + */ + @Override + public Mono>> searchAsync(List vector, + VectorSearchOptions options) { + return searchAsync(VectorSearchQuery.createQuery(vector, options)); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java index 41cc1633..b0acf545 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java @@ -2,7 +2,7 @@ package com.microsoft.semantickernel.connectors.data.azureaisearch; import com.azure.search.documents.indexes.SearchIndexAsyncClient; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; /** * Factory for creating Azure AI Search vector store record collections. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java index 0437bcfd..d1fad192 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java @@ -2,9 +2,9 @@ package com.microsoft.semantickernel.connectors.data.azureaisearch; import com.azure.search.documents.SearchDocument; -import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; import javax.annotation.Nonnull; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java index 3b0eb189..1591918d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java @@ -1,9 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import reactor.core.publisher.Mono; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java index fef57d65..7f2f077b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java @@ -1,13 +1,13 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.annotation.Nonnull; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index cdda7d9a..757253c8 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -2,11 +2,11 @@ package com.microsoft.semantickernel.connectors.data.jdbc; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import java.sql.ResultSet; import java.util.List; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java index 91582106..8db96c20 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -5,12 +5,12 @@ import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreRecordMapper; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.lang.reflect.Field; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java index 32deb0dc..14cdcc27 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import javax.sql.DataSource; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java index 093e3186..c9046f59 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java @@ -1,9 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; -import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java index c9398948..40e654f2 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java @@ -6,10 +6,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java index 046f9941..ee45f8be 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; -import com.microsoft.semantickernel.data.VectorStore; +import com.microsoft.semantickernel.data.vectorstorage.VectorStore; import reactor.core.publisher.Mono; public interface SQLVectorStore diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java index ff12c88b..711344fa 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import reactor.core.publisher.Mono; public interface SQLVectorStoreRecordCollection diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java index c6440bad..cd6977cd 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java @@ -6,10 +6,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreDefaultQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; -import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java index 7c0ba30e..e36104c6 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.postgres; -import com.microsoft.semantickernel.data.recorddefinition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; public enum PostgreSQLVectorDistanceFunction { L2("vector_l2_ops", "<->"), COSINE("vector_cosine_ops", "<=>"), INNER_PRODUCT("vector_ip_ops", diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java index 9983df0f..939caaed 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.postgres; -import com.microsoft.semantickernel.data.recorddefinition.IndexKind; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; public enum PostgreSQLVectorIndexKind { HNSW("hnsw"), IVFFLAT("ivfflat"); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index 80729b20..bcbcc0dc 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -6,11 +6,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreDefaultQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; -import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java index 1a074de2..5db2c112 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java @@ -6,10 +6,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.postgresql.util.PGobject; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java index 1b8079e9..1c6c5e42 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java @@ -2,13 +2,13 @@ package com.microsoft.semantickernel.connectors.data.redis; import com.fasterxml.jackson.databind.ObjectMapper; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java index 74d42544..ab062708 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java @@ -1,9 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; -import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; import javax.annotation.Nonnull; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java index dba568b7..e4f843ad 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java @@ -2,15 +2,14 @@ package com.microsoft.semantickernel.connectors.data.redis; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.exceptions.SKException; import javax.annotation.Nullable; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java index 9beef7ff..f99e2bba 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java @@ -4,13 +4,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java index 5690e424..6c35fa82 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java @@ -1,9 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; -import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; import javax.annotation.Nonnull; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java index d33f6d65..76db6862 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java @@ -4,7 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java index cd7cd27e..b9bf20b9 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java @@ -2,10 +2,9 @@ package com.microsoft.semantickernel.connectors.data.redis; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.data.VectorStore; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.VectorStore; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.ArrayList; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java index 94b213e6..7e139f14 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java @@ -1,10 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java index 3834f163..0752493c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java @@ -1,8 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import redis.clients.jedis.JedisPooled; /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java index 3b5c3efd..0cf1044a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java @@ -1,14 +1,15 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import javax.annotation.Nonnull; -import javax.annotation.Nullable; +import com.microsoft.semantickernel.data.vectorstorage.VectorStore; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.exceptions.SKException; import reactor.core.publisher.Mono; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java index 0d7c3ddd..d8b23d39 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java @@ -3,11 +3,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import java.util.Collections; import java.util.HashSet; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java index f3e50a68..64e95db6 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java @@ -1,7 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import javax.annotation.Nonnull; import javax.annotation.Nullable; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/BasicVectorSearchFilter.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/BasicVectorSearchFilter.java new file mode 100644 index 00000000..642f4589 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/BasicVectorSearchFilter.java @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.filtering; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class BasicVectorSearchFilter { + + /** + * Creates a new instance of the BasicVectorSearchFilter class. + * + * @return A new instance of the BasicVectorSearchFilter class. + */ + public static BasicVectorSearchFilter createDefault() { + return new BasicVectorSearchFilter(); + } + + private final List filterClauses; + + public BasicVectorSearchFilter() { + this(new ArrayList<>()); + } + + /** + * Creates a new instance of the BasicVectorSearchFilter class. + * + * @param filterClauses The filter clauses. + */ + public BasicVectorSearchFilter(List filterClauses) { + this.filterClauses = Collections.unmodifiableList(filterClauses); + } + + /** + * Gets the filter clauses. + * + * @return The filter clauses. + */ + public List getFilterClauses() { + return filterClauses; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private final List filterClauses = new ArrayList<>(); + + /** + * Adds an equality filter clause to the filter. + * + * @param equalityFilterClause The equality filter clause to add. + * @return The builder. + */ + public Builder equality(EqualityFilterClause equalityFilterClause) { + filterClauses.add(equalityFilterClause); + return this; + } + + /** + * Adds a tag list contains filter clause to the filter. + * + * @param tagListContainsFilterClause The tag list contains filter clause to add. + * @return The builder. + */ + public Builder tagListContains(TagListContainsFilterClause tagListContainsFilterClause) { + filterClauses.add(tagListContainsFilterClause); + return this; + } + + public BasicVectorSearchFilter build() { + return new BasicVectorSearchFilter(filterClauses); + } + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/EqualityFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/EqualityFilterClause.java new file mode 100644 index 00000000..ec6072ef --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/EqualityFilterClause.java @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.filtering; + +public abstract class EqualityFilterClause implements FilterClause { + + private final String fieldName; + private final Object value; + + public EqualityFilterClause(String fieldName, Object value) { + this.fieldName = fieldName; + this.value = value; + } + + /** + * Gets the field name to filter on. + * + * @return The field name to filter on. + */ + public String getFieldName() { + return fieldName; + } + + /** + * Gets the value. + * + * @return The value. + */ + public Object getValue() { + return value; + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/FilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/FilterClause.java new file mode 100644 index 00000000..a63e85dd --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/FilterClause.java @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.filtering; + +public interface FilterClause { + + /** + * Gets the filter string. + * + * @return The filter. + */ + String getFilter(); +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/TagListContainsFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/TagListContainsFilterClause.java new file mode 100644 index 00000000..bcf2e64f --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/TagListContainsFilterClause.java @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.filtering; + +public abstract class TagListContainsFilterClause implements FilterClause { + + private final String fieldName; + private final Object value; + + public TagListContainsFilterClause(String fieldName, Object value) { + this.fieldName = fieldName; + this.value = value; + } + + /** + * Gets the field name to filter on. + * + * @return The field name to filter on. + */ + public String getFieldName() { + return fieldName; + } + + /** + * Gets the value. + * + * @return The value. + */ + public Object getValue() { + return value; + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearch.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearch.java new file mode 100644 index 00000000..288b7768 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearch.java @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch; + +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; +import reactor.core.publisher.Mono; + +import java.util.List; + +public interface VectorSearch { + + Mono>> searchAsync(VectorSearchQuery query); +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java new file mode 100644 index 00000000..6e508669 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch; + +public class VectorSearchResult { + private final Record record; + private final double score; + + /** + * Creates a new instance of VectorSearchResult. + * + * @param record The record. + * @param score The score. + */ + public VectorSearchResult(Record record, double score) { + this.record = record; + this.score = score; + } + + /** + * Gets the record. + * + * @return The record. + */ + public Record getRecord() { + return record; + } + + /** + * Gets the score. + * + * @return The score. + */ + public double getScore() { + return score; + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableSearch.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableSearch.java new file mode 100644 index 00000000..37847b75 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableSearch.java @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch; + +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import reactor.core.publisher.Mono; + +import java.util.List; + +public interface VectorizableSearch extends VectorSearch { + /** + * Vectorizable text search. This method searches for records that are similar to the given text. + * + * @param searchText The text to search with. + * @param options The options to use for the search. + * @return A list of search results. + */ + Mono>> searchAsync(String searchText, + VectorSearchOptions options); +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java new file mode 100644 index 00000000..51ab175a --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch; + +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import reactor.core.publisher.Mono; + +import java.util.List; + +public interface VectorizedSearch extends VectorSearch { + + /** + * Vectorized search. This method searches for records that are similar to the given vector. + * + * @param vector The vector to search with. + * @param options The options to use for the search. + * @return A list of search results. + */ + Mono>> searchAsync(List vector, + VectorSearchOptions options); +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java new file mode 100644 index 00000000..f62a9362 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch.queries; + +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; + +import java.util.List; + +public class VectorSearchQuery { + private final VectorSearchQueryType queryType; + private final Object searchOptions; + + public VectorSearchQuery(VectorSearchQueryType queryType, Object searchOptions) { + this.queryType = queryType; + this.searchOptions = searchOptions; + } + + public VectorSearchQueryType getQueryType() { + return queryType; + } + + public Object getSearchOptions() { + return searchOptions; + } + + public static VectorizedSearchQuery createQuery(List vector, + VectorSearchOptions options) { + return new VectorizedSearchQuery(vector, options); + } + + public static VectorizableTextSearchQuery createQuery(String searchText, + VectorSearchOptions options) { + return new VectorizableTextSearchQuery(searchText, options); + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQueryType.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQueryType.java new file mode 100644 index 00000000..3a64ad62 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQueryType.java @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch.queries; + +public enum VectorSearchQueryType { + + VECTORIZED_SEARCH_QUERY("VectorizedSearchQuery"), VECTORIZABLE_TEXT_SEARCH_QUERY( + "VectorizableTextSearchQuery"), HYBRID_TEXT_VECTORIZED_SEARCH_QUERY( + "HybridTextVectorizedSearchQuery"), HYBRID_VECTORIZABLE_TEXT_SEARCH_QUERY( + "HybridVectorizableTextSearchQuery"); + + private final String value; + + VectorSearchQueryType(String value) { + this.value = value; + } + + public String getValue() { + return value; + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizableTextSearchQuery.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizableTextSearchQuery.java new file mode 100644 index 00000000..1df9b6ec --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizableTextSearchQuery.java @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch.queries; + +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; + +import javax.annotation.Nullable; + +public class VectorizableTextSearchQuery extends VectorSearchQuery { + + private final String queryText; + + @Nullable + private final VectorSearchOptions searchOptions; + + public VectorizableTextSearchQuery(String queryText, + @Nullable VectorSearchOptions searchOptions) { + super(VectorSearchQueryType.VECTORIZABLE_TEXT_SEARCH_QUERY, searchOptions); + this.queryText = queryText; + this.searchOptions = searchOptions; + } + + public String getQueryText() { + return queryText; + } + + @Nullable + public VectorSearchOptions getSearchOptions() { + return searchOptions; + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java new file mode 100644 index 00000000..2a33c07d --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch.queries; + +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; + +import javax.annotation.Nullable; +import java.util.Collections; +import java.util.List; + +public class VectorizedSearchQuery extends VectorSearchQuery { + + private final List vector; + @Nullable + private final VectorSearchOptions searchOptions; + + public VectorizedSearchQuery(List vector, VectorSearchOptions searchOptions) { + super(VectorSearchQueryType.VECTORIZED_SEARCH_QUERY, null); + + this.vector = Collections.unmodifiableList(vector); + this.searchOptions = searchOptions; + } + + public List getVector() { + return vector; + } + + public VectorSearchOptions getSearchOptions() { + return searchOptions; + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java similarity index 83% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStore.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java index 664bb281..26e20c15 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java @@ -1,10 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data; +package com.microsoft.semantickernel.data.vectorstorage; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import java.util.List; import javax.annotation.Nonnull; -import javax.annotation.Nullable; + import reactor.core.publisher.Mono; /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java similarity index 90% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollection.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java index 0b7319c0..7efe19bb 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java @@ -1,9 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data; +package com.microsoft.semantickernel.data.vectorstorage; -import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import java.util.List; import reactor.core.publisher.Mono; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java similarity index 76% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollectionOptions.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java index 6b128a13..5782b7b0 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data; +package com.microsoft.semantickernel.data.vectorstorage; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; public interface VectorStoreRecordCollectionOptions { /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java similarity index 97% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordMapper.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java index 100c7e6d..d77e9123 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data; +package com.microsoft.semantickernel.data.vectorstorage; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordDataAttribute.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordDataAttribute.java similarity index 74% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordDataAttribute.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordDataAttribute.java index 10efe24f..a5103105 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordDataAttribute.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordDataAttribute.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recordattributes; +package com.microsoft.semantickernel.data.vectorstorage.attributes; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; @@ -20,17 +20,12 @@ String storageName() default ""; /** - * Whether the field has a vector representation. - */ - boolean hasEmbedding() default false; - - /** - * Name of the field that contains the vector representation. + * Whether the field is filterable. */ - String embeddingFieldName() default ""; + boolean isFilterable() default false; /** - * Whether the field is filterable. + * Whether the field is full text searchable. */ - boolean isFilterable() default false; + boolean isFullTextSearchable() default false; } \ No newline at end of file diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordKeyAttribute.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKeyAttribute.java similarity index 87% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordKeyAttribute.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKeyAttribute.java index ffe2451a..8d54d60b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordKeyAttribute.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKeyAttribute.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recordattributes; +package com.microsoft.semantickernel.data.vectorstorage.attributes; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordVectorAttribute.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java similarity index 92% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordVectorAttribute.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java index 3f404652..efe56285 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordVectorAttribute.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recordattributes; +package com.microsoft.semantickernel.data.vectorstorage.attributes; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/DistanceFunction.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java similarity index 94% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/DistanceFunction.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java index 431df13c..62e6a3f6 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/DistanceFunction.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recorddefinition; +package com.microsoft.semantickernel.data.vectorstorage.definition; public enum DistanceFunction { /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/IndexKind.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java similarity index 92% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/IndexKind.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java index 78f9c7cc..08d330b0 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/IndexKind.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recorddefinition; +package com.microsoft.semantickernel.data.vectorstorage.definition; public enum IndexKind { HNSW("Hnsw"), FLAT("Flat"); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDataField.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java similarity index 53% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDataField.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java index 83ae6c39..76a47187 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDataField.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java @@ -1,14 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recorddefinition; +package com.microsoft.semantickernel.data.vectorstorage.definition; import javax.annotation.Nonnull; import javax.annotation.Nullable; public class VectorStoreRecordDataField extends VectorStoreRecordField { - private final boolean hasEmbedding; - @Nullable - private final String embeddingFieldName; private final boolean isFilterable; + private final boolean isFullTextSearchable; public static Builder builder() { return new Builder(); @@ -20,88 +18,61 @@ public static Builder builder() { * @param name the name of the field * @param storageName the storage name of the field * @param fieldType the field type - * @param hasEmbedding a value indicating whether the field has an embedding - * @param embeddingFieldName the name of the embedding * @param isFilterable a value indicating whether the field is filterable */ public VectorStoreRecordDataField( @Nonnull String name, @Nullable String storageName, @Nonnull Class fieldType, - boolean hasEmbedding, - @Nullable String embeddingFieldName, - boolean isFilterable) { + boolean isFilterable, + boolean isFullTextSearchable) { super(name, storageName, fieldType); - this.hasEmbedding = hasEmbedding; - this.embeddingFieldName = embeddingFieldName; this.isFilterable = isFilterable; + this.isFullTextSearchable = isFullTextSearchable; } /** - * Gets a value indicating whether the field has an embedding. - * - * @return a value indicating whether the field has an embedding - */ - public boolean hasEmbedding() { - return hasEmbedding; - } - - /** - * Gets the name of the embedding. + * Gets a value indicating whether the field is filterable. * - * @return the name of the embedding + * @return a value indicating whether the field is filterable */ - @Nullable - public String getEmbeddingFieldName() { - return embeddingFieldName; + public boolean isFilterable() { + return isFilterable; } /** - * Gets a value indicating whether the field is filterable. + * Gets a value indicating whether the field is full text searchable. * - * @return a value indicating whether the field is filterable + * @return a value indicating whether the field is full text searchable */ - public boolean isFilterable() { - return isFilterable; + public boolean isFullTextSearchable() { + return isFullTextSearchable; } public static class Builder extends VectorStoreRecordField.Builder { - private boolean hasEmbedding; - @Nullable - private String embeddingFieldName; private boolean isFilterable; + private boolean isFullTextSearchable; /** - * Sets a value indicating whether the field has an embedding. - * - * @param hasEmbedding a value indicating whether the field has an embedding - * @return the builder - */ - public Builder withHasEmbedding(boolean hasEmbedding) { - this.hasEmbedding = hasEmbedding; - return this; - } - - /** - * Sets the name of the embedding field. + * Sets a value indicating whether the field is filterable. * - * @param embeddingFieldName the name of the embedding + * @param isFilterable a value indicating whether the field is filterable * @return the builder */ - public Builder withEmbeddingFieldName(String embeddingFieldName) { - this.embeddingFieldName = embeddingFieldName; + public Builder isFilterable(boolean isFilterable) { + this.isFilterable = isFilterable; return this; } /** - * Sets a value indicating whether the field is filterable. + * Sets a value indicating whether the field is full text searchable. * - * @param isFilterable a value indicating whether the field is filterable + * @param isFullTextSearchable a value indicating whether the field is full text searchable * @return the builder */ - public Builder withIsFilterable(boolean isFilterable) { - this.isFilterable = isFilterable; + public Builder isFullTextSearchable(boolean isFullTextSearchable) { + this.isFullTextSearchable = isFullTextSearchable; return this; } @@ -118,18 +89,13 @@ public VectorStoreRecordDataField build() { if (fieldType == null) { throw new IllegalArgumentException("fieldType is required"); } - if (hasEmbedding && embeddingFieldName == null) { - throw new IllegalArgumentException( - "embeddingFieldName is required when hasEmbedding is true"); - } return new VectorStoreRecordDataField( name, storageName, fieldType, - hasEmbedding, - embeddingFieldName, - isFilterable); + isFilterable, + isFullTextSearchable); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java similarity index 91% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java index f17f5c74..1acf0a13 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java @@ -1,16 +1,17 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recorddefinition; +package com.microsoft.semantickernel.data.vectorstorage.definition; import com.fasterxml.jackson.annotation.JsonProperty; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -57,6 +58,12 @@ public List getNonVectorFields() { return fields; } + public Map getStorageNames() { + return getAllFields().stream() + .collect(Collectors.toMap(VectorStoreRecordField::getName, + VectorStoreRecordField::getEffectiveStorageName)); + } + private VectorStoreRecordDefinition( VectorStoreRecordKeyField keyField, List dataFields, @@ -145,9 +152,7 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) .withName(field.getName()) .withStorageName(storageName) .withFieldType(field.getType()) - .withHasEmbedding(dataAttribute.hasEmbedding()) - .withEmbeddingFieldName(dataAttribute.embeddingFieldName()) - .withIsFilterable(dataAttribute.isFilterable()) + .isFilterable(dataAttribute.isFilterable()) .build()); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordField.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java similarity index 97% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordField.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java index 6c8cd65a..9d85f2c9 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordField.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recorddefinition; +package com.microsoft.semantickernel.data.vectorstorage.definition; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordKeyField.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java similarity index 94% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordKeyField.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java index 8cbcbe00..b255e561 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordKeyField.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recorddefinition; +package com.microsoft.semantickernel.data.vectorstorage.definition; /** * Represents the key field in a record. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordVectorField.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java similarity index 98% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordVectorField.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java index c2d996aa..47f47fd4 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordVectorField.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recorddefinition; +package com.microsoft.semantickernel.data.vectorstorage.definition; import javax.annotation.Nonnull; import javax.annotation.Nullable; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordoptions/DeleteRecordOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java similarity index 66% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordoptions/DeleteRecordOptions.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java index ad68a90f..ed0dbb92 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordoptions/DeleteRecordOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recordoptions; +package com.microsoft.semantickernel.data.vectorstorage.options; /** * Options for deleting a record. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordoptions/GetRecordOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java similarity index 95% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordoptions/GetRecordOptions.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java index 6c81b5c9..bff9d6a1 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordoptions/GetRecordOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recordoptions; +package com.microsoft.semantickernel.data.vectorstorage.options; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordoptions/UpsertRecordOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java similarity index 66% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordoptions/UpsertRecordOptions.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java index 132eaf92..a0a9325b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordoptions/UpsertRecordOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recordoptions; +package com.microsoft.semantickernel.data.vectorstorage.options; /** * Options for upserting a record. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java new file mode 100644 index 00000000..cfdb6908 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java @@ -0,0 +1,141 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorstorage.options; + +import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.data.filtering.BasicVectorSearchFilter; +import com.microsoft.semantickernel.exceptions.SKException; + +import javax.annotation.Nullable; + +public class VectorSearchOptions { + + public static final int DEFAULT_RESULT_LIMIT = 3; + + /** + * Creates a new instance of the VectorSearchOptions class with default values. + * + * @param vectorFieldName The name of the vector field. + * @return A new instance of the VectorSearchOptions class with default values. + */ + public static VectorSearchOptions createDefault(String vectorFieldName) { + return VectorSearchOptions.builder() + .withVectorFieldName(vectorFieldName).build(); + } + + @Nullable + private final BasicVectorSearchFilter basicVectorSearchFilter; + @Nullable + private final String vectorFieldName; + private final int limit; + private final int offset; + private final boolean includeVectors; + + public VectorSearchOptions(BasicVectorSearchFilter basicVectorSearchFilter, + String vectorFieldName, int limit, int offset, boolean includeVectors) { + this.basicVectorSearchFilter = basicVectorSearchFilter; + this.vectorFieldName = vectorFieldName; + this.limit = Math.max(1, limit); + this.offset = Math.max(0, offset); + this.includeVectors = includeVectors; + } + + /** + * Gets the basic vector search filter. + * + * @return The basic vector search filter. + */ + @Nullable + public BasicVectorSearchFilter getBasicVectorSearchFilter() { + return basicVectorSearchFilter; + } + + /** + * Gets the name of the vector field. + * + * @return The name of the vector field. + */ + @Nullable + public String getVectorFieldName() { + return vectorFieldName; + } + + /** + * Gets the limit of the number of results to return. + * + * @return The limit of the number of results to return. + */ + public int getLimit() { + return limit; + } + + /** + * Gets the offset of the results to return. + * + * @return The offset of the results to return. + */ + public int getOffset() { + return offset; + } + + /** + * Gets a value indicating whether to include vectors in the results. + * + * @return A value indicating whether to include vectors in the results. + */ + public boolean isIncludeVectors() { + return includeVectors; + } + + /** + * Creates a new instance of the Builder class. + * + * @return A new instance of the Builder class. + */ + public static Builder builder() { + return new Builder(); + } + + public static class Builder implements SemanticKernelBuilder { + private BasicVectorSearchFilter basicVectorSearchFilter; + private String vectorFieldName; + private int limit; + private int offset; + private boolean includeVectors; + + public Builder() { + this.limit = DEFAULT_RESULT_LIMIT; + } + + public Builder withBasicVectorSearchFilter( + BasicVectorSearchFilter basicVectorSearchFilter) { + this.basicVectorSearchFilter = basicVectorSearchFilter; + return this; + } + + public Builder withVectorFieldName(String vectorFieldName) { + this.vectorFieldName = vectorFieldName; + return this; + } + + public Builder withLimit(int limit) { + this.limit = limit; + return this; + } + + public Builder withOffset(int offset) { + this.offset = offset; + return this; + } + + public Builder withIncludeVectors(boolean includeVectors) { + this.includeVectors = includeVectors; + return this; + } + + @Override + public VectorSearchOptions build() { + return new VectorSearchOptions(basicVectorSearchFilter, vectorFieldName, limit, offset, + includeVectors); + } + } +} diff --git a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java index 6e895714..9da92923 100644 --- a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java +++ b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java @@ -1,9 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; import java.util.List; @@ -14,7 +14,7 @@ public class Hotel { private final String name; @VectorStoreRecordDataAttribute private final int code; - @VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "descriptionEmbedding") + @VectorStoreRecordDataAttribute() private final String description; @VectorStoreRecordVectorAttribute(dimensions = 3) private final List descriptionEmbedding;