diff --git a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/GeminiService.java b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/GeminiService.java index d4012be1..a65e96af 100644 --- a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/GeminiService.java +++ b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/GeminiService.java @@ -6,7 +6,6 @@ import javax.annotation.Nullable; - /** * Makes a Gemini service available to the Semantic Kernel. */ diff --git a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java index e45bfa23..e226b4c6 100644 --- a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java +++ b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java @@ -50,7 +50,6 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; - /** * A chat completion service that uses the Gemini model to generate chat completions. */ @@ -66,7 +65,7 @@ public class GeminiChatCompletion extends GeminiService implements ChatCompletio public GeminiChatCompletion(VertexAI client, String modelId) { super(client, modelId); } - + /** * Create a new instance of {@link GeminiChatCompletion.Builder}. * diff --git a/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/HuggingFaceClient.java b/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/HuggingFaceClient.java index 28952880..4e8f9bb4 100644 --- a/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/HuggingFaceClient.java +++ b/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/HuggingFaceClient.java @@ -157,7 +157,7 @@ public static Builder builder() { /** * Builder for a Hugging Face client. */ - public static class Builder { + public static class Builder { @Nullable private KeyCredential key = null; diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java index c2b93c2d..d78a333d 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java @@ -2,9 +2,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; @@ -12,39 +12,43 @@ public class Hotel { @JsonProperty("hotelId") - @VectorStoreRecordKeyAttribute + @VectorStoreRecordKey private final String id; - @VectorStoreRecordDataAttribute(isFilterable = true) + @VectorStoreRecordData(isFilterable = true) private final String name; - @VectorStoreRecordDataAttribute + @VectorStoreRecordData private final int code; @JsonProperty("summary") - @VectorStoreRecordDataAttribute() + @VectorStoreRecordData() private final String description; @JsonProperty("summaryEmbedding1") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) private final List euclidean; @JsonProperty("summaryEmbedding2") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE) + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List cosineDistance; @JsonProperty("summaryEmbedding3") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.DOT_PRODUCT) + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.DOT_PRODUCT) private final List dotProduct; @JsonProperty("indexedSummaryEmbedding") - @VectorStoreRecordVectorAttribute(dimensions = 8, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) + @VectorStoreRecordVector(dimensions = 8, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) private final List indexedEuclidean; - @VectorStoreRecordDataAttribute + + @VectorStoreRecordData + private final List tags; + + @VectorStoreRecordData private double rating; public Hotel() { - this(null, null, 0, null, null, null, null, null, 0.0); + this(null, null, 0, null, null, null, null, null, 0.0, null); } @JsonCreator @@ -57,7 +61,8 @@ public Hotel( @JsonProperty("summaryEmbedding2") List cosineDistance, @JsonProperty("summaryEmbedding3") List dotProduct, @JsonProperty("indexedSummaryEmbedding") List indexedEuclidean, - @JsonProperty("rating") double rating) { + @JsonProperty("rating") double rating, + @JsonProperty("tags") List tags) { this.id = id; this.name = name; this.code = code; @@ -67,6 +72,7 @@ public Hotel( this.dotProduct = euclidean; this.indexedEuclidean = euclidean; this.rating = rating; + this.tags = tags; } public String getId() { @@ -97,6 +103,10 @@ public double getRating() { return rating; } + public List getTags() { + return tags; + } + public void setRating(double rating) { this.rating = rating; } diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java index ec63d9ec..8342f489 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -8,7 +8,6 @@ import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.connectors.data.jdbc.filter.SQLEqualToFilterClause; import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.sqlite.SQLiteVectorStoreQueryProvider; @@ -159,19 +158,19 @@ private List getHotels() { return Arrays.asList( new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), null, null, null, - 4.0), + 4.0, Arrays.asList("luxury", "city")), new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), null, null, null, - 4.0), + 4.0, Arrays.asList("luxury", "city")), new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), null, null, null, - 5.0), + 5.0, Arrays.asList("luxury", "beach")), new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), null, null, null, - 4.0), + 4.0, Arrays.asList("luxury", "city")), new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), null, null, null, - 4.0) + 4.0, Arrays.asList("luxury", "city")) ); } @@ -396,14 +395,13 @@ public void getBatchWithNoVectors(QueryProvider provider) { } private static Stream provideSearchParameters() { - return Stream.of( - Arguments.of(QueryProvider.MySQL, "euclidean"), - Arguments.of(QueryProvider.MySQL, "cosineDistance"), - Arguments.of(QueryProvider.MySQL, "dotProduct"), - Arguments.of(QueryProvider.PostgreSQL, "euclidean"), - Arguments.of(QueryProvider.PostgreSQL, "cosineDistance"), - Arguments.of(QueryProvider.PostgreSQL, "dotProduct") - ); + return Arrays.stream(QueryProvider.values()).map(provider -> + Stream.of( + Arguments.of(provider, "euclidean"), + Arguments.of(provider, "cosineDistance"), + Arguments.of(provider, "dotProduct") + ) + ).flatMap(s -> s); } @ParameterizedTest @@ -464,13 +462,13 @@ public void approximateSearch(QueryProvider provider) { assertNotNull(results); assertEquals(5, results.size()); // The third hotel should be the most similar - assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + assertEquals("id_3", results.get(0).getRecord().getId()); } @ParameterizedTest @MethodSource("provideSearchParameters") - public void searchWithFilter(QueryProvider provider, String embeddingName) { - String collectionName = "searchWithFilter"; + public void searchWithFilterEqualToFilter(QueryProvider provider, String embeddingName) { + String collectionName = "searchWithFilterEqualToFilter"; JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); @@ -482,7 +480,7 @@ public void searchWithFilter(QueryProvider provider, String embeddingName) { .withLimit(3) .withVectorSearchFilter( VectorSearchFilter.builder() - .withEqualToFilterClause(new SQLEqualToFilterClause("rating", 4.0)).build()) + .equalTo("rating", 4.0).build()) .build(); // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned @@ -491,7 +489,34 @@ public void searchWithFilter(QueryProvider provider, String embeddingName) { assertNotNull(results); assertEquals(3, results.size()); // The first hotel should be the most similar - assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); + assertEquals("id_1", results.get(0).getRecord().getId()); + } + + @ParameterizedTest + @MethodSource("provideSearchParameters") + public void searchWithAnyTagEqualToFilter(QueryProvider provider, String embeddingName) { + String collectionName = "searchWithAnyTagEqualToFilter"; + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, + collectionName); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions options = VectorSearchOptions.builder() + .withVectorFieldName(embeddingName) + .withLimit(3) + .withVectorSearchFilter( + VectorSearchFilter.builder() + .anyTagEqualTo("tags", "city").build()) + .build(); + + // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, + options).block(); + assertNotNull(results); + assertEquals(3, results.size()); + // The first hotel should be the most similar + assertEquals("id_1", results.get(0).getRecord().getId()); } // MySQL will always return the vectors as they're needed to compute the distances @@ -520,7 +545,7 @@ public void postgresSearchIncludeAndNotIncludeVectors() { assertNotNull(results); assertEquals(3, results.size()); // The third hotel should be the most similar - assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + assertEquals("id_3", results.get(0).getRecord().getId()); assertNotNull(results.get(0).getRecord().getEuclidean()); } } diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java index 81b962bb..da64efb0 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java @@ -2,9 +2,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; @@ -12,31 +12,32 @@ public class Hotel { - @VectorStoreRecordKeyAttribute + @VectorStoreRecordKey private final String id; - @VectorStoreRecordDataAttribute(isFilterable = true) + @VectorStoreRecordData(isFilterable = true) private final String name; - @VectorStoreRecordDataAttribute + @VectorStoreRecordData private final int code; @JsonProperty("summary") - @VectorStoreRecordDataAttribute() + @VectorStoreRecordData() private final String description; @JsonProperty("summaryEmbedding1") - @VectorStoreRecordVectorAttribute(dimensions = 8, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) + @VectorStoreRecordVector(dimensions = 8, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) private final List euclidean; @JsonProperty("summaryEmbedding2") - @VectorStoreRecordVectorAttribute(dimensions = 8) + @VectorStoreRecordVector(dimensions = 8) private final List cosineDistance; @JsonProperty("summaryEmbedding3") - @VectorStoreRecordVectorAttribute(dimensions = 8, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.DOT_PRODUCT) + @VectorStoreRecordVector(dimensions = 8, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.DOT_PRODUCT) private final List dotProduct; - @VectorStoreRecordDataAttribute + + @VectorStoreRecordData(isFilterable = true) private double rating; public Hotel() { diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index 8e07fdd2..a2b5e2c1 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -2,6 +2,7 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; @@ -94,6 +95,7 @@ static void setup() { fields.add(VectorStoreRecordDataField.builder() .withName("rating") .withFieldType(Double.class) + .isFilterable(true) .build()); VectorStoreRecordDefinition recordDefinition = VectorStoreRecordDefinition.fromFields(fields); @@ -440,4 +442,30 @@ public void searchWithOffSet(RecordCollectionOptions options, String embeddingNa // The first hotel should be the most similar assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId(), indexingFailureMessage); } + + @ParameterizedTest + @MethodSource("provideSearchParameters") + public void searchWithFilterEqualToFilter(RecordCollectionOptions recordCollectionOptions, String embeddingName) { + String collectionName = getCollectionName("search", recordCollectionOptions); + RedisHashSetVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(recordCollectionOptions), collectionName); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions options = VectorSearchOptions.builder() + .withVectorFieldName(embeddingName) + .withLimit(3) + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("rating", 4.0).build()) + .build(); + + // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, + options).block(); + assertNotNull(results); + assertEquals(3, results.size()); + // The first hotel should be the most similar + assertEquals("id_1", results.get(0).getRecord().getId()); + } } diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index df41e84e..393690fd 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -2,6 +2,7 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; @@ -94,6 +95,7 @@ static void setup() { fields.add(VectorStoreRecordDataField.builder() .withName("rating") .withFieldType(Double.class) + .isFilterable(true) .build()); VectorStoreRecordDefinition recordDefinition = VectorStoreRecordDefinition.fromFields(fields); @@ -440,4 +442,30 @@ public void searchWithOffSet(RecordCollectionOptions options, String embeddingNa // The first hotel should be the most similar assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId(), indexingFailureMessage); } + + @ParameterizedTest + @MethodSource("provideSearchParameters") + public void searchWithFilterEqualToFilter(RecordCollectionOptions recordCollectionOptions, String embeddingName) { + String collectionName = getCollectionName("search", recordCollectionOptions); + RedisJsonVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(recordCollectionOptions), collectionName); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions options = VectorSearchOptions.builder() + .withVectorFieldName(embeddingName) + .withLimit(3) + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("rating", 4.0).build()) + .build(); + + // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, + options).block(); + assertNotNull(results); + assertEquals(3, results.size()); + // The first hotel should be the most similar + assertEquals("id_1", results.get(0).getRecord().getId()); + } } diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java index f5d9d7f1..8dc1372d 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java @@ -11,9 +11,9 @@ import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.VolatileVectorStore; import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -39,13 +39,13 @@ public class InMemoryVolatileVectorStore { static class GitHubFile { @JsonProperty("fileId") // Set a different name for the storage field if needed - @VectorStoreRecordKeyAttribute() + @VectorStoreRecordKey() private final String id; - @VectorStoreRecordDataAttribute() + @VectorStoreRecordData() private final String description; - @VectorStoreRecordDataAttribute + @VectorStoreRecordData private final String link; - @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE) + @VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List embedding; public GitHubFile( diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java index 9de59ed7..dca8599b 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java @@ -17,9 +17,9 @@ import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import java.nio.charset.StandardCharsets; import java.util.Arrays; @@ -53,14 +53,13 @@ public class VectorStoreWithAzureAISearch { static class GitHubFile { @JsonProperty("fileId") // Set a different name for the storage field if needed - @VectorStoreRecordKeyAttribute() + @VectorStoreRecordKey() private final String id; - @VectorStoreRecordDataAttribute() - @VectorStoreRecordVectorAttribute(distanceFunction = DistanceFunction.COSINE_DISTANCE, dimensions = EMBEDDING_DIMENSIONS) + @VectorStoreRecordData() private final String description; - @VectorStoreRecordDataAttribute + @VectorStoreRecordData private final String link; - @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE) + @VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_SIMILARITY) private final List embedding; public GitHubFile() { @@ -161,8 +160,8 @@ private static Mono>> search( VectorStoreRecordCollection recordCollection, OpenAITextEmbeddingGenerationService embeddingGeneration) { // Generate embeddings for the search text and search for the closest records - return embeddingGeneration.generateEmbeddingsAsync(Collections.singletonList(searchText)) - .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); + return embeddingGeneration.generateEmbeddingAsync(searchText) + .flatMap(r -> recordCollection.searchAsync(r.getVector(), null)); } private static Mono> storeData( diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java index 6cd4953f..2099886b 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java @@ -5,7 +5,6 @@ import com.azure.ai.openai.OpenAIClientBuilder; import com.azure.core.credential.AzureKeyCredential; import com.azure.core.credential.KeyCredential; -import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStore; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions; @@ -13,9 +12,9 @@ import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.mysql.cj.jdbc.MysqlDataSource; import java.nio.charset.StandardCharsets; @@ -42,14 +41,14 @@ public class VectorStoreWithJDBC { private static final int EMBEDDING_DIMENSIONS = 1536; static class GitHubFile { - @JsonProperty("fileId") // Set a different name for the storage field if needed - @VectorStoreRecordKeyAttribute() + + @VectorStoreRecordKey() private final String id; - @VectorStoreRecordDataAttribute() + @VectorStoreRecordData() private final String description; - @VectorStoreRecordDataAttribute + @VectorStoreRecordData private final String link; - @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, distanceFunction = DistanceFunction.COSINE_DISTANCE) + @VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List embedding; public GitHubFile() { diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java index aee00a5b..0310c9dd 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java @@ -5,21 +5,17 @@ import com.azure.ai.openai.OpenAIClientBuilder; import com.azure.core.credential.AzureKeyCredential; import com.azure.core.credential.KeyCredential; -import com.azure.core.util.ClientOptions; -import com.azure.core.util.MetricsOptions; -import com.azure.core.util.TracingOptions; import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; -import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.redis.RedisStorageType; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStore; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreOptions; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import java.util.Arrays; import java.util.Collections; @@ -47,13 +43,13 @@ public class VectorStoreWithRedis { public static class GitHubFile { @JsonProperty("fileId") // Set a different name for the storage field if needed - @VectorStoreRecordKeyAttribute() + @VectorStoreRecordKey() private final String id; - @VectorStoreRecordDataAttribute() + @VectorStoreRecordData() private final String description; - @VectorStoreRecordDataAttribute + @VectorStoreRecordData private final String link; - @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE) + @VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List embedding; public GitHubFile() { diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index f042a3ef..d4b8aec4 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -62,6 +62,16 @@ com.microsoft.semantic-kernel semantickernel-aiservices-openai + + com.microsoft.semantic-kernel + semantickernel-experimental + + + com.mysql + mysql-connector-j + 9.0.0 + compile + diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java index 98e310fd..07390692 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java @@ -66,7 +66,8 @@ public static SKCheckedException build( Throwable wrappedCause = cause.getCause(); - if ((cause instanceof SKCheckedException || cause instanceof SKException) && wrappedCause != null) { + if ((cause instanceof SKCheckedException || cause instanceof SKException) + && wrappedCause != null) { return new SKCheckedException(message, wrappedCause); } else { return new SKCheckedException(message, cause); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java index 45ea84b8..3f14a2f5 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java @@ -65,7 +65,8 @@ public static SKException build( Throwable wrappedCause = cause.getCause(); - if ((cause instanceof SKCheckedException || cause instanceof SKException) && wrappedCause != null) { + if ((cause instanceof SKCheckedException || cause instanceof SKException) + && wrappedCause != null) { return new SKException(message, wrappedCause); } else { return new SKException(message, cause); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PreToolCallEvent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PreToolCallEvent.java index c728a917..c991efca 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PreToolCallEvent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PreToolCallEvent.java @@ -47,7 +47,7 @@ public PreToolCallEvent( public KernelFunctionArguments getArguments() { return arguments; } - + /** * Get the tool call function. * @return The tool call function. diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseFormat.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseFormat.java index c4caf6c4..9ebcbd66 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseFormat.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseFormat.java @@ -18,8 +18,8 @@ "TEXT" }), @JsonSubTypes.Type(value = JsonObjectResponseFormat.class, name = "json_object", names = { "json_object", "JSON_OBJECT" }), - - }) + +}) public abstract class ResponseFormat { /** diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseSchemaGenerator.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseSchemaGenerator.java index 93d933a1..676f2872 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseSchemaGenerator.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseSchemaGenerator.java @@ -30,15 +30,15 @@ public interface ResponseSchemaGenerator { public static ResponseSchemaGenerator jacksonGenerator() { try { return loadGenerator( - "com.microsoft.semantickernel.aiservices.openai.chatcompletion.responseformat.JacksonResponseFormatGenerator"); + "com.microsoft.semantickernel.aiservices.openai.chatcompletion.responseformat.JacksonResponseFormatGenerator"); } catch (NoClassDefFoundError e) { LOGGER.error( - "The Jackson response schema generator relies on the optional dependencies 'com.github.victools:jsonschema-generator', and 'com.github.victools:jsonschema-module-jackson'. To use this feature, please add this dependency to your project."); + "The Jackson response schema generator relies on the optional dependencies 'com.github.victools:jsonschema-generator', and 'com.github.victools:jsonschema-module-jackson'. To use this feature, please add this dependency to your project."); throw new SKException( - "The Jackson response schema generator relies on the optional dependency 'com.github.victools:jsonschema-generator', and 'com.github.victools:jsonschema-module-jackson'. To use this feature, please add this dependency to your project."); + "The Jackson response schema generator relies on the optional dependency 'com.github.victools:jsonschema-generator', and 'com.github.victools:jsonschema-module-jackson'. To use this feature, please add this dependency to your project."); } } - + /** * Load a response schema generator based on the given class name. * The class must implement the {@link ResponseSchemaGenerator} interface. diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java index a760194a..680d93c0 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java @@ -156,7 +156,7 @@ private static MethodDetails getMethodDetails( * @return the function representing the method */ @SuppressWarnings("unchecked") - public static ImplementationFunc getFunction(Method method, Object instance) { + public static ImplementationFunc getFunction(Method method, Object instance) { return (kernel, function, arguments, variableType, invocationContext) -> { InvocationContext context; if (invocationContext == null) { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java index 5aeb5ef3..e1e64048 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java @@ -29,9 +29,9 @@ public class AzureAISearchVectorStore implements VectorStore { */ @SuppressFBWarnings("EI_EXPOSE_REP2") public AzureAISearchVectorStore(@Nonnull SearchIndexAsyncClient searchIndexAsyncClient, - @Nonnull AzureAISearchVectorStoreOptions options) { + @Nullable AzureAISearchVectorStoreOptions options) { this.searchIndexAsyncClient = searchIndexAsyncClient; - this.options = options; + this.options = options == null ? new AzureAISearchVectorStoreOptions() : options; } /** @@ -131,9 +131,6 @@ public AzureAISearchVectorStore build() { if (searchIndexAsyncClient == null) { throw new SKException("searchIndexAsyncClient is required"); } - if (options == null) { - throw new SKException("options is required"); - } return new AzureAISearchVectorStore(searchIndexAsyncClient, options); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java index eb3a5ff2..4abbe1fb 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java @@ -1,45 +1,92 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.azureaisearch; -import com.microsoft.semantickernel.connectors.data.azureaisearch.filter.AzureAISearchEqualToFilterClause; -import com.microsoft.semantickernel.connectors.data.azureaisearch.filter.AzureAISearchAnyTagEqualToFilterClause; +import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; +import com.microsoft.semantickernel.data.filter.EqualToFilterClause; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.filter.FilterMapping; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; import java.util.stream.Collectors; -class AzureAISearchVectorStoreCollectionSearchMapping { - public static String buildFilterString(VectorSearchFilter vectorSearchFilter, +class AzureAISearchVectorStoreCollectionSearchMapping + implements FilterMapping { + + private AzureAISearchVectorStoreCollectionSearchMapping() { + } + + private static class AzureAISearchVectorStoreCollectionSearchMappingHolder { + private static final AzureAISearchVectorStoreCollectionSearchMapping INSTANCE = new AzureAISearchVectorStoreCollectionSearchMapping(); + } + + static AzureAISearchVectorStoreCollectionSearchMapping getInstance() { + return AzureAISearchVectorStoreCollectionSearchMappingHolder.INSTANCE; + } + + public String getFilter(VectorSearchFilter vectorSearchFilter, VectorStoreRecordDefinition recordDefinition) { if (vectorSearchFilter == null || vectorSearchFilter.getFilterClauses().isEmpty()) { return ""; } - return String.join(" and ", - vectorSearchFilter.getFilterClauses().stream().map(filterClause -> { - if (filterClause instanceof AzureAISearchEqualToFilterClause) { - AzureAISearchEqualToFilterClause azureFilterClause = (AzureAISearchEqualToFilterClause) filterClause; - // Create new instance with the storage name of the field - return new AzureAISearchEqualToFilterClause( - recordDefinition.getField(azureFilterClause.getFieldName()) - .getEffectiveStorageName(), - azureFilterClause.getValue()) - .getFilter(); - } else if (filterClause instanceof AzureAISearchAnyTagEqualToFilterClause) { - AzureAISearchAnyTagEqualToFilterClause azureFilterClause = (AzureAISearchAnyTagEqualToFilterClause) filterClause; - // Create new instance with the storage name of the field - return new AzureAISearchAnyTagEqualToFilterClause( - recordDefinition.getField(azureFilterClause.getFieldName()) - .getEffectiveStorageName(), - azureFilterClause.getValue()) - .getFilter(); - } else { - throw new SKException("Unsupported filter clause type '" - + filterClause.getClass().getSimpleName() + "'."); - } - }) - .collect(Collectors.toList())); + return vectorSearchFilter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof EqualToFilterClause) { + EqualToFilterClause equalToFilterClause = (EqualToFilterClause) filterClause; + // Create new instance with the storage name of the field + return getEqualToFilter(new EqualToFilterClause( + recordDefinition.getField(equalToFilterClause.getFieldName()) + .getEffectiveStorageName(), + equalToFilterClause.getValue())); + } else if (filterClause instanceof AnyTagEqualToFilterClause) { + AnyTagEqualToFilterClause anyTagEqualToFilterClause = (AnyTagEqualToFilterClause) filterClause; + // Create new instance with the storage name of the field + return getAnyTagEqualToFilter(new AnyTagEqualToFilterClause( + recordDefinition.getField(anyTagEqualToFilterClause.getFieldName()) + .getEffectiveStorageName(), + anyTagEqualToFilterClause.getValue())); + } else { + throw new SKException("Unsupported filter clause type '" + + filterClause.getClass().getSimpleName() + "'."); + } + }).collect(Collectors.joining(" and ")); + } + + @Override + public String getEqualToFilter(EqualToFilterClause filterClause) { + String fieldName = filterClause.getFieldName(); + Object value = filterClause.getValue(); + + if (value instanceof String) { + return String.format("%s eq '%s'", fieldName, value); + } else if (value instanceof Boolean) { + return String.format("%s eq %s", fieldName, + value.toString().toLowerCase()); + } else if (value instanceof Integer) { + return String.format("%s eq %d", fieldName, (Integer) value); + } else if (value instanceof Long) { + return String.format("%s eq %d", fieldName, (Long) value); + } else if (value instanceof Float) { + return String.format("%s eq %f", fieldName, (Float) value); + } else if (value instanceof Double) { + return String.format("%s eq %f", fieldName, (Double) value); + } else if (value instanceof OffsetDateTime) { + return String.format("%s eq %s", fieldName, ((OffsetDateTime) value) + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + } else if (value == null) { + return String.format("%s eq null", fieldName); + } else { + throw new SKException("Unsupported filter value type '" + + value.getClass().getSimpleName() + "'."); + } + } + + @Override + public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { + return String.format("%s/any(t: t eq '%s')", filterClause.getFieldName(), + filterClause.getValue()); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index 622bc6d0..02971935 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -70,7 +70,8 @@ public class AzureAISearchVectorStoreRecordCollection implements double.class, Boolean.class, boolean.class, - OffsetDateTime.class)); + OffsetDateTime.class, + List.class)); private static final HashSet> supportedVectorTypes = new HashSet<>( Arrays.asList( @@ -290,8 +291,8 @@ private Mono>> searchAndMapAsync( List vectorQueries, VectorSearchOptions options, GetRecordOptions getRecordOptions) { - String filter = AzureAISearchVectorStoreCollectionSearchMapping - .buildFilterString(options.getVectorSearchFilter(), recordDefinition); + String filter = AzureAISearchVectorStoreCollectionSearchMapping.getInstance() + .getFilter(options.getVectorSearchFilter(), recordDefinition); SearchOptions searchOptions = new SearchOptions() .setFilter(filter) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchAnyTagEqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchAnyTagEqualToFilterClause.java deleted file mode 100644 index 81a4bb72..00000000 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchAnyTagEqualToFilterClause.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.azureaisearch.filter; - -import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; - -/** - * A filter clause that filters on any tag equal to a value. - */ -public class AzureAISearchAnyTagEqualToFilterClause extends AnyTagEqualToFilterClause { - - /** - * Initializes a new instance of the AzureAISearchTagListContainsFilterClause class. - * - * @param fieldName The field name to filter on. - * @param value The value to filter on. - */ - public AzureAISearchAnyTagEqualToFilterClause(String fieldName, Object value) { - super(fieldName, value); - } - - /** - * Gets the filter string. - * - * @return The filter string. - */ - @Override - public String getFilter() { - return String.format("%s/any(t: t eq '%s')", getFieldName(), getValue()); - } -} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchEqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchEqualToFilterClause.java deleted file mode 100644 index aa03b532..00000000 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchEqualToFilterClause.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.azureaisearch.filter; - -import com.microsoft.semantickernel.data.filter.EqualToFilterClause; -import com.microsoft.semantickernel.exceptions.SKException; - -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; - -/** - * A filter clause that filters on equality. - */ -public class AzureAISearchEqualToFilterClause extends EqualToFilterClause { - - /** - * Initializes a new instance of the AzureAISearchEqualityFilterClause class. - * - * @param fieldName The field name to filter on. - * @param value The value to filter on. - */ - public AzureAISearchEqualToFilterClause(String fieldName, Object value) { - super(fieldName, value); - } - - /** - * Gets the filter string. - * - * @return The filter string. - */ - @Override - public String getFilter() { - String fieldName = getFieldName(); - Object value = getValue(); - - if (value instanceof String) { - return String.format("%s eq '%s'", fieldName, value); - } else if (value instanceof Boolean) { - return String.format("%s eq %s", fieldName, - value.toString().toLowerCase()); - } else if (value instanceof Integer) { - return String.format("%s eq %d", fieldName, (Integer) value); - } else if (value instanceof Long) { - return String.format("%s eq %d", fieldName, (Long) value); - } else if (value instanceof Float) { - return String.format("%s eq %f", fieldName, (Float) value); - } else if (value instanceof Double) { - return String.format("%s eq %f", fieldName, (Double) value); - } else if (value instanceof OffsetDateTime) { - return String.format("%s eq %s", fieldName, ((OffsetDateTime) value) - .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); - } else if (value == null) { - return String.format("%s eq null", fieldName); - } else { - throw new SKException("Unsupported filter value type '" - + value.getClass().getSimpleName() + "'."); - } - } -} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/hsqldb/HSQLDBVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/hsqldb/HSQLDBVectorStoreQueryProvider.java index 86f7ec6e..730458a5 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/hsqldb/HSQLDBVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/hsqldb/HSQLDBVectorStoreQueryProvider.java @@ -2,8 +2,10 @@ package com.microsoft.semantickernel.connectors.data.hsqldb; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; @@ -29,12 +31,15 @@ */ public class HSQLDBVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider { + private final ObjectMapper objectMapper; + @SuppressFBWarnings("EI_EXPOSE_REP2") private HSQLDBVectorStoreQueryProvider( DataSource dataSource, String collectionsTable, String prefixForCollectionTables, - int defaultVarCharLength) { + int defaultVarCharLength, + ObjectMapper objectMapper) { super( dataSource, collectionsTable, @@ -42,6 +47,7 @@ private HSQLDBVectorStoreQueryProvider( buildSupportedKeyTypes(defaultVarCharLength), buildSupportedDataTypes(defaultVarCharLength), buildSupportedVectorTypes(defaultVarCharLength)); + this.objectMapper = objectMapper; } private static Map, String> buildSupportedVectorTypes(int defaultVarCharLength) { @@ -66,6 +72,7 @@ private static Map, String> buildSupportedDataTypes(int defaultVarCharL supportedDataTypes.put(Boolean.class, "BOOLEAN"); supportedDataTypes.put(boolean.class, "BOOLEAN"); supportedDataTypes.put(OffsetDateTime.class, "TIMESTAMPTZ"); + supportedDataTypes.put(List.class, "TEXT"); return supportedDataTypes; } @@ -75,34 +82,32 @@ private static HashMap, String> buildSupportedKeyTypes(int defaultVarCh return supportedKeyTypes; } - private void setStatementValues(PreparedStatement statement, Object record, + private void setUpsertStatementValues(PreparedStatement statement, Object record, List fields) { + JsonNode jsonNode = objectMapper.valueToTree(record); + for (int i = 0; i < fields.size(); ++i) { VectorStoreRecordField field = fields.get(i); try { - Field recordField = record.getClass().getDeclaredField(field.getName()); - recordField.setAccessible(true); - Object value = recordField.get(record); - - if (field instanceof VectorStoreRecordKeyField) { - statement.setObject(i + 1, (String) value); - } else if (field instanceof VectorStoreRecordVectorField) { - Class vectorType = record.getClass().getDeclaredField(field.getName()) - .getType(); - - // If the vector field is other than String, serialize it to JSON - if (vectorType.equals(String.class)) { - statement.setObject(i + 1, value); - } else { - // Serialize the vector to JSON - statement.setObject(i + 1, new ObjectMapper().writeValueAsString(value)); + JsonNode valueNode = jsonNode.get(field.getEffectiveStorageName()); + + if (field instanceof VectorStoreRecordVectorField) { + // Convert the vector field to a string + if (!field.getFieldType().equals(String.class)) { + statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); + continue; + } + } else if (field instanceof VectorStoreRecordDataField) { + // Convert List field to a string + if (field.getFieldType().equals(List.class)) { + statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); + continue; } - } else { - statement.setObject(i + 1, value); } - } catch (NoSuchFieldException | IllegalAccessException | SQLException e) { - throw new SKException("Failed to set statement values", e); - } catch (JsonProcessingException e) { + + statement.setObject(i + 1, + objectMapper.convertValue(valueNode, field.getFieldType())); + } catch (SQLException | JsonProcessingException e) { throw new RuntimeException(e); } } @@ -155,7 +160,7 @@ public void upsertRecords(String collectionName, List records, try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(query)) { for (Object record : records) { - setStatementValues(statement, record, recordDefinition.getAllFields()); + setUpsertStatementValues(statement, record, recordDefinition.getAllFields()); statement.addBatch(); } @@ -184,6 +189,7 @@ public static class Builder private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; private int defaultVarCharLength = 255; + private ObjectMapper objectMapper = new ObjectMapper(); /** * Sets the data source. @@ -230,6 +236,18 @@ public Builder setDefaultVarCharLength(int defaultVarCharLength) { return this; } + /** + * Sets the object mapper. + * + * @param objectMapper the object mapper + * @return the builder + */ + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withObjectMapper(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + return this; + } + /** * Builds the HSQLDB vector store query provider. * @@ -244,7 +262,8 @@ public HSQLDBVectorStoreQueryProvider build() { dataSource, collectionsTable, prefixForCollectionTables, - defaultVarCharLength); + defaultVarCharLength, + objectMapper); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index 7970ffad..f33374aa 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -1,7 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; +import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; +import com.microsoft.semantickernel.data.filter.EqualToFilterClause; import com.microsoft.semantickernel.data.vectorsearch.VectorOperations; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; @@ -37,14 +40,16 @@ * A JDBC vector store query provider. */ public class JDBCVectorStoreQueryProvider - implements SQLVectorStoreQueryProvider { + implements SQLVectorStoreQueryProvider, + SQLVectorStoreFilterQueryProvider { private static final Logger LOGGER = LoggerFactory .getLogger(JDBCVectorStoreQueryProvider.class); - private final Map, String> supportedKeyTypes; - private final Map, String> supportedDataTypes; - private final Map, String> supportedVectorTypes; + protected final Map, String> supportedKeyTypes; + protected final Map, String> supportedDataTypes; + protected final Map, String> supportedVectorTypes; + protected final DataSource dataSource; private final String collectionsTable; private final String prefixForCollectionTables; @@ -74,6 +79,7 @@ protected JDBCVectorStoreQueryProvider( supportedDataTypes.put(Boolean.class, "BOOLEAN"); supportedDataTypes.put(boolean.class, "BOOLEAN"); supportedDataTypes.put(OffsetDateTime.class, "TIMESTAMPTZ"); + supportedDataTypes.put(List.class, "TEXT"); supportedVectorTypes = new HashMap<>(); supportedVectorTypes.put(String.class, "TEXT"); @@ -539,10 +545,8 @@ public List> search(String collectionName, : (VectorStoreRecordVectorField) recordDefinition .getField(options.getVectorFieldName()); - String filter = SQLVectorStoreRecordCollectionSearchMapping - .buildFilter(options.getVectorSearchFilter(), recordDefinition); - List parameters = SQLVectorStoreRecordCollectionSearchMapping - .getFilterParameters(options.getVectorSearchFilter()); + String filter = getFilter(options.getVectorSearchFilter(), recordDefinition); + List parameters = getFilterParameters(options.getVectorSearchFilter()); List records = getRecordsWithFilter(collectionName, recordDefinition, mapper, new GetRecordOptions(true), filter, parameters); @@ -581,6 +585,103 @@ public String formatQuery(String query, String... args) { return String.format(query, (Object[]) args); } + /** + * Gets the filter query string for the given vector search filter and record definition. + * + * @param filter The filter to get the filter string for. + * @param recordDefinition The record definition to get the filter string for. + * @return The filter string. + */ + @Override + public String getFilter(VectorSearchFilter filter, + VectorStoreRecordDefinition recordDefinition) { + if (filter == null + || filter.getFilterClauses().isEmpty()) { + return ""; + } + + return filter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof EqualToFilterClause) { + EqualToFilterClause equalToFilterClause = (EqualToFilterClause) filterClause; + return getEqualToFilter(new EqualToFilterClause( + recordDefinition.getField(equalToFilterClause.getFieldName()) + .getEffectiveStorageName(), + equalToFilterClause.getValue())); + } else if (filterClause instanceof AnyTagEqualToFilterClause) { + AnyTagEqualToFilterClause anyTagEqualToFilterClause = (AnyTagEqualToFilterClause) filterClause; + return getAnyTagEqualToFilter(new AnyTagEqualToFilterClause( + recordDefinition.getField(anyTagEqualToFilterClause.getFieldName()) + .getEffectiveStorageName(), + anyTagEqualToFilterClause.getValue())); + } else { + throw new SKException("Unsupported filter clause type '" + + filterClause.getClass().getSimpleName() + "'."); + } + }).collect(Collectors.joining(" AND ")); + } + + /** + * Gets the filter parameters for the given vector search filter to associate with the filter string + * generated by the getFilter method. + * + * @param filter The filter to get the filter parameters for. + * @return The filter parameters. + */ + @Override + public List getFilterParameters(VectorSearchFilter filter) { + if (filter == null + || filter.getFilterClauses().isEmpty()) { + return Collections.emptyList(); + } + + return filter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof EqualToFilterClause) { + EqualToFilterClause equalToFilterClause = (EqualToFilterClause) filterClause; + return equalToFilterClause.getValue(); + } else if (filterClause instanceof AnyTagEqualToFilterClause) { + AnyTagEqualToFilterClause anyTagEqualToFilterClause = (AnyTagEqualToFilterClause) filterClause; + return String.format("%%\"%s\"%%", anyTagEqualToFilterClause.getValue()); + } else { + throw new SKException("Unsupported filter clause type '" + + filterClause.getClass().getSimpleName() + "'."); + } + }).collect(Collectors.toList()); + } + + @Override + public String getEqualToFilter(EqualToFilterClause filterClause) { + String fieldName = JDBCVectorStoreQueryProvider + .validateSQLidentifier(filterClause.getFieldName()); + Object value = filterClause.getValue(); + + if (value instanceof String) { + return String.format("%s = ?", fieldName); + } else if (value instanceof Boolean) { + return String.format("%s = ?", fieldName); + } else if (value instanceof Integer) { + return String.format("%s = ?", fieldName); + } else if (value instanceof Long) { + return String.format("%s = ?", fieldName); + } else if (value instanceof Float) { + return String.format("%s = ?", fieldName); + } else if (value instanceof Double) { + return String.format("%s = ?", fieldName); + } else if (value instanceof OffsetDateTime) { + return String.format("%s = ?", fieldName); + } else { + throw new SKException("Unsupported filter value type '" + + value.getClass().getSimpleName() + "'."); + } + } + + @Override + public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { + String fieldName = JDBCVectorStoreQueryProvider + .validateSQLidentifier(filterClause.getFieldName()); + + return String.format("%s LIKE ?", fieldName); + } + /** * The builder for {@link JDBCVectorStoreQueryProvider}. */ diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java index c7f839c2..31b25bef 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -320,7 +320,7 @@ public Mono>> searchAsync(List vector, vectorStoreRecordMapper)) .subscribeOn(Schedulers.boundedElastic()); } - + /** * Builder for a JDBCVectorStoreRecordCollection. * @param the type of the records in the collection diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java index 35fafedd..880d93df 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; @@ -123,7 +124,6 @@ public JDBCVectorStoreRecordMapper build() { // Select fields from the record definition. List fields; - ResultSetMetaData metaData = resultSet.getMetaData(); if (options != null && options.isIncludeVectors()) { fields = vectorStoreRecordDefinition.getAllFields(); } else { @@ -132,13 +132,17 @@ public JDBCVectorStoreRecordMapper build() { for (VectorStoreRecordField field : fields) { Object value = resultSet.getObject(field.getEffectiveStorageName()); + Class fieldType = field.getFieldType(); if (field instanceof VectorStoreRecordVectorField) { - Class vectorType = field.getFieldType(); - // If the vector field is other than String, deserialize it from the JSON string - if (!vectorType.equals(String.class)) { - value = objectMapper.readValue((String) value, vectorType); + if (!fieldType.equals(String.class)) { + value = objectMapper.readValue((String) value, fieldType); + } + } else if (field instanceof VectorStoreRecordDataField) { + // If the field is List, deserialize it from the JSON string + if (fieldType.equals(List.class)) { + value = objectMapper.readValue((String) value, fieldType); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreFilterQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreFilterQueryProvider.java new file mode 100644 index 00000000..330df60d --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreFilterQueryProvider.java @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.jdbc; + +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.filter.FilterMapping; + +import java.util.List; + +public interface SQLVectorStoreFilterQueryProvider extends FilterMapping { + /** + * Gets the filter parameters for the given vector search filter to associate with the filter string + * generated by the getFilter method. + * + * @param filter The filter to get the filter parameters for. + * @return The filter parameters. + */ + List getFilterParameters(VectorSearchFilter filter); +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java index d33a686f..747e3c84 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java @@ -2,6 +2,7 @@ package com.microsoft.semantickernel.connectors.data.jdbc; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.data.filter.FilterClause; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollectionSearchMapping.java deleted file mode 100644 index 81408b8e..00000000 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollectionSearchMapping.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.jdbc; - -import com.microsoft.semantickernel.connectors.data.jdbc.filter.SQLEqualToFilterClause; -import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; -import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.exceptions.SKException; - -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - -/** - * A mapping for searching a collection of vector records in SQL. - */ -public class SQLVectorStoreRecordCollectionSearchMapping { - - /** - * Builds a filter for searching a collection of vector records in SQL. - * - * @param vectorSearchFilter the search filter - * @param recordDefinition the record definition - * @return the filter - */ - public static String buildFilter(VectorSearchFilter vectorSearchFilter, - VectorStoreRecordDefinition recordDefinition) { - if (vectorSearchFilter == null - || vectorSearchFilter.getFilterClauses().isEmpty()) { - return ""; - } - - return vectorSearchFilter.getFilterClauses().stream().map(filterClause -> { - if (filterClause instanceof SQLEqualToFilterClause) { - SQLEqualToFilterClause equalityFilterClause = (SQLEqualToFilterClause) filterClause; - // Create new instance with the storage name of the field - return new SQLEqualToFilterClause( - recordDefinition.getField(equalityFilterClause.getFieldName()) - .getEffectiveStorageName(), - equalityFilterClause.getValue()).getFilter(); - } else { - throw new SKException("Unsupported filter clause type '" - + filterClause.getClass().getSimpleName() + "'."); - } - }).collect(Collectors.joining(" AND ")); - } - - /** - * Gets the filter parameters. - * - * @param vectorSearchFilter the search filter - * @return the filter parameters - */ - public static List getFilterParameters(VectorSearchFilter vectorSearchFilter) { - if (vectorSearchFilter == null - || vectorSearchFilter.getFilterClauses().isEmpty()) { - return Collections.emptyList(); - } - - return vectorSearchFilter.getFilterClauses().stream().map(filterClause -> { - if (filterClause instanceof SQLEqualToFilterClause) { - SQLEqualToFilterClause equalityFilterClause = (SQLEqualToFilterClause) filterClause; - return equalityFilterClause.getValue(); - } else { - throw new SKException("Unsupported filter clause type '" - + filterClause.getClass().getSimpleName() + "'."); - } - }).collect(Collectors.toList()); - } -} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/filter/SQLEqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/filter/SQLEqualToFilterClause.java deleted file mode 100644 index 488c1473..00000000 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/filter/SQLEqualToFilterClause.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.jdbc.filter; - -import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.filter.EqualToFilterClause; -import com.microsoft.semantickernel.exceptions.SKException; - -import java.time.OffsetDateTime; - -/** - * Represents an equality filter clause for SQL. - */ -public class SQLEqualToFilterClause extends EqualToFilterClause { - - /** - * Initializes a new instance of the SQLEqualityFilterClause class. - * - * @param fieldName The field name to filter on. - * @param value The value. - */ - public SQLEqualToFilterClause(String fieldName, Object value) { - super(fieldName, value); - } - - /** - * Gets the filter string. - * - * @return The filter string. - */ - @Override - public String getFilter() { - String fieldName = JDBCVectorStoreQueryProvider.validateSQLidentifier(getFieldName()); - Object value = getValue(); - - if (value instanceof String) { - return String.format("%s = ?", fieldName); - } else if (value instanceof Boolean) { - return String.format("%s = ?", fieldName); - } else if (value instanceof Integer) { - return String.format("%s = ?", fieldName); - } else if (value instanceof Long) { - return String.format("%s = ?", fieldName); - } else if (value instanceof Float) { - return String.format("%s = ?", fieldName); - } else if (value instanceof Double) { - return String.format("%s = ?", fieldName); - } else if (value instanceof OffsetDateTime) { - return String.format("%s = ?", fieldName); - } else { - throw new SKException("Unsupported filter value type '" - + value.getClass().getSimpleName() + "'."); - } - } -} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java index 08b133cf..89e8f857 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; @@ -29,7 +30,6 @@ public class MySQLVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider implements SQLVectorStoreQueryProvider { - private final DataSource dataSource; private final ObjectMapper objectMapper; @SuppressFBWarnings("EI_EXPOSE_REP2") @@ -39,7 +39,6 @@ private MySQLVectorStoreQueryProvider( @Nonnull String prefixForCollectionTables, @Nonnull ObjectMapper objectMapper) { super(dataSource, collectionsTable, prefixForCollectionTables); - this.dataSource = dataSource; this.objectMapper = objectMapper; } @@ -66,6 +65,12 @@ private void setUpsertStatementValues(PreparedStatement statement, Object record statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); continue; } + } else if (field instanceof VectorStoreRecordDataField) { + // Convert List field to a string + if (field.getFieldType().equals(List.class)) { + statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); + continue; + } } statement.setObject(i + 1, diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index a353cdcb..6be72981 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -6,9 +6,12 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreRecordCollectionSearchMapping; +import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; +import com.microsoft.semantickernel.data.filter.EqualToFilterClause; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; @@ -29,6 +32,7 @@ import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -41,12 +45,6 @@ */ public class PostgreSQLVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider implements SQLVectorStoreQueryProvider { - - private final Map, String> supportedKeyTypes; - private final Map, String> supportedDataTypes; - private final Map, String> supportedVectorTypes; - - private final DataSource dataSource; private final String collectionsTable; private final String prefixForCollectionTables; private final ObjectMapper objectMapper; @@ -57,16 +55,28 @@ private PostgreSQLVectorStoreQueryProvider( @Nonnull String collectionsTable, @Nonnull String prefixForCollectionTables, @Nonnull ObjectMapper objectMapper) { - super(dataSource, collectionsTable, prefixForCollectionTables); - this.dataSource = dataSource; + super( + dataSource, + collectionsTable, + prefixForCollectionTables, + buildSupportedKeyTypes(), + buildSupportedDataTypes(), + buildSupportedVectorTypes()); this.collectionsTable = collectionsTable; this.prefixForCollectionTables = prefixForCollectionTables; this.objectMapper = objectMapper; + } - supportedKeyTypes = new HashMap<>(); - supportedKeyTypes.put(String.class, "VARCHAR(255)"); + private static Map, String> buildSupportedVectorTypes() { + HashMap, String> supportedVectorTypes = new HashMap<>(); + supportedVectorTypes.put(String.class, "TEXT"); + supportedVectorTypes.put(List.class, "VECTOR(%d)"); + supportedVectorTypes.put(Collection.class, "VECTOR(%d)"); + return supportedVectorTypes; + } - supportedDataTypes = new HashMap<>(); + private static Map, String> buildSupportedDataTypes() { + HashMap, String> supportedDataTypes = new HashMap<>(); supportedDataTypes.put(String.class, "TEXT"); supportedDataTypes.put(Integer.class, "INTEGER"); supportedDataTypes.put(int.class, "INTEGER"); @@ -79,41 +89,14 @@ private PostgreSQLVectorStoreQueryProvider( supportedDataTypes.put(Boolean.class, "BOOLEAN"); supportedDataTypes.put(boolean.class, "BOOLEAN"); supportedDataTypes.put(OffsetDateTime.class, "TIMESTAMPTZ"); - - supportedVectorTypes = new HashMap<>(); - supportedDataTypes.put(String.class, "TEXT"); - supportedVectorTypes.put(List.class, "VECTOR(%d)"); - supportedVectorTypes.put(Collection.class, "VECTOR(%d)"); - } - - /** - * Gets the supported key types and their corresponding SQL types. - * - * @return the supported key types - */ - @Override - public Map, String> getSupportedKeyTypes() { - return new HashMap<>(this.supportedKeyTypes); - } - - /** - * Gets the supported data types and their corresponding SQL types. - * - * @return the supported data types - */ - @Override - public Map, String> getSupportedDataTypes() { - return new HashMap<>(this.supportedDataTypes); + supportedDataTypes.put(List.class, "JSONB"); + return supportedDataTypes; } - /** - * Gets the supported vector types and their corresponding SQL types. - * - * @return the supported vector types - */ - @Override - public Map, String> getSupportedVectorTypes() { - return new HashMap<>(this.supportedVectorTypes); + private static HashMap, String> buildSupportedKeyTypes() { + HashMap, String> supportedKeyTypes = new HashMap<>(); + supportedKeyTypes.put(String.class, "VARCHAR(255)"); + return supportedKeyTypes; } /** @@ -258,6 +241,12 @@ private void setUpsertStatementValues(PreparedStatement statement, Object record statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); continue; } + } else if (field instanceof VectorStoreRecordDataField) { + // Convert List field to a string + if (field.getFieldType().equals(List.class)) { + statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); + continue; + } } statement.setObject(i + 1, @@ -276,6 +265,12 @@ private String getWildcardStringWithCast(List fields) { if (field instanceof VectorStoreRecordVectorField) { wildcard += "::vector"; } + if (field instanceof VectorStoreRecordDataField) { + // Add casting for List fields + if (field.getFieldType().equals(List.class)) { + wildcard += "::jsonb"; + } + } return wildcard; }) .collect(Collectors.joining(", ")); @@ -372,11 +367,8 @@ public List> search(String collectionName, "Distance function is required for vector field: " + vectorField.getName()); } - String filter = SQLVectorStoreRecordCollectionSearchMapping.buildFilter( - options.getVectorSearchFilter(), - recordDefinition); - List parameters = SQLVectorStoreRecordCollectionSearchMapping - .getFilterParameters(options.getVectorSearchFilter()); + String filter = getFilter(options.getVectorSearchFilter(), recordDefinition); + List parameters = getFilterParameters(options.getVectorSearchFilter()); String filterClause = filter.isEmpty() ? "" : "WHERE " + filter; String searchQuery = formatQuery( @@ -418,6 +410,42 @@ public List> search(String collectionName, } } + /** + * Gets the filter parameters for the given vector search filter to associate with the filter string + * generated by the getFilter method. + * + * @param filter The filter to get the filter parameters for. + * @return The filter parameters. + */ + @Override + public List getFilterParameters(VectorSearchFilter filter) { + if (filter == null + || filter.getFilterClauses().isEmpty()) { + return Collections.emptyList(); + } + + return filter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof EqualToFilterClause) { + EqualToFilterClause equalToFilterClause = (EqualToFilterClause) filterClause; + return equalToFilterClause.getValue(); + } else if (filterClause instanceof AnyTagEqualToFilterClause) { + AnyTagEqualToFilterClause anyTagEqualToFilterClause = (AnyTagEqualToFilterClause) filterClause; + return String.format("[\"%s\"]", anyTagEqualToFilterClause.getValue()); + } else { + throw new SKException("Unsupported filter clause type '" + + filterClause.getClass().getSimpleName() + "'."); + } + }).collect(Collectors.toList()); + } + + @Override + public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { + String fieldName = JDBCVectorStoreQueryProvider + .validateSQLidentifier(filterClause.getFieldName()); + + return String.format("%s @> ?::jsonb", fieldName); + } + /** * A builder for the PostgreSQLVectorStoreQueryProvider class. */ diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java index 50cbbd76..4b784a0a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; @@ -123,15 +124,20 @@ public PostgreSQLVectorStoreRecordMapper build() { for (VectorStoreRecordField field : fields) { Object value = resultSet.getObject(field.getEffectiveStorageName()); + Class fieldType = field.getFieldType(); if (field instanceof VectorStoreRecordVectorField) { - Class vectorType = field.getFieldType(); - // If the vector field is other than String, deserialize it from the JSON string - if (!vectorType.equals(String.class)) { + if (!fieldType.equals(String.class)) { // Deserialize the pgvector string to the vector type value = objectMapper.readValue(((PGobject) value).getValue(), - vectorType); + fieldType); + } + } else if (field instanceof VectorStoreRecordDataField) { + // If the field is List, deserialize it from the JSON string + if (fieldType.equals(List.class)) { + value = objectMapper.readValue(((PGobject) value).getValue(), + fieldType); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java index 23f9aa7c..b65fe9f9 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java @@ -400,6 +400,7 @@ public Mono>> searchAsync(List vector, return createCollectionIfNotExistsAsync().flatMap(collection -> Mono.fromCallable(() -> { Pair ftSearchParams = RedisVectorStoreCollectionSearchMapping + .getInstance() .buildQuery(vector, options, recordDefinition, RedisStorageType.HASH_SET); SearchResult searchResult = client.ftSearch(collectionName, ftSearchParams.getLeft(), diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java index b6eefd7a..4af97846 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java @@ -417,7 +417,7 @@ public Mono>> searchAsync(List vector, return createCollectionIfNotExistsAsync().flatMap(collection -> Mono.fromCallable(() -> { Pair ftSearchParams = RedisVectorStoreCollectionSearchMapping - .buildQuery(vector, options, recordDefinition, RedisStorageType.JSON); + .getInstance().buildQuery(vector, options, recordDefinition, RedisStorageType.JSON); SearchResult searchResult = client.ftSearch(collectionName, ftSearchParams.getLeft(), ftSearchParams.getRight()); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java index ce2b7de0..de51b8e7 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java @@ -1,8 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; -import com.microsoft.semantickernel.connectors.data.redis.filter.RedisEqualToFilterClause; +import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; +import com.microsoft.semantickernel.data.filter.EqualToFilterClause; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.filter.FilterMapping; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; @@ -19,10 +21,21 @@ /** * A mapping for searching a collection of vector records in Redis. */ -public class RedisVectorStoreCollectionSearchMapping { +public class RedisVectorStoreCollectionSearchMapping implements FilterMapping { static final String VECTOR_SCORE_FIELD = "vector_score"; + private RedisVectorStoreCollectionSearchMapping() { + } + + static class RedisVectorStoreCollectionSearchMappingHolder { + static final RedisVectorStoreCollectionSearchMapping INSTANCE = new RedisVectorStoreCollectionSearchMapping(); + } + + static RedisVectorStoreCollectionSearchMapping getInstance() { + return RedisVectorStoreCollectionSearchMappingHolder.INSTANCE; + } + /** * Builds a query for searching a collection of vector records in Redis. * @param vector the vector to search for @@ -31,7 +44,7 @@ public class RedisVectorStoreCollectionSearchMapping { * @param storageType the storage type * @return the query and search parameters */ - public static Pair buildQuery(List vector, + public Pair buildQuery(List vector, VectorSearchOptions options, VectorStoreRecordDefinition recordDefinition, RedisStorageType storageType) { @@ -45,7 +58,7 @@ public static Pair buildQuery(List vector, : (VectorStoreRecordVectorField) recordDefinition .getField(options.getVectorFieldName()); - String filter = buildFilter(options.getVectorSearchFilter(), recordDefinition); + String filter = getFilter(options.getVectorSearchFilter(), recordDefinition); String knn = String.format("%s=>[KNN $K @%s $BLOB AS %s]", filter, vectorField.getEffectiveStorageName(), VECTOR_SCORE_FIELD); @@ -104,30 +117,67 @@ public static List convertByteArrayToList(byte[] bytes) { } /** - * Builds a filter for searching a collection of vector records in Redis. - * @param vectorSearchFilter the search filter - * @param recordDefinition the record definition - * @return the filter + * Gets the filter string for the given vector search filter and record definition. + * + * @param filter The filter to get the filter string for. + * @param recordDefinition The record definition to get the filter string for. + * @return The filter string. */ - public static String buildFilter(VectorSearchFilter vectorSearchFilter, + @Override + public String getFilter(VectorSearchFilter filter, VectorStoreRecordDefinition recordDefinition) { - if (vectorSearchFilter == null - || vectorSearchFilter.getFilterClauses().isEmpty()) { + if (filter == null + || filter.getFilterClauses().isEmpty()) { return "*"; } return String.format("(%s)", - vectorSearchFilter.getFilterClauses().stream().map(filterClause -> { - if (filterClause instanceof RedisEqualToFilterClause) { - RedisEqualToFilterClause equalToFilterClause = (RedisEqualToFilterClause) filterClause; - return new RedisEqualToFilterClause( + filter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof EqualToFilterClause) { + EqualToFilterClause equalToFilterClause = (EqualToFilterClause) filterClause; + return getEqualToFilter(new EqualToFilterClause( recordDefinition.getField(equalToFilterClause.getFieldName()) .getEffectiveStorageName(), - equalToFilterClause.getValue()).getFilter(); + equalToFilterClause.getValue())); } else { throw new SKException("Unsupported filter clause type '" + filterClause.getClass().getSimpleName() + "'."); } }).collect(Collectors.joining(" "))); } + + /** + * Gets the filter string for the given equal to filter clause. + * + * @param filterClause The equal to filter clause to get the filter string for. + * @return The filter string. + */ + @Override + public String getEqualToFilter(EqualToFilterClause filterClause) { + String fieldName = filterClause.getFieldName(); + Object value = filterClause.getValue(); + String formattedValue; + + if (value instanceof String) { + formattedValue = String.format("\"%s\"", value); + } else if (value instanceof Number) { + formattedValue = String.format("[%s %s]", value, value); + } else { + throw new SKException("Unsupported filter value type '" + + value.getClass().getSimpleName() + "'."); + } + + return String.format("@%s:%s", fieldName, formattedValue); + } + + /** + * Gets the filter string for the given any tag equal to filter clause. + * + * @param filterClause The any tag equal to filter clause to get the filter string for. + * @return The filter string. + */ + @Override + public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { + return String.format("@%s:\"%s\"", filterClause.getFieldName(), filterClause.getValue()); + } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/filter/RedisEqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/filter/RedisEqualToFilterClause.java deleted file mode 100644 index 0cdae1f0..00000000 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/filter/RedisEqualToFilterClause.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.redis.filter; - -import com.microsoft.semantickernel.data.filter.EqualToFilterClause; -import com.microsoft.semantickernel.exceptions.SKException; - -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; - -/** - * Represents a Redis filter clause. - */ -public class RedisEqualToFilterClause extends EqualToFilterClause { - - /** - * Creates a new instance of RedisEqualToFilterClause. - * @param fieldName The name of the field to filter on. - * @param value The value to filter for. - */ - public RedisEqualToFilterClause(String fieldName, Object value) { - super(fieldName, value); - } - - /** - * Gets the filter string. - * - * @return The filter. - */ - @Override - public String getFilter() { - String fieldName = getFieldName(); - Object value = getValue(); - String formattedValue; - - if (value instanceof String) { - formattedValue = String.format("\"%s\"", value); - } else if (value instanceof Boolean) { - formattedValue = value.toString().toLowerCase(); - } else if (value instanceof Number) { - formattedValue = value.toString(); - } else if (value instanceof OffsetDateTime) { - formattedValue = ((OffsetDateTime) value) - .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); - } else { - throw new SKException("Unsupported filter value type '" - + value.getClass().getSimpleName() + "'."); - } - - return String.format("@%s:%s", fieldName, formattedValue); - } -} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/sqlite/SQLiteVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/sqlite/SQLiteVectorStoreQueryProvider.java index 8e639007..b9776f19 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/sqlite/SQLiteVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/sqlite/SQLiteVectorStoreQueryProvider.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; @@ -64,6 +65,12 @@ private void setUpsertStatementValues(PreparedStatement statement, Object record statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); continue; } + } else if (field instanceof VectorStoreRecordDataField) { + // Convert List field to a string + if (field.getFieldType().equals(List.class)) { + statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); + continue; + } } statement.setObject(i + 1, diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java index fd27b6d7..dbb56da4 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java @@ -52,7 +52,7 @@ public VectorStoreRecordCollection getCollection( collections, (VolatileVectorStoreRecordCollectionOptions) options); } - + /** * Gets the names of all collections in the vector store. * diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java index 5cbfe9e7..81eea80b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java @@ -4,7 +4,7 @@ /** * A filter clause that filters on any tag equal to a value. */ -public abstract class AnyTagEqualToFilterClause implements FilterClause { +public class AnyTagEqualToFilterClause implements FilterClause { private final String fieldName; private final Object value; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java index b12f934d..c75b04dc 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java @@ -5,7 +5,6 @@ * A filter clause that filters on a field equal to a value. */ public class EqualToFilterClause implements FilterClause { - private final String fieldName; private final Object value; @@ -37,15 +36,4 @@ public String getFieldName() { public Object getValue() { return value; } - - /** - * Gets the filter string. - * - * @return The filter. - */ - @Override - public String getFilter() { - throw new UnsupportedOperationException(String.format( - "Not implemented. Use one of %s derived classes.", this.getClass().getSimpleName())); - } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java index c8b86ca1..8e9a6e60 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java @@ -5,11 +5,4 @@ * A filter clause for a query. */ public interface FilterClause { - - /** - * Gets the filter string. - * - * @return The filter. - */ - String getFilter(); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java new file mode 100644 index 00000000..6483ab40 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.filter; + +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; + +public interface FilterMapping { + /** + * Gets the filter string for the given vector search filter and record definition. + * + * @param filter The filter to get the filter string for. + * @param recordDefinition The record definition to get the filter string for. + * @return The filter string. + */ + String getFilter(VectorSearchFilter filter, VectorStoreRecordDefinition recordDefinition); + + /** + * Gets the filter string for the given equal to filter clause. + * + * @param filterClause The equal to filter clause to get the filter string for. + * @return The filter string. + */ + String getEqualToFilter(EqualToFilterClause filterClause); + + /** + * Gets the filter string for the given any tag equal to filter clause. + * + * @param filterClause The any tag equal to filter clause to get the filter string for. + * @return The filter string. + */ + String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause); +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java index 1d0b8bf4..7006135f 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java @@ -68,23 +68,24 @@ public static class Builder { /** * Adds an EqualToFilterClause to the filter. * - * @param equalToFilterClause The EqualToFilterClause to add. + * @param fieldName The field name to filter on. + * @param value The value. * @return The builder. */ - public Builder withEqualToFilterClause(EqualToFilterClause equalToFilterClause) { - filterClauses.add(equalToFilterClause); + public Builder equalTo(String fieldName, Object value) { + filterClauses.add(new EqualToFilterClause(fieldName, value)); return this; } /** * Adds an AnyTagEqualToFilterClause to the filter. * - * @param anyTagEqualToFilterClause The AnyTagEqualToFilterClause clause to add. + * @param fieldName The field name to filter on. + * @param value The value. * @return The builder. */ - public Builder withAnyTagEqualToFilterClause( - AnyTagEqualToFilterClause anyTagEqualToFilterClause) { - filterClauses.add(anyTagEqualToFilterClause); + public Builder anyTagEqualTo(String fieldName, Object value) { + filterClauses.add(new AnyTagEqualToFilterClause(fieldName, value)); return this; } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordDataAttribute.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java similarity index 89% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordDataAttribute.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java index 151785a1..532cffdb 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordDataAttribute.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorstorage.attributes; +package com.microsoft.semantickernel.data.vectorstorage.annotations; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; @@ -11,7 +11,7 @@ */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.FIELD) -public @interface VectorStoreRecordDataAttribute { +public @interface VectorStoreRecordData { /** * Storage name of the field. * This value is only used when JSON Serialization using Jackson is not supported in a VectorStore. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKeyAttribute.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java similarity index 79% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKeyAttribute.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java index 27ba5fa7..8f8c98a7 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKeyAttribute.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorstorage.attributes; +package com.microsoft.semantickernel.data.vectorstorage.annotations; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; @@ -11,7 +11,7 @@ */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.FIELD) -public @interface VectorStoreRecordKeyAttribute { +public @interface VectorStoreRecordKey { /** * Storage name of the field. * @return The storage name of the field. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java similarity index 91% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java index ef22c32c..90128b41 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorstorage.attributes; +package com.microsoft.semantickernel.data.vectorstorage.annotations; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; @@ -15,7 +15,7 @@ */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.FIELD) -public @interface VectorStoreRecordVectorAttribute { +public @interface VectorStoreRecordVector { /** * Number of dimensions in the vector. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java index 876c96e9..54b2bf2b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java @@ -2,9 +2,9 @@ package com.microsoft.semantickernel.data.vectorstorage.definition; import com.fasterxml.jackson.annotation.JsonProperty; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import com.microsoft.semantickernel.exceptions.SKException; import java.lang.reflect.Field; import java.util.ArrayList; @@ -167,9 +167,9 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) storageName = field.getAnnotation(JsonProperty.class).value(); } - if (field.isAnnotationPresent(VectorStoreRecordKeyAttribute.class)) { - VectorStoreRecordKeyAttribute keyAttribute = field - .getAnnotation(VectorStoreRecordKeyAttribute.class); + if (field.isAnnotationPresent(VectorStoreRecordKey.class)) { + VectorStoreRecordKey keyAttribute = field + .getAnnotation(VectorStoreRecordKey.class); if (storageName == null) { storageName = keyAttribute.storageName().isEmpty() ? field.getName() @@ -182,9 +182,9 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) .build()); } - if (field.isAnnotationPresent(VectorStoreRecordDataAttribute.class)) { - VectorStoreRecordDataAttribute dataAttribute = field - .getAnnotation(VectorStoreRecordDataAttribute.class); + if (field.isAnnotationPresent(VectorStoreRecordData.class)) { + VectorStoreRecordData dataAttribute = field + .getAnnotation(VectorStoreRecordData.class); if (storageName == null) { storageName = dataAttribute.storageName().isEmpty() ? field.getName() @@ -198,9 +198,9 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) .build()); } - if (field.isAnnotationPresent(VectorStoreRecordVectorAttribute.class)) { - VectorStoreRecordVectorAttribute vectorAttribute = field - .getAnnotation(VectorStoreRecordVectorAttribute.class); + if (field.isAnnotationPresent(VectorStoreRecordVector.class)) { + VectorStoreRecordVector vectorAttribute = field + .getAnnotation(VectorStoreRecordVector.class); if (storageName == null) { storageName = vectorAttribute.storageName().isEmpty() ? field.getName() diff --git a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java index 7d5b62e5..f6741e5a 100644 --- a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java +++ b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java @@ -3,43 +3,43 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import java.util.List; public class Hotel { - @VectorStoreRecordKeyAttribute + @VectorStoreRecordKey private final String id; - @VectorStoreRecordDataAttribute(isFilterable = true) + @VectorStoreRecordData(isFilterable = true) private final String name; - @VectorStoreRecordDataAttribute + @VectorStoreRecordData private final int code; @JsonProperty("summary") - @VectorStoreRecordDataAttribute() + @VectorStoreRecordData() private final String description; @JsonProperty("summaryEmbedding1") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) private final List euclidean; @JsonProperty("summaryEmbedding2") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE) + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List cosineDistance; @JsonProperty("summaryEmbedding3") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.COSINE_SIMILARITY) + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_SIMILARITY) private final List cosineSimilarity; @JsonProperty("summaryEmbedding4") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.DOT_PRODUCT) + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.DOT_PRODUCT) private final List dotProduct; - @VectorStoreRecordDataAttribute + @VectorStoreRecordData private double rating; public Hotel() { diff --git a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java index d2084967..23b1ac78 100644 --- a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java +++ b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java @@ -6,13 +6,10 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollection; -import com.microsoft.semantickernel.connectors.data.jdbc.filter.SQLEqualToFilterClause; import com.microsoft.semantickernel.data.filter.EqualToFilterClause; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; @@ -23,7 +20,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; -import org.junit.jupiter.params.provider.MethodSource; public class VolatileVectorStoreRecordCollectionTest { @@ -198,7 +194,7 @@ public void searchWithFilter(DistanceFunction distanceFunction) { .withLimit(3) .withVectorSearchFilter( VectorSearchFilter.builder() - .withEqualToFilterClause(new EqualToFilterClause("rating", 4.0)).build()) + .equalTo("rating", 4.0).build()) .build(); // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned