From 564273845f2a4ccb3c2bd077a7899695ca44268a Mon Sep 17 00:00:00 2001 From: GitHub Date: Wed, 14 Aug 2024 20:21:07 +0100 Subject: [PATCH 001/277] [maven-release-plugin] prepare for next development iteration --- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- pom.xml | 4 ++-- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-bom/pom.xml | 4 ++-- semantickernel-experimental/pom.xml | 2 +- 20 files changed, 22 insertions(+), 22 deletions(-) diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index f9ff65a4..7899fe11 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.2 + 1.2.3-SNAPSHOT ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index ccfbb689..bce95e64 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.2 + 1.2.3-SNAPSHOT ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index d6a3762e..bcc7f135 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.2 + 1.2.3-SNAPSHOT ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index b3ec6563..6def803a 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.2.2 + 1.2.3-SNAPSHOT ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index 47d10b04..15ed5bb6 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.2 + 1.2.3-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 84e20d0c..c301d7f9 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.2 + 1.2.3-SNAPSHOT pom https://www.github.com/microsoft/semantic-kernel @@ -825,6 +825,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.2.2 + HEAD diff --git a/samples/pom.xml b/samples/pom.xml index a62aa7c4..61d34a0b 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.2 + 1.2.3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index cd4b855c..24a745b0 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.2 + 1.2.3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 27c3420d..1850e6f0 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.2.2 + 1.2.3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index a0b84de8..59b998bf 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.2.2 + 1.2.3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index b5b4fe7b..755964a2 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.2 + 1.2.3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index e9fab05f..ee374eae 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.2.2 + 1.2.3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index 8300dcec..56fd812a 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.2.2 + 1.2.3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index c8fb38ba..94e9c598 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.2 + 1.2.3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index 71d41a94..f9a1f140 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.2 + 1.2.3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index e1804b6f..7d3ac488 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.2.2 + 1.2.3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index e5ce6b0f..fd85b83a 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.2.2 + 1.2.3-SNAPSHOT ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index c6bbae4a..16581d2f 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.2 + 1.2.3-SNAPSHOT ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 1322666f..dfb1eca2 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.2.2 + 1.2.3-SNAPSHOT pom Semantic Kernel Java BOM @@ -256,6 +256,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.2.2 + HEAD diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 44772dc2..5467df12 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.2 + 1.2.3-SNAPSHOT semantickernel-experimental From 958ec4591dc6fe16775d6a03d5a7981d84c23741 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Wed, 14 Aug 2024 15:57:02 -0700 Subject: [PATCH 002/277] Add RedisHashSetVectorStoreRecordCollection implementation --- .../chatcompletion/OpenAIChatCompletion.java | 8 +- ...ashSetVectorStoreRecordCollectionTest.java | 304 +++++++++++++++ ...sJsonVectorStoreRecordCollectionTest.java} | 38 +- .../memory/redis/RedisVectorStoreTest.java | 29 +- .../memory/Redis_DataStorage.java | 5 +- .../services/chatcompletion/ChatHistory.java | 3 +- .../AzureAISearchVectorStore.java | 6 +- ...rchVectorStoreRecordCollectionFactory.java | 7 +- .../connectors/data/jdbc/JDBCVectorStore.java | 8 +- ...DBCVectorStoreRecordCollectionFactory.java | 11 +- ...disHashSetVectorStoreRecordCollection.java | 353 ++++++++++++++++++ ...SetVectorStoreRecordCollectionOptions.java | 156 ++++++++ .../RedisHashSetVectorStoreRecordMapper.java | 162 ++++++++ ...RedisJsonVectorStoreRecordCollection.java} | 10 +- ...onVectorStoreRecordCollectionOptions.java} | 10 +- ... => RedisJsonVectorStoreRecordMapper.java} | 14 +- .../data/redis/RedisStorageType.java | 16 + .../data/redis/RedisVectorStore.java | 30 +- .../data/redis/RedisVectorStoreOptions.java | 36 +- ...disVectorStoreRecordCollectionFactory.java | 11 +- 20 files changed, 1134 insertions(+), 83 deletions(-) create mode 100644 api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java rename api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/{RedisVectorStoreRecordCollectionTest.java => RedisJsonVectorStoreRecordCollectionTest.java} (82%) create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/{RedisVectorStoreRecordCollection.java => RedisJsonVectorStoreRecordCollection.java} (97%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/{RedisVectorStoreRecordCollectionOptions.java => RedisJsonVectorStoreRecordCollectionOptions.java} (93%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/{RedisVectorStoreRecordMapper.java => RedisJsonVectorStoreRecordMapper.java} (89%) create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisStorageType.java diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java index ca8eb5b3..5442e51e 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java @@ -357,7 +357,8 @@ private Mono internalChatMessageContentsAsync( // If we don't want to attempt to invoke any functions // Or if we are auto-invoking, but we somehow end up with other than 1 choice even though only 1 was requested if (autoInvokeAttempts == 0 || responseMessages.size() != 1) { - List> chatMessageContents = getChatMessageContentsAsync(completions); + List> chatMessageContents = getChatMessageContentsAsync( + completions); return Mono.just(messages.addChatMessage(chatMessageContents)); } // Or if there are no tool calls to be done @@ -365,7 +366,7 @@ private Mono internalChatMessageContentsAsync( List toolCalls = response.getToolCalls(); if (toolCalls == null || toolCalls.isEmpty()) { List> chatMessageContents = getChatMessageContentsAsync( - completions); + completions); return Mono.just(messages.addChatMessage(chatMessageContents)); } @@ -603,8 +604,7 @@ private List> getChatMessageContentsAsync( .filter(Objects::nonNull) .collect(Collectors.toList()); - List> chatMessageContent = - responseMessages + List> chatMessageContent = responseMessages .stream() .map(response -> { try { diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java new file mode 100644 index 00000000..d39eed92 --- /dev/null +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -0,0 +1,304 @@ +package com.microsoft.semantickernel.tests.connectors.memory.redis; + +import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollection; +import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; +import com.microsoft.semantickernel.tests.connectors.memory.Hotel; +import com.redis.testcontainers.RedisContainer; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import redis.clients.jedis.JedisPooled; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +@Testcontainers +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class RedisHashSetVectorStoreRecordCollectionTest { + + @Container private static final RedisContainer redisContainer = new RedisContainer("redis/redis-stack:latest"); + + private static final Map> optionsMap = new HashMap<>(); + + public enum RecordCollectionOptions { + DEFAULT, WITH_CUSTOM_DEFINITION + } + + @BeforeAll + static void setup() { + optionsMap.put(RecordCollectionOptions.DEFAULT, RedisHashSetVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); + + List fields = new ArrayList<>(); + fields.add(VectorStoreRecordKeyField.builder() + .withName("id") + .build()); + fields.add(VectorStoreRecordDataField.builder() + .withName("name") + .withFieldType(String.class) + .build()); + fields.add(VectorStoreRecordDataField.builder() + .withName("code") + .withFieldType(Integer.class) + .build()); + fields.add(VectorStoreRecordDataField.builder() + .withName("description") + .withFieldType(String.class) + .withHasEmbedding(true) + .withEmbeddingFieldName("descriptionEmbedding") + .build()); + fields.add(VectorStoreRecordVectorField.builder() + .withName("descriptionEmbedding") + .withDimensions(768) + .build()); + fields.add(VectorStoreRecordDataField.builder() + .withName("rating") + .withFieldType(Double.class) + .build()); + VectorStoreRecordDefinition recordDefinition = VectorStoreRecordDefinition.fromFields(fields); + + optionsMap.put(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, RedisHashSetVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .withRecordDefinition(recordDefinition) + .build()); + } + + private RedisHashSetVectorStoreRecordCollection buildrecordCollection(@Nonnull RedisHashSetVectorStoreRecordCollectionOptions options, @Nonnull String collectionName) { + return new RedisHashSetVectorStoreRecordCollection<>(new JedisPooled(redisContainer.getRedisURI()), collectionName, RedisHashSetVectorStoreRecordCollectionOptions.builder() + .withRecordClass(options.getRecordClass()) + .withVectorStoreRecordMapper(options.getVectorStoreRecordMapper()) + .withRecordDefinition(options.getRecordDefinition()) + .withPrefixCollectionName(options.isPrefixCollectionName()) + .build()); + } + + private List getHotels() { + return List.of( + new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(1.0f, 2.0f, 3.0f), 4.0), + new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(1.0f, 2.0f, 3.0f), 3.0), + new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(1.0f, 2.0f, 3.0f), 5.0), + new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(1.0f, 2.0f, 3.0f), 4.0), + new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(1.0f, 2.0f, 3.0f), 5.0) + ); + } + + @Order(1) + @ParameterizedTest + @EnumSource(RecordCollectionOptions.class) + public void buildrecordCollection(RecordCollectionOptions options) { + assertNotNull(buildrecordCollection(optionsMap.get(options), options.name())); + } + + @Order(2) + @ParameterizedTest + @EnumSource(RecordCollectionOptions.class) + public void createCollectionAsync(RecordCollectionOptions options) { + RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + + assertEquals(false, recordCollection.collectionExistsAsync().block()); + recordCollection.createCollectionAsync().block(); + assertEquals(true, recordCollection.collectionExistsAsync().block()); + } + + @Test + public void deleteCollectionAsync() { + RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(RecordCollectionOptions.DEFAULT), "deleteCollectionAsync"); + + assertEquals(false, recordCollection.collectionExistsAsync().block()); + recordCollection.createCollectionAsync().block(); + recordCollection.deleteCollectionAsync().block(); + assertEquals(false, recordCollection.collectionExistsAsync().block()); + } + + @ParameterizedTest + @EnumSource(RecordCollectionOptions.class) + public void upsertAndGetRecordAsync(RecordCollectionOptions options) { + RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + + List hotels = getHotels(); + for (Hotel hotel : hotels) { + recordCollection.upsertAsync(hotel, null).block(); + } + + for (Hotel hotel : hotels) { + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block(); + assertNotNull(retrievedHotel); + assertEquals(hotel.getId(), retrievedHotel.getId()); + } + } + + @ParameterizedTest + @EnumSource(RecordCollectionOptions.class) + public void getBatchAsync(RecordCollectionOptions options) { + RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + + List hotels = getHotels(); + for (Hotel hotel : hotels) { + recordCollection.upsertAsync(hotel, null).block(); + } + + List ids = new ArrayList<>(); + hotels.forEach(hotel -> ids.add(hotel.getId())); + + List retrievedHotels = recordCollection.getBatchAsync(ids, null).block(); + + assertNotNull(retrievedHotels); + assertEquals(hotels.size(), retrievedHotels.size()); + for (int i = 0; i < hotels.size(); i++) { + assertEquals(hotels.get(i).getId(), retrievedHotels.get(i).getId()); + } + } + + @ParameterizedTest + @EnumSource(RecordCollectionOptions.class) + public void upsertBatchAsync(RecordCollectionOptions options) { + RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + + List hotels = getHotels(); + List keys = recordCollection.upsertBatchAsync(hotels, null).block(); + assertNotNull(keys); + + List retrievedHotels = (List) recordCollection.getBatchAsync(keys, null).block(); + + assertNotNull(retrievedHotels); + assertEquals(hotels.size(), retrievedHotels.size()); + for (int i = 0; i < hotels.size(); i++) { + assertEquals(hotels.get(i).getId(), retrievedHotels.get(i).getId()); + } + } + + @ParameterizedTest + @EnumSource(RecordCollectionOptions.class) + public void deleteAsync(RecordCollectionOptions options) { + RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + for (Hotel hotel : hotels) { + recordCollection.deleteAsync(hotel.getId(), null).block(); + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block(); + assertNull(retrievedHotel); + } + } + + @ParameterizedTest + @EnumSource(RecordCollectionOptions.class) + public void deleteBatchAsync(RecordCollectionOptions options) { + RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + List ids = new ArrayList<>(); + hotels.forEach(hotel -> ids.add(hotel.getId())); + + recordCollection.deleteBatchAsync(ids, null).block(); + + for (String id : ids) { + Hotel retrievedHotel = recordCollection.getAsync(id, null).block(); + assertNull(retrievedHotel); + } + } + + @ParameterizedTest + @EnumSource(RecordCollectionOptions.class) + public void getAsyncWithVectors(RecordCollectionOptions options) { + RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + for (Hotel hotel : hotels) { + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block(); + assertNotNull(retrievedHotel); + assertNotNull(retrievedHotel.getDescriptionEmbedding()); + assertEquals(hotel.getId(), retrievedHotel.getId()); + assertEquals(hotel.getDescription(), retrievedHotel.getDescription()); + } + } + + @ParameterizedTest + @EnumSource(RecordCollectionOptions.class) + public void getBatchAsyncWithVectors(RecordCollectionOptions options) { + RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + List ids = new ArrayList<>(); + hotels.forEach(hotel -> ids.add(hotel.getId())); + + List retrievedHotels = recordCollection.getBatchAsync(ids, null).block(); + + assertNotNull(retrievedHotels); + assertEquals(hotels.size(), retrievedHotels.size()); + for (int i = 0; i < hotels.size(); i++) { + assertEquals(hotels.get(i).getId(), retrievedHotels.get(i).getId()); + assertEquals(hotels.get(i).getDescription(), retrievedHotels.get(i).getDescription()); + assertNotNull(retrievedHotels.get(i).getDescriptionEmbedding()); + } + } + + @ParameterizedTest + @EnumSource(RecordCollectionOptions.class) + public void getAsyncWithNoVectors(RecordCollectionOptions options) { + RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + GetRecordOptions getRecordOptions = GetRecordOptions.builder().includeVectors(false).build(); + for (Hotel hotel : hotels) { + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), getRecordOptions).block(); + assertNotNull(retrievedHotel); + assertNull(retrievedHotel.getDescriptionEmbedding()); + assertEquals(hotel.getId(), retrievedHotel.getId()); + assertEquals(hotel.getDescription(), retrievedHotel.getDescription()); + } + } + + @ParameterizedTest + @EnumSource(RecordCollectionOptions.class) + public void getBatchAsyncWithNoVectors(RecordCollectionOptions options) { + RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + GetRecordOptions getRecordOptions = GetRecordOptions.builder().includeVectors(false).build(); + List ids = new ArrayList<>(); + hotels.forEach(hotel -> ids.add(hotel.getId())); + + List retrievedHotels = recordCollection.getBatchAsync(ids, getRecordOptions).block(); + + assertNotNull(retrievedHotels); + assertEquals(hotels.size(), retrievedHotels.size()); + for (int i = 0; i < hotels.size(); i++) { + assertEquals(hotels.get(i).getId(), retrievedHotels.get(i).getId()); + assertEquals(hotels.get(i).getDescription(), retrievedHotels.get(i).getDescription()); + assertNull(retrievedHotels.get(i).getDescriptionEmbedding()); + } + } +} diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java similarity index 82% rename from api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreRecordCollectionTest.java rename to api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index db3b6d6c..d5fb7cd7 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -1,7 +1,7 @@ package com.microsoft.semantickernel.tests.connectors.memory.redis; -import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreRecordCollection; -import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollection; +import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; @@ -35,11 +35,11 @@ @Testcontainers @TestMethodOrder(MethodOrderer.OrderAnnotation.class) -public class RedisVectorStoreRecordCollectionTest { +public class RedisJsonVectorStoreRecordCollectionTest { @Container private static final RedisContainer redisContainer = new RedisContainer("redis/redis-stack:latest"); - private static final Map> optionsMap = new HashMap<>(); + private static final Map> optionsMap = new HashMap<>(); public enum RecordCollectionOptions { DEFAULT, WITH_CUSTOM_DEFINITION @@ -47,7 +47,7 @@ public enum RecordCollectionOptions { @BeforeAll static void setup() { - optionsMap.put(RecordCollectionOptions.DEFAULT, RedisVectorStoreRecordCollectionOptions.builder() + optionsMap.put(RecordCollectionOptions.DEFAULT, RedisJsonVectorStoreRecordCollectionOptions.builder() .withRecordClass(Hotel.class) .build()); @@ -79,14 +79,14 @@ static void setup() { .build()); VectorStoreRecordDefinition recordDefinition = VectorStoreRecordDefinition.fromFields(fields); - optionsMap.put(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, RedisVectorStoreRecordCollectionOptions.builder() + optionsMap.put(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, RedisJsonVectorStoreRecordCollectionOptions.builder() .withRecordClass(Hotel.class) .withRecordDefinition(recordDefinition) .build()); } - private RedisVectorStoreRecordCollection buildrecordCollection(@Nonnull RedisVectorStoreRecordCollectionOptions options, @Nonnull String collectionName) { - return new RedisVectorStoreRecordCollection<>(new JedisPooled(redisContainer.getRedisURI()), collectionName, RedisVectorStoreRecordCollectionOptions.builder() + private RedisJsonVectorStoreRecordCollection buildrecordCollection(@Nonnull RedisJsonVectorStoreRecordCollectionOptions options, @Nonnull String collectionName) { + return new RedisJsonVectorStoreRecordCollection<>(new JedisPooled(redisContainer.getRedisURI()), collectionName, RedisJsonVectorStoreRecordCollectionOptions.builder() .withRecordClass(options.getRecordClass()) .withVectorStoreRecordMapper(options.getVectorStoreRecordMapper()) .withRecordDefinition(options.getRecordDefinition()) @@ -115,7 +115,7 @@ public void buildrecordCollection(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void createCollectionAsync(RecordCollectionOptions options) { - RedisVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); assertEquals(false, recordCollection.collectionExistsAsync().block()); recordCollection.createCollectionAsync().block(); @@ -124,7 +124,7 @@ public void createCollectionAsync(RecordCollectionOptions options) { @Test public void deleteCollectionAsync() { - RedisVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(RecordCollectionOptions.DEFAULT), "deleteCollectionAsync"); + RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(RecordCollectionOptions.DEFAULT), "deleteCollectionAsync"); assertEquals(false, recordCollection.collectionExistsAsync().block()); recordCollection.createCollectionAsync().block(); @@ -135,7 +135,7 @@ public void deleteCollectionAsync() { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void upsertAndGetRecordAsync(RecordCollectionOptions options) { - RedisVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); for (Hotel hotel : hotels) { @@ -152,7 +152,7 @@ public void upsertAndGetRecordAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getBatchAsync(RecordCollectionOptions options) { - RedisVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); for (Hotel hotel : hotels) { @@ -174,7 +174,7 @@ public void getBatchAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void upsertBatchAsync(RecordCollectionOptions options) { - RedisVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); List keys = recordCollection.upsertBatchAsync(hotels, null).block(); @@ -192,7 +192,7 @@ public void upsertBatchAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void deleteAsync(RecordCollectionOptions options) { - RedisVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -207,7 +207,7 @@ public void deleteAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void deleteBatchAsync(RecordCollectionOptions options) { - RedisVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -226,7 +226,7 @@ public void deleteBatchAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getAsyncWithVectors(RecordCollectionOptions options) { - RedisVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -243,7 +243,7 @@ public void getAsyncWithVectors(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getBatchAsyncWithVectors(RecordCollectionOptions options) { - RedisVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -265,7 +265,7 @@ public void getBatchAsyncWithVectors(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getAsyncWithNoVectors(RecordCollectionOptions options) { - RedisVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -283,7 +283,7 @@ public void getAsyncWithNoVectors(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getBatchAsyncWithNoVectors(RecordCollectionOptions options) { - RedisVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java index 96341053..a8082e9b 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java @@ -1,12 +1,15 @@ package com.microsoft.semantickernel.tests.connectors.memory.redis; +import com.microsoft.semantickernel.connectors.data.redis.RedisStorageType; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStore; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreOptions; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.tests.connectors.memory.Hotel; import com.redis.testcontainers.RedisContainer; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; import redis.clients.jedis.JedisPooled; @@ -21,17 +24,25 @@ @Testcontainers public class RedisVectorStoreTest { @Container - private static final RedisContainer redisContainer = new RedisContainer("redis/redis-stack:latest"); - private static JedisPooled jedis; + private static final RedisContainer redisJsonContainer = new RedisContainer("redis/redis-stack:latest"); + @Container + private static final RedisContainer redisHashSetContainer = new RedisContainer("redis/redis-stack:latest"); - @BeforeAll - public static void setUp() { - jedis = new JedisPooled(redisContainer.getRedisURI()); + public static JedisPooled buildClient(RedisStorageType storageType) { + if (storageType == RedisStorageType.JSON) { + return new JedisPooled(redisJsonContainer.getRedisURI()); + } else { + return new JedisPooled(redisHashSetContainer.getRedisURI()); + } } - @Test - public void getCollectionNamesAsync() { - RedisVectorStore vectorStore = new RedisVectorStore(jedis, new RedisVectorStoreOptions()); + @ParameterizedTest + @EnumSource(RedisStorageType.class) + public void getCollectionNamesAsync(RedisStorageType storageType) { + RedisVectorStore vectorStore = new RedisVectorStore(buildClient(storageType), RedisVectorStoreOptions.builder() + .withStorageType(storageType) + .build()); + List collectionNames = Arrays.asList("collection1", "collection2", "collection3"); for (String collectionName : collectionNames) { diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java index 6cd1db3d..68db6e56 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java @@ -15,14 +15,13 @@ import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; -import java.nio.charset.StandardCharsets; + import java.util.Arrays; -import java.util.Base64; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import com.microsoft.semantickernel.samples.syntaxexamples.memory.AzureAISearch_DataStorage.GitHubFile; + import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import redis.clients.jedis.JedisPooled; diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java index a8303061..903669b3 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java @@ -69,7 +69,8 @@ public Optional> getLastMessage() { if (chatMessageContents.isEmpty()) { return Optional.empty(); } - return Optional.of(((ConcurrentLinkedQueue>)chatMessageContents).peek()); + return Optional + .of(((ConcurrentLinkedQueue>) chatMessageContents).peek()); } /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java index a7b7a7c8..b79616c5 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java @@ -61,10 +61,8 @@ public AzureAISearchVectorStoreRecordCollection getCollection( .createVectorStoreRecordCollection( client, collectionName, - AzureAISearchVectorStoreRecordCollectionOptions.builder() - .withRecordClass(recordClass) - .withRecordDefinition(recordDefinition) - .build()); + recordClass, + recordDefinition); } return new AzureAISearchVectorStoreRecordCollection<>( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java index ec08ba03..41cc1633 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java @@ -2,6 +2,7 @@ package com.microsoft.semantickernel.connectors.data.azureaisearch; import com.azure.search.documents.indexes.SearchIndexAsyncClient; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; /** * Factory for creating Azure AI Search vector store record collections. @@ -13,11 +14,13 @@ public interface AzureAISearchVectorStoreRecordCollectionFactory { * * @param client The Azure AI Search client. * @param collectionName The name of the collection. - * @param options The options for the collection. + * @param recordClass The class type of the record. + * @param recordDefinition The record definition. * @return The new Azure AI Search vector store record collection. */ AzureAISearchVectorStoreRecordCollection createVectorStoreRecordCollection( SearchIndexAsyncClient client, String collectionName, - AzureAISearchVectorStoreRecordCollectionOptions options); + Class recordClass, + VectorStoreRecordDefinition recordDefinition); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java index 66ad995b..51775cd6 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; -import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreRecordCollection; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -92,11 +91,8 @@ public JDBCVectorStoreRecordCollection getCollection( .createVectorStoreRecordCollection( dataSource, collectionName, - JDBCVectorStoreRecordCollectionOptions.builder() - .withRecordClass(recordClass) - .withRecordDefinition(recordDefinition) - .withQueryProvider(this.queryProvider) - .build()); + recordClass, + recordDefinition); } return new JDBCVectorStoreRecordCollection<>( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java index 6cfcdcad..32deb0dc 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java @@ -1,20 +1,27 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; + import javax.sql.DataSource; /** * Factory for creating JDBC vector store record collections. */ public interface JDBCVectorStoreRecordCollectionFactory { + /** * Creates a new JDBC vector store record collection. * - * @param options The options for the collection. + * @param dataSource The JDBC data source. + * @param collectionName The name of the collection. + * @param recordClass The class type of the + * @param recordDefinition The record definition. * @return The new JDBC vector store record collection. */ JDBCVectorStoreRecordCollection createVectorStoreRecordCollection( DataSource dataSource, String collectionName, - JDBCVectorStoreRecordCollectionOptions options); + Class recordClass, + VectorStoreRecordDefinition recordDefinition); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java new file mode 100644 index 00000000..1c034774 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java @@ -0,0 +1,353 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.redis; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.microsoft.semantickernel.data.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; +import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; +import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.json.JSONArray; +import org.json.JSONObject; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; +import redis.clients.jedis.JedisPooled; +import redis.clients.jedis.Pipeline; +import redis.clients.jedis.Response; +import redis.clients.jedis.exceptions.JedisDataException; +import redis.clients.jedis.json.Path2; +import redis.clients.jedis.search.IndexDefinition; +import redis.clients.jedis.search.IndexOptions; +import redis.clients.jedis.search.Schema; + +import javax.annotation.Nonnull; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class RedisHashSetVectorStoreRecordCollection + implements VectorStoreRecordCollection { + + private static final HashSet> supportedKeyTypes = new HashSet<>( + Collections.singletonList( + String.class)); + + private static final HashSet> supportedVectorTypes = new HashSet<>( + Arrays.asList( + List.class, + Collection.class)); + + private final JedisPooled client; + private final String collectionName; + private final RedisHashSetVectorStoreRecordCollectionOptions options; + private final VectorStoreRecordMapper>> vectorStoreRecordMapper; + private final VectorStoreRecordDefinition recordDefinition; + private final String[] dataFields; + private final ObjectMapper objectMapper = new ObjectMapper(); + + /** + * Creates a new instance of the RedisVectorRecordStore. + * + * @param client The Redis client. + * @param collectionName The name of the collection. + * @param options The options for the store. + */ + @SuppressFBWarnings("EI_EXPOSE_REP2") + public RedisHashSetVectorStoreRecordCollection( + @Nonnull JedisPooled client, + @Nonnull String collectionName, + @Nonnull RedisHashSetVectorStoreRecordCollectionOptions options) { + this.client = client; + this.collectionName = collectionName; + this.options = options; + + // If record definition is not provided, create one from the record class + if (options.getRecordDefinition() == null) { + this.recordDefinition = VectorStoreRecordDefinition.fromRecordClass( + options.getRecordClass()); + } else { + this.recordDefinition = options.getRecordDefinition(); + } + + // Validate supported types + VectorStoreRecordDefinition.validateSupportedTypes( + Collections + .singletonList(recordDefinition.getKeyDeclaredField(this.options.getRecordClass())), + supportedKeyTypes); + VectorStoreRecordDefinition.validateSupportedTypes( + recordDefinition.getVectorDeclaredFields(this.options.getRecordClass()), + supportedVectorTypes); + + // If mapper is not provided, set a default one + if (options.getVectorStoreRecordMapper() == null) { + vectorStoreRecordMapper = new RedisHashSetVectorStoreRecordMapper.Builder() + .withRecordClass(options.getRecordClass()) + .withVectorStoreRecordDefinition(recordDefinition) + .build(); + } else { + vectorStoreRecordMapper = options.getVectorStoreRecordMapper(); + } + + // Creates a list of paths to retrieve from Redis when no vectors are requested + // Paths are in the format of $.field + this.dataFields = recordDefinition.getDataFields().stream() + .map(VectorStoreRecordDataField::getName) + .toArray(String[]::new); + } + + /** + * Gets the name of the collection. + * + * @return The name of the collection. + */ + @Override + public String getCollectionName() { + return collectionName; + } + + /** + * Checks if the collection exists in the store. + * + * @return A Mono emitting a boolean indicating if the collection exists. + */ + @Override + public Mono collectionExistsAsync() { + return Mono.fromCallable(() -> { + try { + Map info = this.client.ftInfo(collectionName); + return info != null && !info.isEmpty(); + } catch (Exception e) { + if (!(e instanceof JedisDataException)) { + throw e; + } + return false; + } + }).subscribeOn(Schedulers.boundedElastic()); + } + + /** + * Creates the collection in the store. + * + * @return A Mono representing the completion of the creation operation. + */ + @Override + public Mono> createCollectionAsync() { + return Mono.fromRunnable(() -> { + Schema schema = RedisVectorStoreCollectionCreateMapping + .mapToSchema(recordDefinition.getAllFields()); + + IndexDefinition indexDefinition = new IndexDefinition(IndexDefinition.Type.HASH) + .setPrefixes(collectionName + ":"); + + client.ftCreate( + collectionName, + IndexOptions.defaultOptions().setDefinition(indexDefinition), + schema); + }) + .subscribeOn(Schedulers.boundedElastic()) + .then(Mono.just(this)); + } + + /** + * Creates the collection in the store if it does not exist. + * + * @return A Mono representing the completion of the creation operation. + */ + @Override + public Mono> createCollectionIfNotExistsAsync() { + return collectionExistsAsync().flatMap(exists -> { + if (!exists) { + return createCollectionAsync(); + } + + return Mono.just(this); + }); + } + + /** + * Deletes the collection from the store. + * + * @return A Mono representing the completion of the deletion operation. + */ + @Override + public Mono deleteCollectionAsync() { + return Mono.fromRunnable(() -> client.ftDropIndex(collectionName)) + .subscribeOn(Schedulers.boundedElastic()) + .then(); + } + + private String getRedisKey(String key, String collectionName) { + return options.isPrefixCollectionName() ? collectionName + ":" + key : key; + } + + private Map addDataFieldNames(List result) { + Map dataFields = new HashMap<>(); + for (int i = 0; i < result.size(); i++) { + dataFields.put(this.dataFields[i], result.get(i)); + } + return dataFields; + } + + /** + * Gets a record from the store. + * + * @param key The key of the record to get. + * @param options The options for getting the record. + * @return A Mono emitting the record. + */ + @Override + public Mono getAsync(String key, GetRecordOptions options) { + return getBatchAsync(Collections.singletonList(key), options) + .mapNotNull(records -> { + if (records.isEmpty()) { + return null; + } + + return records.get(0); + }); + } + + /** + * Gets a batch of records from the store. + * + * @param keys The keys of the records to get. + * @param options The options for getting the records. + * @return A Mono emitting a list of records. + */ + @Override + public Mono> getBatchAsync(List keys, + GetRecordOptions options) { + Pipeline pipeline = client.pipelined(); + List>> responses = new ArrayList<>(keys.size()); + keys.forEach(key -> { + String redisKey = getRedisKey(key, collectionName); + + if (options == null || options.includeVectors()) { + // Returns Map with the fields and values + responses.add(new AbstractMap.SimpleEntry<>(key, pipeline.hgetAll(redisKey))); + } else { + // Returns List with the values of the fields + responses + .add(new AbstractMap.SimpleEntry<>(key, pipeline.hmget(redisKey, dataFields))); + } + }); + + return Mono.defer(() -> { + pipeline.sync(); + + try { + return Mono.just(responses.stream() + .map(entry -> { + if (options == null || options.includeVectors()) { + // Results directly in a Map + return this.vectorStoreRecordMapper + .mapStorageModeltoRecord( + new AbstractMap.SimpleEntry<>(entry.getKey(), + (Map) entry.getValue().get())); + } + + // Results in a List with the values of the fields + return this.vectorStoreRecordMapper + .mapStorageModeltoRecord( + new AbstractMap.SimpleEntry<>(entry.getKey(), + addDataFieldNames((List) entry.getValue().get()))); + }) + .collect(Collectors.toList())); + } catch (Exception e) { + return Mono.error(e); + } + }).subscribeOn(Schedulers.boundedElastic()); + } + + /** + * Inserts or updates a record in the store. + * + * @param data The record to upsert. + * @param options The options for upserting the record. + * @return A Mono emitting the key of the upserted record. + */ + @Override + public Mono upsertAsync(Record data, UpsertRecordOptions options) { + Map.Entry> redisObject = this.vectorStoreRecordMapper + .mapRecordToStorageModel(data); + String redisKey = getRedisKey(redisObject.getKey(), collectionName); + + return Mono.fromRunnable(() -> client.hset(redisKey, redisObject.getValue())) + .subscribeOn(Schedulers.boundedElastic()) + .thenReturn(redisObject.getKey()); + } + + /** + * Inserts or updates a batch of records in the store. + * + * @param data The records to upsert. + * @param options The options for upserting the records. + * @return A Mono emitting a collection of keys of the upserted records. + */ + @Override + public Mono> upsertBatchAsync(List data, UpsertRecordOptions options) { + Pipeline pipeline = client.pipelined(); + List keys = new ArrayList<>(data.size()); + + data.forEach(record -> { + Map.Entry> redisObject = this.vectorStoreRecordMapper + .mapRecordToStorageModel(record); + String redisKey = getRedisKey(redisObject.getKey(), collectionName); + + keys.add(redisObject.getKey()); + pipeline.hset(redisKey, redisObject.getValue()); + }); + + return Mono.fromRunnable(pipeline::sync) + .subscribeOn(Schedulers.boundedElastic()) + .thenReturn(keys); + } + + /** + * Deletes a record from the store. + * + * @param key The key of the record to delete. + * @param options The options for deleting the record. + * @return A Mono representing the completion of the deletion operation. + */ + @Override + public Mono deleteAsync(String key, DeleteRecordOptions options) { + String redisKey = getRedisKey(key, collectionName); + + return Mono.fromRunnable(() -> client.del(redisKey)) + .subscribeOn(Schedulers.boundedElastic()) + .then(); + } + + /** + * Deletes a batch of records from the store. + * + * @param strings The keys of the records to delete. + * @param options The options for deleting the records. + * @return A Mono representing the completion of the deletion operation. + */ + @Override + public Mono deleteBatchAsync(List strings, DeleteRecordOptions options) { + Pipeline pipeline = client.pipelined(); + strings.forEach(key -> { + String redisKey = getRedisKey(key, collectionName); + pipeline.del(redisKey); + }); + + return Mono.fromRunnable(pipeline::sync) + .subscribeOn(Schedulers.boundedElastic()) + .then(); + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java new file mode 100644 index 00000000..c4130ccd --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java @@ -0,0 +1,156 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.redis; + +import com.microsoft.semantickernel.data.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Map; +import java.util.Map.Entry; + +public class RedisHashSetVectorStoreRecordCollectionOptions { + private final Class recordClass; + @Nullable + private final VectorStoreRecordMapper>> vectorStoreRecordMapper; + @Nullable + private final VectorStoreRecordDefinition recordDefinition; + private final boolean prefixCollectionName; + + private RedisHashSetVectorStoreRecordCollectionOptions( + @Nonnull Class recordClass, + @Nullable VectorStoreRecordMapper>> vectorStoreRecordMapper, + @Nullable VectorStoreRecordDefinition recordDefinition, + boolean prefixCollectionName) { + this.recordClass = recordClass; + this.vectorStoreRecordMapper = vectorStoreRecordMapper; + this.recordDefinition = recordDefinition; + this.prefixCollectionName = prefixCollectionName; + } + + /** + * Creates a new builder. + * + * @param the record type + * @return the builder + */ + public static Builder builder() { + return new Builder<>(); + } + + /** + * Gets the record class. + * + * @return the record class + */ + public Class getRecordClass() { + return recordClass; + } + + /** + * Gets the record definition. + * + * @return the record definition + */ + @Nullable + public VectorStoreRecordDefinition getRecordDefinition() { + return recordDefinition; + } + + /** + * Gets the vector store record mapper. + * + * @return the vector store record mapper + */ + @Nullable + public VectorStoreRecordMapper>> getVectorStoreRecordMapper() { + return vectorStoreRecordMapper; + } + + /** + * Gets whether to prefix the collection name to the redis key. + * + * @return whether to prefix the collection name to the redis key + */ + public boolean isPrefixCollectionName() { + return prefixCollectionName; + } + + /** + * Builder for {@link RedisHashSetVectorStoreRecordCollectionOptions}. + * + * @param the record type + */ + public static class Builder { + @Nullable + private VectorStoreRecordMapper>> vectorStoreRecordMapper; + @Nullable + private Class recordClass; + @Nullable + private VectorStoreRecordDefinition recordDefinition; + private boolean prefixCollectionName = true; + + /** + * Sets the record class. + * + * @param recordClass the record class + * @return the builder + */ + public Builder withRecordClass(Class recordClass) { + this.recordClass = recordClass; + return this; + } + + /** + * Sets the vector store record mapper. + * + * @param vectorStoreRecordMapper the vector store record mapper + * @return the builder + */ + public Builder withVectorStoreRecordMapper( + VectorStoreRecordMapper>> vectorStoreRecordMapper) { + this.vectorStoreRecordMapper = vectorStoreRecordMapper; + return this; + } + + /** + * Sets the record definition. + * + * @param recordDefinition the record definition + * @return the builder + */ + public Builder withRecordDefinition(VectorStoreRecordDefinition recordDefinition) { + this.recordDefinition = recordDefinition; + return this; + } + + /** + * Sets whether to prefix the collection name to the redis key. + * Default is true. + * + * @param prefixCollectionName whether to prefix the collection name to the redis key + * @return the builder + */ + public Builder withPrefixCollectionName(boolean prefixCollectionName) { + this.prefixCollectionName = prefixCollectionName; + return this; + } + + /** + * Builds the options. + * + * @return the options + */ + public RedisHashSetVectorStoreRecordCollectionOptions build() { + if (recordClass == null) { + throw new IllegalArgumentException("recordClass must be provided"); + } + + return new RedisHashSetVectorStoreRecordCollectionOptions<>( + recordClass, + vectorStoreRecordMapper, + recordDefinition, + prefixCollectionName); + } + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java new file mode 100644 index 00000000..42d5967d --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java @@ -0,0 +1,162 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.redis; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.data.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.exceptions.SKException; + +import javax.annotation.Nullable; +import java.util.AbstractMap; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.function.Function; + +public class RedisHashSetVectorStoreRecordMapper + extends VectorStoreRecordMapper>> { + + private RedisHashSetVectorStoreRecordMapper( + Function>> toStorageModelMapper, + Function>, Record> toRecordMapper) { + super(toStorageModelMapper, toRecordMapper); + } + + /** + * Creates a new builder. + * + * @param the record type + * @return the builder + */ + public static Builder builder() { + return new Builder<>(); + } + + /** + * Creates a new builder. + * + * @param the record type + */ + public static class Builder + implements SemanticKernelBuilder> { + @Nullable + private Class recordClass; + + @Nullable + private VectorStoreRecordDefinition recordDefinition; + + /** + * Sets the record class. + * + * @param recordClass the record class + * @return the builder + */ + public Builder withRecordClass(Class recordClass) { + this.recordClass = recordClass; + return this; + } + + /** + * Sets the vector store record definition. + * + * @param recordDefinition the vector store record definition + * @return the builder + */ + public Builder withVectorStoreRecordDefinition( + VectorStoreRecordDefinition recordDefinition) { + this.recordDefinition = recordDefinition; + return this; + } + + /** + * Builds the {@link RedisHashSetVectorStoreRecordMapper}. + * + * @return the {@link RedisHashSetVectorStoreRecordMapper} + */ + @Override + public RedisHashSetVectorStoreRecordMapper build() { + if (recordClass == null) { + throw new IllegalArgumentException("recordClass is required"); + } + if (recordDefinition == null) { + throw new IllegalArgumentException("vectorStoreRecordDefinition is required"); + } + + ObjectMapper mapper = new ObjectMapper(); + + return new RedisHashSetVectorStoreRecordMapper<>(record -> { + try { + ObjectNode jsonNode = mapper.valueToTree(record); + String key = jsonNode.get(recordDefinition.getKeyField().getName()).asText(); + jsonNode.remove(recordDefinition.getKeyField().getName()); + + Map resultMap = new HashMap<>(); + Iterator> fields = jsonNode.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + if (field.getValue().isTextual()) { + resultMap.put(field.getKey(), field.getValue().asText()); + } else { + resultMap.put(field.getKey(), + mapper.valueToTree(field.getValue()).toString()); + } + } + + return new AbstractMap.SimpleEntry<>(key, resultMap); + } catch (Exception e) { + throw new SKException( + "Failure to serialize object, by default the Redis connector uses Jackson, ensure your model object can be serialized by Jackson, i.e the class is visible, has getters, constructor, annotations etc.", + e); + } + }, storageModel -> { + try { + // Empty map means no record found + if (storageModel.getValue() == null || storageModel.getValue().isEmpty()) { + return null; + } + + ObjectNode jsonNode = mapper.createObjectNode(); + jsonNode.set(recordDefinition.getKeyField().getName(), + mapper.valueToTree(storageModel.getKey())); + + for (VectorStoreRecordDataField field : recordDefinition.getDataFields()) { + jsonNode.put(field.getName(), storageModel.getValue().get(field.getName())); + } + + for (VectorStoreRecordVectorField field : recordDefinition.getVectorFields()) { + String value = storageModel.getValue().get(field.getName()); + + // If vector fields were not requested, skip + if (value == null) { + continue; + } + + Class valueType = recordClass.getDeclaredField(field.getName()) + .getType(); + + if (valueType.equals(String.class)) { + jsonNode.put(field.getName(), value); + } else { + // Convert the String stored in Redis back to the correct type and then put the JSON node + jsonNode.set(field.getName(), + mapper.valueToTree(mapper.readValue(value, valueType))); + } + } + + return mapper.convertValue(jsonNode, recordClass); + } catch (Exception e) { + throw new SKException( + "Failure to deserialize object, by default the Redis connector uses Jackson, ensure your model object can be serialized by Jackson, i.e the class is visible, has getters, constructor, annotations etc.", + e); + } + }); + } + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java similarity index 97% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollection.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java index b0f8858b..5fc9a73c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java @@ -36,7 +36,7 @@ import redis.clients.jedis.search.IndexOptions; import redis.clients.jedis.search.Schema; -public class RedisVectorStoreRecordCollection +public class RedisJsonVectorStoreRecordCollection implements VectorStoreRecordCollection { private static final HashSet> supportedKeyTypes = new HashSet<>( @@ -50,7 +50,7 @@ public class RedisVectorStoreRecordCollection private final JedisPooled client; private final String collectionName; - private final RedisVectorStoreRecordCollectionOptions options; + private final RedisJsonVectorStoreRecordCollectionOptions options; private final VectorStoreRecordMapper> vectorStoreRecordMapper; private final VectorStoreRecordDefinition recordDefinition; private final Path2[] dataFields; @@ -63,10 +63,10 @@ public class RedisVectorStoreRecordCollection * @param options The options for the store. */ @SuppressFBWarnings("EI_EXPOSE_REP2") - public RedisVectorStoreRecordCollection( + public RedisJsonVectorStoreRecordCollection( @Nonnull JedisPooled client, @Nonnull String collectionName, - @Nonnull RedisVectorStoreRecordCollectionOptions options) { + @Nonnull RedisJsonVectorStoreRecordCollectionOptions options) { this.client = client; this.collectionName = collectionName; this.options = options; @@ -90,7 +90,7 @@ public RedisVectorStoreRecordCollection( // If mapper is not provided, set a default one if (options.getVectorStoreRecordMapper() == null) { - vectorStoreRecordMapper = new RedisVectorStoreRecordMapper.Builder() + vectorStoreRecordMapper = new RedisJsonVectorStoreRecordMapper.Builder() .withKeyFieldName(recordDefinition.getKeyField().getName()) .withRecordClass(options.getRecordClass()) .build(); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java similarity index 93% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionOptions.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java index 93c30707..8fbaf90b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java @@ -8,7 +8,7 @@ import javax.annotation.Nullable; import java.util.Map.Entry; -public class RedisVectorStoreRecordCollectionOptions { +public class RedisJsonVectorStoreRecordCollectionOptions { private final Class recordClass; @Nullable private final VectorStoreRecordMapper> vectorStoreRecordMapper; @@ -16,7 +16,7 @@ public class RedisVectorStoreRecordCollectionOptions { private final VectorStoreRecordDefinition recordDefinition; private final boolean prefixCollectionName; - private RedisVectorStoreRecordCollectionOptions( + private RedisJsonVectorStoreRecordCollectionOptions( @Nonnull Class recordClass, @Nullable VectorStoreRecordMapper> vectorStoreRecordMapper, @Nullable VectorStoreRecordDefinition recordDefinition, @@ -76,7 +76,7 @@ public boolean isPrefixCollectionName() { } /** - * Builder for {@link RedisVectorStoreRecordCollectionOptions}. + * Builder for {@link RedisJsonVectorStoreRecordCollectionOptions}. * * @param the record type */ @@ -140,12 +140,12 @@ public Builder withPrefixCollectionName(boolean prefixCollectionName) { * * @return the options */ - public RedisVectorStoreRecordCollectionOptions build() { + public RedisJsonVectorStoreRecordCollectionOptions build() { if (recordClass == null) { throw new IllegalArgumentException("recordClass must be provided"); } - return new RedisVectorStoreRecordCollectionOptions<>( + return new RedisJsonVectorStoreRecordCollectionOptions<>( recordClass, vectorStoreRecordMapper, recordDefinition, diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java similarity index 89% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordMapper.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java index f33a6b1d..42dabb5b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java @@ -11,10 +11,10 @@ import java.util.function.Function; import javax.annotation.Nullable; -public class RedisVectorStoreRecordMapper +public class RedisJsonVectorStoreRecordMapper extends VectorStoreRecordMapper> { - private RedisVectorStoreRecordMapper( + private RedisJsonVectorStoreRecordMapper( Function> toStorageModelMapper, Function, Record> toRecordMapper) { super(toStorageModelMapper, toRecordMapper); @@ -36,7 +36,7 @@ public static Builder builder() { * @param the record type */ public static class Builder - implements SemanticKernelBuilder> { + implements SemanticKernelBuilder> { @Nullable private String keyFieldName; @Nullable @@ -65,12 +65,12 @@ public Builder withRecordClass(Class recordClass) { } /** - * Builds the {@link RedisVectorStoreRecordMapper}. + * Builds the {@link RedisJsonVectorStoreRecordMapper}. * - * @return the {@link RedisVectorStoreRecordMapper} + * @return the {@link RedisJsonVectorStoreRecordMapper} */ @Override - public RedisVectorStoreRecordMapper build() { + public RedisJsonVectorStoreRecordMapper build() { if (keyFieldName == null) { throw new IllegalArgumentException("keyFieldName is required"); } @@ -79,7 +79,7 @@ public RedisVectorStoreRecordMapper build() { } ObjectMapper mapper = new ObjectMapper(); - return new RedisVectorStoreRecordMapper<>(record -> { + return new RedisJsonVectorStoreRecordMapper<>(record -> { try { ObjectNode jsonNode = mapper.valueToTree(record); String key = jsonNode.get(keyFieldName).asText(); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisStorageType.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisStorageType.java new file mode 100644 index 00000000..c2f7d753 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisStorageType.java @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.redis; + +/** + * The storage type for the Redis vector store. + */ +public enum RedisStorageType { + /** + * Redis storage with JSON module. + */ + JSON, + /** + * Redis storage with hash set. + */ + HASH_SET +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java index 02f728f0..4ff45f77 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java @@ -55,7 +55,7 @@ public VectorStoreRecordCollection getCollection( * @param recordDefinition The record definition. * @return The collection. */ - public RedisVectorStoreRecordCollection getCollection( + public VectorStoreRecordCollection getCollection( @Nonnull String collectionName, @Nonnull Class recordClass, @Nullable VectorStoreRecordDefinition recordDefinition) { @@ -65,17 +65,27 @@ public RedisVectorStoreRecordCollection getCollection( .createVectorStoreRecordCollection( client, collectionName, - RedisVectorStoreRecordCollectionOptions.builder() - .withRecordClass(recordClass) - .withRecordDefinition(recordDefinition) - .build()); + recordClass, + recordDefinition); } - return new RedisVectorStoreRecordCollection<>(client, collectionName, - RedisVectorStoreRecordCollectionOptions.builder() - .withRecordClass(recordClass) - .withRecordDefinition(recordDefinition) - .build()); + if (options.getStorageType() == RedisStorageType.JSON) { + return new RedisJsonVectorStoreRecordCollection<>( + client, + collectionName, + RedisJsonVectorStoreRecordCollectionOptions.builder() + .withRecordClass(recordClass) + .withRecordDefinition(recordDefinition) + .build()); + } else { + return new RedisHashSetVectorStoreRecordCollection<>( + client, + collectionName, + RedisHashSetVectorStoreRecordCollectionOptions.builder() + .withRecordClass(recordClass) + .withRecordDefinition(recordDefinition) + .build()); + } } /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreOptions.java index 9b70fb49..d7ba95fa 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreOptions.java @@ -8,13 +8,18 @@ public class RedisVectorStoreOptions { @Nullable private final RedisVectorStoreRecordCollectionFactory vectorStoreRecordCollectionFactory; + @Nonnull + private final RedisStorageType storageType; + /** * Creates a new instance of the Redis vector store options. * * @param vectorStoreRecordCollectionFactory The vector store record collection factory. */ public RedisVectorStoreOptions( + @Nonnull RedisStorageType storageType, @Nullable RedisVectorStoreRecordCollectionFactory vectorStoreRecordCollectionFactory) { + this.storageType = storageType; this.vectorStoreRecordCollectionFactory = vectorStoreRecordCollectionFactory; } @@ -22,7 +27,7 @@ public RedisVectorStoreOptions( * Creates a new instance of the Redis vector store options. */ public RedisVectorStoreOptions() { - this(null); + this(RedisStorageType.JSON, null); } /** @@ -44,12 +49,24 @@ public RedisVectorStoreRecordCollectionFactory getVectorStoreRecordCollectionFac return vectorStoreRecordCollectionFactory; } + /** + * Gets the storage type. + * + * @return the storage type + */ + @Nonnull + public RedisStorageType getStorageType() { + return storageType; + } + /** * Builder for Redis vector store options. */ public static class Builder { @Nullable private RedisVectorStoreRecordCollectionFactory vectorStoreRecordCollectionFactory; + @Nullable + private RedisStorageType storageType; /** * Sets the vector store record collection factory. @@ -63,13 +80,28 @@ public Builder withVectorStoreRecordCollectionFactory( return this; } + /** + * Sets the storage type. + * + * @param storageType The storage type. + * @return The updated builder instance. + */ + public Builder withStorageType(RedisStorageType storageType) { + this.storageType = storageType; + return this; + } + /** * Builds the options. * * @return The options. */ public RedisVectorStoreOptions build() { - return new RedisVectorStoreOptions(vectorStoreRecordCollectionFactory); + if (storageType == null) { + throw new IllegalArgumentException("storageType is required"); + } + + return new RedisVectorStoreOptions(storageType, vectorStoreRecordCollectionFactory); } } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java index df1e7544..3834f163 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java @@ -1,7 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; -import com.azure.search.documents.indexes.SearchIndexAsyncClient; +import com.microsoft.semantickernel.data.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import redis.clients.jedis.JedisPooled; /** @@ -14,11 +15,13 @@ public interface RedisVectorStoreRecordCollectionFactory { * * @param client The Redis client. * @param collectionName The name of the collection. - * @param options The options for the collection. + * @param recordClass The class type of the record. + * @param recordDefinition The record definition. * @return The collection. */ - RedisVectorStoreRecordCollection createVectorStoreRecordCollection( + VectorStoreRecordCollection createVectorStoreRecordCollection( JedisPooled client, String collectionName, - RedisVectorStoreRecordCollectionOptions options); + Class recordClass, + VectorStoreRecordDefinition recordDefinition); } From 5d9448a91328e0ea911a376a883a4ee919f26091 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Fri, 16 Aug 2024 15:47:33 -0700 Subject: [PATCH 003/277] Add Postgres vector index support --- .../tests/connectors/memory/Hotel.java | 6 ++ ...rchVectorStoreCollectionCreateMapping.java | 2 +- .../PostgreSQLVectorDistanceFunction.java | 42 ++++++++++++ .../postgres/PostgreSQLVectorIndexKind.java | 33 ++++++++++ .../PostgreSQLVectorStoreQueryProvider.java | 65 +++++++++++++++---- ...disVectorStoreCollectionCreateMapping.java | 2 +- .../recorddefinition/DistanceFunction.java | 15 ++++- .../data/recorddefinition/IndexKind.java | 2 +- 8 files changed, 150 insertions(+), 17 deletions(-) create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java index ad10ad64..599e43ab 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java @@ -17,6 +17,8 @@ public class Hotel { private final String description; @VectorStoreRecordVectorAttribute(dimensions = 3) private final List descriptionEmbedding; + @VectorStoreRecordVectorAttribute(dimensions = 3, indexKind = "hnsw", distanceFunction = "cosine") + private final List additionalEmbedding; @VectorStoreRecordDataAttribute private double rating; @@ -30,6 +32,7 @@ public Hotel(String id, String name, int code, String description, List d this.code = code; this.description = description; this.descriptionEmbedding = descriptionEmbedding; + this.additionalEmbedding = descriptionEmbedding; this.rating = rating; } @@ -52,6 +55,9 @@ public String getDescription() { public List getDescriptionEmbedding() { return descriptionEmbedding; } + public List getAdditionalEmbedding() { + return additionalEmbedding; + } public double getRating() { return rating; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java index c57e6f9d..0530ac81 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java @@ -34,7 +34,7 @@ private static VectorSearchAlgorithmMetric getAlgorithmMetric( } switch (vectorField.getDistanceFunction()) { - case COSINE_SIMILARITY: + case COSINE: return VectorSearchAlgorithmMetric.COSINE; case DOT_PRODUCT: return VectorSearchAlgorithmMetric.DOT_PRODUCT; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java new file mode 100644 index 00000000..7c0ba30e --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.postgres; + +import com.microsoft.semantickernel.data.recorddefinition.DistanceFunction; + +public enum PostgreSQLVectorDistanceFunction { + L2("vector_l2_ops", "<->"), COSINE("vector_cosine_ops", "<=>"), INNER_PRODUCT("vector_ip_ops", + "<#>"); + + private final String value; + private final String operator; + + PostgreSQLVectorDistanceFunction(String value, String operator) { + this.value = value; + this.operator = operator; + } + + public String getValue() { + return value; + } + + public String getOperator() { + return operator; + } + + public static PostgreSQLVectorDistanceFunction fromDistanceFunction(DistanceFunction function) { + if (function == null) { + return null; + } + + switch (function) { + case EUCLIDEAN: + return L2; + case COSINE: + return COSINE; + case DOT_PRODUCT: + return INNER_PRODUCT; + default: + throw new IllegalArgumentException("Unsupported distance function: " + function); + } + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java new file mode 100644 index 00000000..9983df0f --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.postgres; + +import com.microsoft.semantickernel.data.recorddefinition.IndexKind; + +public enum PostgreSQLVectorIndexKind { + HNSW("hnsw"), IVFFLAT("ivfflat"); + + private final String value; + + PostgreSQLVectorIndexKind(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + public static PostgreSQLVectorIndexKind fromIndexKind(IndexKind indexKind) { + if (indexKind == null) { + return null; + } + + switch (indexKind) { + case HNSW: + return HNSW; + case FLAT: + return IVFFLAT; + default: + throw new IllegalArgumentException("Unsupported index kind: " + indexKind); + } + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index d9d5deff..e91546d6 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -5,6 +5,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreDefaultQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.recorddefinition.DistanceFunction; +import com.microsoft.semantickernel.data.recorddefinition.IndexKind; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; @@ -19,6 +21,7 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collection; @@ -155,6 +158,29 @@ private String getColumnNamesAndTypesForVectorFields(List recordClass, VectorStoreRecordDefinition recordDefinition) { Field keyDeclaredField = recordDefinition.getKeyDeclaredField(recordClass); List dataDeclaredFields = recordDefinition.getDataDeclaredFields(recordClass); - String createStorageTable = "CREATE TABLE IF NOT EXISTS " - + getCollectionTableName(collectionName) - + " (" + keyDeclaredField.getName() + " VARCHAR(255) PRIMARY KEY, " - + getColumnNamesAndTypes(dataDeclaredFields, supportedDataTypes) + ", " - + getColumnNamesAndTypesForVectorFields(recordDefinition.getVectorFields(), recordClass) - + ");"; - - String insertCollectionQuery = "INSERT INTO " + validateSQLidentifier(collectionsTable) - + " (collectionId) VALUES (?)"; + List vectorFields = recordDefinition.getVectorFields(); try (Connection connection = dataSource.getConnection(); - PreparedStatement createTable = connection.prepareStatement(createStorageTable)) { - createTable.execute(); + Statement createTableAndIndexes = connection.createStatement()) { + + String createStorageTable = "CREATE TABLE IF NOT EXISTS " + + getCollectionTableName(collectionName) + + " (" + keyDeclaredField.getName() + " VARCHAR(255) PRIMARY KEY, " + + getColumnNamesAndTypes(dataDeclaredFields, supportedDataTypes) + ", " + + getColumnNamesAndTypesForVectorFields(vectorFields, recordClass) + + ");"; + + createTableAndIndexes.addBatch(createStorageTable); + for (VectorStoreRecordVectorField vectorField : vectorFields) { + String createVectorIndex = createIndexForVectorField(collectionName, vectorField); + + if (createVectorIndex != null) { + createTableAndIndexes.addBatch(createVectorIndex); + } + } + + createTableAndIndexes.executeBatch(); } catch (SQLException e) { throw new SKException("Failed to create collection", e); } + String insertCollectionQuery = "INSERT INTO " + validateSQLidentifier(collectionsTable) + + " (collectionId) VALUES (?)"; + try (Connection connection = dataSource.getConnection(); PreparedStatement insert = connection.prepareStatement(insertCollectionQuery)) { insert.setObject(1, collectionName); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java index fe044c73..8e059fd2 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java @@ -35,7 +35,7 @@ private static String getAlgorithmMetric( } switch (vectorField.getDistanceFunction()) { - case COSINE_SIMILARITY: + case COSINE: return RedisVectorDistanceMetric.COSINE; case DOT_PRODUCT: return RedisVectorDistanceMetric.DOT_PRODUCT; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/DistanceFunction.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/DistanceFunction.java index b41aa32e..431df13c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/DistanceFunction.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/DistanceFunction.java @@ -2,7 +2,18 @@ package com.microsoft.semantickernel.data.recorddefinition; public enum DistanceFunction { - COSINE_SIMILARITY("cosineSimilarity"), DOT_PRODUCT("dotProduct"), EUCLIDEAN("euclidean"); + /** + * Cosine (angular) similarity function. + */ + COSINE("cosine"), + /** + * Dot product between two vectors. + */ + DOT_PRODUCT("dotProduct"), + /** + * Euclidean distance function. Also known as L2 norm. + */ + EUCLIDEAN("euclidean"); private final String value; @@ -23,7 +34,7 @@ public String getValue() { */ public static DistanceFunction fromString(String text) { if (text == null || text.isEmpty()) { - return COSINE_SIMILARITY; + return null; } for (DistanceFunction b : DistanceFunction.values()) { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/IndexKind.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/IndexKind.java index 62832de2..78f9c7cc 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/IndexKind.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/IndexKind.java @@ -23,7 +23,7 @@ public String getValue() { */ public static IndexKind fromString(String text) { if (text == null || text.isEmpty()) { - return FLAT; + return null; } for (IndexKind b : IndexKind.values()) { From d34db137254be7237026080794f0ed1d015a0067 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 20 Aug 2024 23:50:24 -0700 Subject: [PATCH 004/277] Add different storage name support --- .../tests/connectors/memory/Hotel.java | 13 +- .../memory/jdbc/JDBCVectorStoreTest.java | 1 - .../memory/AzureAISearch_DataStorage.java | 14 ++- ...rchVectorStoreCollectionCreateMapping.java | 12 +- ...reAISearchVectorStoreRecordCollection.java | 13 +- .../JDBCVectorStoreDefaultQueryProvider.java | 47 +++---- .../jdbc/JDBCVectorStoreQueryProvider.java | 7 +- .../jdbc/JDBCVectorStoreRecordCollection.java | 7 +- .../jdbc/JDBCVectorStoreRecordMapper.java | 70 +++++------ .../mysql/MySQLVectorStoreQueryProvider.java | 59 +++++---- .../PostgreSQLVectorStoreQueryProvider.java | 116 +++++++++--------- .../PostgreSQLVectorStoreRecordMapper.java | 64 +++++----- ...disHashSetVectorStoreRecordCollection.java | 12 +- .../RedisHashSetVectorStoreRecordMapper.java | 20 +-- .../RedisJsonVectorStoreRecordCollection.java | 9 +- .../RedisJsonVectorStoreRecordMapper.java | 22 +++- ...disVectorStoreCollectionCreateMapping.java | 14 ++- .../VolatileVectorStoreRecordCollection.java | 10 +- .../VectorStoreRecordDataAttribute.java | 2 + .../VectorStoreRecordDataField.java | 37 ++---- .../VectorStoreRecordDefinition.java | 79 +++++------- .../VectorStoreRecordField.java | 48 +++++++- .../VectorStoreRecordKeyField.java | 14 ++- .../VectorStoreRecordVectorField.java | 10 +- 24 files changed, 366 insertions(+), 334 deletions(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java index ad10ad64..b5d81e97 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java @@ -1,5 +1,7 @@ package com.microsoft.semantickernel.tests.connectors.memory; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; @@ -13,8 +15,10 @@ public class Hotel { private final String name; @VectorStoreRecordDataAttribute private final int code; + @JsonProperty("summary") @VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "descriptionEmbedding") private final String description; + @JsonProperty("summaryVector") @VectorStoreRecordVectorAttribute(dimensions = 3) private final List descriptionEmbedding; @VectorStoreRecordDataAttribute @@ -24,7 +28,14 @@ public Hotel() { this(null, null, 0, null, null, 0.0); } - public Hotel(String id, String name, int code, String description, List descriptionEmbedding, double rating) { + @JsonCreator + public Hotel( + @JsonProperty("id") String id, + @JsonProperty("name") String name, + @JsonProperty("code") int code, + @JsonProperty("summary") String description, + @JsonProperty("summaryVector") List descriptionEmbedding, + @JsonProperty("rating") double rating) { this.id = id; this.name = name; this.code = code; diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java index 8c2fbfd0..6c422c9c 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java @@ -16,7 +16,6 @@ import org.testcontainers.junit.jupiter.Testcontainers; import org.testcontainers.utility.DockerImageName; -import javax.annotation.Nonnull; import javax.sql.DataSource; import java.util.Arrays; import java.util.List; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java index 03cc93e3..3634181c 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java @@ -10,6 +10,8 @@ import com.azure.core.util.TracingOptions; import com.azure.search.documents.indexes.SearchIndexAsyncClient; import com.azure.search.documents.indexes.SearchIndexClientBuilder; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStore; import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreOptions; @@ -45,7 +47,7 @@ public class AzureAISearch_DataStorage { private static final int EMBEDDING_DIMENSIONS = 1536; static class GitHubFile { - + @JsonProperty("StorageId") // Set a different name for the storage field @VectorStoreRecordKeyAttribute() private final String id; @VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "embedding") @@ -60,10 +62,10 @@ public GitHubFile() { } public GitHubFile( - String id, - String description, - String link, - List embedding) { + @JsonProperty("StorageId") String id, + @JsonProperty("description") String description, + @JsonProperty("link") String link, + @JsonProperty("embedding") List embedding) { this.id = id; this.description = description; this.link = link; @@ -120,7 +122,7 @@ public static void dataStorageWithAzureAISearch( .withOptions(new AzureAISearchVectorStoreOptions()) .build(); - String collectionName = "skgithubfiles"; + String collectionName = "skgithubfiles2"; var collection = azureAISearchVectorStore.getCollection( collectionName, GitHubFile.class, diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java index c57e6f9d..af7c0d41 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java @@ -20,11 +20,11 @@ public class AzureAISearchVectorStoreCollectionCreateMapping { private static String getVectorSearchProfileName(VectorStoreRecordVectorField vectorField) { - return vectorField.getName() + "Profile"; + return vectorField.getEffectiveStorageName() + "Profile"; } private static String getAlgorithmConfigName(VectorStoreRecordVectorField vectorField) { - return vectorField.getName() + "AlgorithmConfig"; + return vectorField.getEffectiveStorageName() + "AlgorithmConfig"; } private static VectorSearchAlgorithmMetric getAlgorithmMetric( @@ -68,7 +68,7 @@ private static VectorSearchAlgorithmConfiguration getAlgorithmConfig( } public static SearchField mapKeyField(VectorStoreRecordKeyField keyField) { - return new SearchField(keyField.getName(), SearchFieldDataType.STRING) + return new SearchField(keyField.getEffectiveStorageName(), SearchFieldDataType.STRING) .setKey(true) .setFilterable(true); } @@ -76,16 +76,16 @@ public static SearchField mapKeyField(VectorStoreRecordKeyField keyField) { public static SearchField mapDataField(VectorStoreRecordDataField dataField) { if (dataField.getFieldType() == null) { throw new IllegalArgumentException( - "Field type is required: " + dataField.getName()); + "Field type is required: " + dataField.getEffectiveStorageName()); } - return new SearchField(dataField.getName(), + return new SearchField(dataField.getEffectiveStorageName(), getSearchFieldDataType(dataField.getFieldType())) .setFilterable(dataField.isFilterable()); } public static SearchField mapVectorField(VectorStoreRecordVectorField vectorField) { - return new SearchField(vectorField.getName(), + return new SearchField(vectorField.getEffectiveStorageName(), SearchFieldDataType.collection(SearchFieldDataType.SINGLE)) .setSearchable(true) .setVectorSearchDimensions(vectorField.getDimensions()) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index 5937d928..84a91f2b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -90,20 +90,19 @@ public AzureAISearchVectorStoreRecordCollection( // Validate supported types VectorStoreRecordDefinition.validateSupportedTypes( - Collections - .singletonList(recordDefinition.getKeyDeclaredField(this.options.getRecordClass())), + Collections.singletonList(recordDefinition.getKeyField()), supportedKeyTypes); VectorStoreRecordDefinition.validateSupportedTypes( - recordDefinition.getDataDeclaredFields(this.options.getRecordClass()), + new ArrayList<>(recordDefinition.getDataFields()), supportedDataTypes); VectorStoreRecordDefinition.validateSupportedTypes( - recordDefinition.getVectorDeclaredFields(this.options.getRecordClass()), + new ArrayList<>(recordDefinition.getVectorFields()), supportedVectorTypes); // Add non-vector fields to the list - nonVectorFields.add(this.recordDefinition.getKeyField().getName()); + nonVectorFields.add(this.recordDefinition.getKeyField().getEffectiveStorageName()); nonVectorFields.addAll(this.recordDefinition.getDataFields().stream() - .map(VectorStoreRecordDataField::getName) + .map(VectorStoreRecordDataField::getEffectiveStorageName) .collect(Collectors.toList())); } @@ -256,7 +255,7 @@ public Mono deleteBatchAsync(List keys, DeleteRecordOptions option return client.deleteDocuments(keys.stream().map(key -> { SearchDocument document = new SearchDocument(); - document.put(this.recordDefinition.getKeyField().getName(), key); + document.put(this.recordDefinition.getKeyField().getEffectiveStorageName(), key); return document; }).collect(Collectors.toList())).then(); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java index f1795083..455aed49 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java @@ -12,7 +12,6 @@ import javax.annotation.Nonnull; import javax.sql.DataSource; -import java.lang.reflect.Field; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -30,9 +29,9 @@ public class JDBCVectorStoreDefaultQueryProvider implements JDBCVectorStoreQueryProvider { - private Map, String> supportedKeyTypes; - private Map, String> supportedDataTypes; - private Map, String> supportedVectorTypes; + private final Map, String> supportedKeyTypes; + private final Map, String> supportedDataTypes; + private final Map, String> supportedVectorTypes; private final DataSource dataSource; private final String collectionsTable; private final String prefixForCollectionTables; @@ -94,7 +93,7 @@ protected String getWildcardString(int wildcards) { * @return the formatted query columns */ protected String getQueryColumnsFromFields(List fields) { - return fields.stream().map(VectorStoreRecordField::getName) + return fields.stream().map(VectorStoreRecordField::getEffectiveStorageName) .collect(Collectors.joining(", ")); } @@ -104,9 +103,10 @@ protected String getQueryColumnsFromFields(List fields) * @param types the types * @return the formatted column names and types */ - protected String getColumnNamesAndTypes(List fields, Map, String> types) { + protected String getColumnNamesAndTypes(List fields, + Map, String> types) { List columns = fields.stream() - .map(field -> field.getName() + " " + types.get(field.getType())) + .map(field -> field.getEffectiveStorageName() + " " + types.get(field.getFieldType())) .collect(Collectors.toList()); return String.join(", ", columns); @@ -169,20 +169,20 @@ public void prepareVectorStore() { /** * Checks if the types of the record class fields are supported. * - * @param recordClass the record class * @param recordDefinition the record definition * @throws IllegalArgumentException if the types are not supported */ @Override - public void validateSupportedTypes(Class recordClass, - VectorStoreRecordDefinition recordDefinition) { + public void validateSupportedTypes(VectorStoreRecordDefinition recordDefinition) { + VectorStoreRecordDefinition.validateSupportedTypes( - Collections.singletonList(recordDefinition.getKeyDeclaredField(recordClass)), + Collections.singletonList(recordDefinition.getKeyField()), getSupportedKeyTypes().keySet()); VectorStoreRecordDefinition.validateSupportedTypes( - recordDefinition.getDataDeclaredFields(recordClass), getSupportedDataTypes().keySet()); + new ArrayList<>(recordDefinition.getDataFields()), + getSupportedDataTypes().keySet()); VectorStoreRecordDefinition.validateSupportedTypes( - recordDefinition.getVectorDeclaredFields(recordClass), + new ArrayList<>(recordDefinition.getVectorFields()), getSupportedVectorTypes().keySet()); } @@ -212,23 +212,24 @@ public boolean collectionExists(String collectionName) { * Creates a collection. * * @param collectionName the collection name - * @param recordClass the record class * @param recordDefinition the record definition * @throws SKException if an error occurs while creating the collection */ @Override @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") // SQL query is generated dynamically with valid identifiers - public void createCollection(String collectionName, Class recordClass, + public void createCollection(String collectionName, VectorStoreRecordDefinition recordDefinition) { - Field keyDeclaredField = recordDefinition.getKeyDeclaredField(recordClass); - List dataDeclaredFields = recordDefinition.getDataDeclaredFields(recordClass); - List vectorDeclaredFields = recordDefinition.getVectorDeclaredFields(recordClass); String createStorageTable = "CREATE TABLE IF NOT EXISTS " + getCollectionTableName(collectionName) - + " (" + keyDeclaredField.getName() + " VARCHAR(255) PRIMARY KEY, " - + getColumnNamesAndTypes(dataDeclaredFields, getSupportedDataTypes()) + ", " - + getColumnNamesAndTypes(vectorDeclaredFields, getSupportedVectorTypes()) + ");"; + + " (" + recordDefinition.getKeyField().getEffectiveStorageName() + + " VARCHAR(255) PRIMARY KEY, " + + getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), + getSupportedDataTypes()) + + ", " + + getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getVectorFields()), + getSupportedVectorTypes()) + + ");"; String insertCollectionQuery = "INSERT INTO " + validateSQLidentifier(collectionsTable) + " (collectionId) VALUES (?)"; @@ -329,7 +330,7 @@ public List getRecords(String collectionName, List keys String query = "SELECT " + getQueryColumnsFromFields(fields) + " FROM " + getCollectionTableName(collectionName) - + " WHERE " + recordDefinition.getKeyField().getName() + + " WHERE " + recordDefinition.getKeyField().getEffectiveStorageName() + " IN (" + getWildcardString(keys.size()) + ")"; try (Connection connection = dataSource.getConnection(); @@ -371,7 +372,7 @@ public void upsertRecords(String collectionName, List records, public void deleteRecords(String collectionName, List keys, VectorStoreRecordDefinition recordDefinition, DeleteRecordOptions options) { String query = "DELETE FROM " + getCollectionTableName(collectionName) - + " WHERE " + recordDefinition.getKeyField().getName() + + " WHERE " + recordDefinition.getKeyField().getEffectiveStorageName() + " IN (" + getWildcardString(keys.size()) + ")"; try (Connection connection = dataSource.getConnection(); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index 6009b885..cdda7d9a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -57,10 +57,9 @@ public interface JDBCVectorStoreQueryProvider { /** * Checks if the types of the record class fields are supported. * - * @param recordClass the record class * @param recordDefinition the record definition */ - void validateSupportedTypes(Class recordClass, VectorStoreRecordDefinition recordDefinition); + void validateSupportedTypes(VectorStoreRecordDefinition recordDefinition); /** * Checks if a collection exists. @@ -74,11 +73,9 @@ public interface JDBCVectorStoreQueryProvider { * Creates a collection. * * @param collectionName the collection name - * @param recordClass the record class * @param recordDefinition the record definition */ - void createCollection(String collectionName, Class recordClass, - VectorStoreRecordDefinition recordDefinition); + void createCollection(String collectionName, VectorStoreRecordDefinition recordDefinition); /** * Deletes a collection. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java index 44fc2338..f5bb7e44 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -87,7 +87,7 @@ public JDBCVectorStoreRecordCollection( } // Check if the types are supported - queryProvider.validateSupportedTypes(options.getRecordClass(), recordDefinition); + queryProvider.validateSupportedTypes(recordDefinition); } /** @@ -122,8 +122,7 @@ public Mono collectionExistsAsync() { @Override public Mono> createCollectionAsync() { return Mono.fromRunnable( - () -> queryProvider.createCollection(this.collectionName, options.getRecordClass(), - recordDefinition)) + () -> queryProvider.createCollection(this.collectionName, recordDefinition)) .subscribeOn(Schedulers.boundedElastic()) .then(Mono.just(this)); } @@ -200,7 +199,7 @@ public Mono> getBatchAsync(List keys, GetRecordOptions opti protected String getKeyFromRecord(Record data) { try { Field keyField = data.getClass() - .getDeclaredField(recordDefinition.getKeyField().getName()); + .getDeclaredField(recordDefinition.getKeyField().getEffectiveStorageName()); keyField.setAccessible(true); return (String) keyField.get(data); } catch (NoSuchFieldException | IllegalAccessException e) { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java index 6eff0c7d..da53a99e 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java @@ -2,19 +2,19 @@ package com.microsoft.semantickernel.connectors.data.jdbc; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.exceptions.SKException; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.exceptions.SKException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.sql.ResultSetMetaData; import java.util.List; -import java.lang.reflect.Constructor; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; import java.sql.ResultSet; import java.sql.SQLException; import java.util.function.Function; @@ -53,6 +53,7 @@ public static class Builder implements SemanticKernelBuilder> { private Class recordClass; private VectorStoreRecordDefinition vectorStoreRecordDefinition; + private ObjectMapper objectMapper = new ObjectMapper(); /** * Sets the record class. @@ -77,6 +78,18 @@ public Builder withVectorStoreRecordDefinition( return this; } + /** + * Sets the object mapper. + * + * @param objectMapper the object mapper + * @return the builder + */ + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withObjectMapper(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + return this; + } + /** * Builds the {@link JDBCVectorStoreRecordMapper}. * @@ -93,12 +106,10 @@ public JDBCVectorStoreRecordMapper build() { return new JDBCVectorStoreRecordMapper<>( resultSet -> { try { - Constructor constructor = recordClass.getDeclaredConstructor(); - constructor.setAccessible(true); - Record record = (Record) constructor.newInstance(); + // Create an ObjectNode to hold the values + ObjectNode objectNode = objectMapper.createObjectNode(); // Select fields from the record definition. - // Check if vector fields are present in the result set. List fields; ResultSetMetaData metaData = resultSet.getMetaData(); if (metaData.getColumnCount() == vectorStoreRecordDefinition.getAllFields() @@ -109,40 +120,27 @@ public JDBCVectorStoreRecordMapper build() { } for (VectorStoreRecordField field : fields) { - Object value = resultSet.getObject(field.getName()); - Field recordField = recordClass.getDeclaredField(field.getName()); - recordField.setAccessible(true); + Object value = resultSet.getObject(field.getEffectiveStorageName()); - // If the field is a vector field, deserialize the JSON string if (field instanceof VectorStoreRecordVectorField) { - Class vectorType = recordField.getType(); - - // If the vector type is a string, set the value directly - if (vectorType.equals(String.class)) { - recordField.set(record, value); - } else { - // Deserialize the JSON string to the vector type - recordField.set(record, - new ObjectMapper().readValue((String) value, vectorType)); + Class vectorType = field.getFieldType(); + + // If the vector field is other than String, deserialize it from the JSON string + if (!vectorType.equals(String.class)) { + value = objectMapper.readValue((String) value, vectorType); } - } else { - recordField.set(record, value); } + + JsonNode genericNode = objectMapper.valueToTree(value); + objectNode.set(field.getEffectiveStorageName(), genericNode); } - return record; - } catch (NoSuchMethodException e) { - throw new SKException("Default constructor not found.", e); - } catch (InstantiationException | InvocationTargetException e) { - throw new SKException(String.format( - "SK cannot instantiate %s. A custom mapper is required.", - recordClass.getName()), e); - } catch (JsonProcessingException e) { - throw new SKException(String.format( - "SK cannot deserialize %s. A custom mapper is required.", - recordClass.getName()), e); - } catch (SQLException | NoSuchFieldException | IllegalAccessException e) { - throw new RuntimeException(e); + // Deserialize the object node to the record class + return objectMapper.convertValue(objectNode, recordClass); + } catch (SQLException | JsonProcessingException e) { + throw new SKException( + "Failure to serialize object, by default the JDBC connector uses Jackson, ensure your model object can be serialized by Jackson, i.e the class is visible, has getters, constructor, annotations etc.", + e); } }); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java index ff19017c..5eb61805 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java @@ -2,19 +2,18 @@ package com.microsoft.semantickernel.connectors.data.mysql; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreDefaultQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.sql.DataSource; -import java.lang.reflect.Field; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -24,12 +23,14 @@ public class MySQLVectorStoreQueryProvider extends JDBCVectorStoreDefaultQueryProvider implements JDBCVectorStoreQueryProvider { private final DataSource dataSource; + private final ObjectMapper objectMapper; @SuppressFBWarnings("EI_EXPOSE_REP2") private MySQLVectorStoreQueryProvider(DataSource dataSource, String collectionsTable, - String prefixForCollectionTables) { + String prefixForCollectionTables, ObjectMapper objectMapper) { super(dataSource, collectionsTable, prefixForCollectionTables); this.dataSource = dataSource; + this.objectMapper = objectMapper; } /** @@ -42,32 +43,24 @@ public static Builder builder() { private void setStatementValues(PreparedStatement statement, Object record, List fields) { + JsonNode jsonNode = objectMapper.valueToTree(record); + for (int i = 0; i < fields.size(); ++i) { VectorStoreRecordField field = fields.get(i); try { - Field recordField = record.getClass().getDeclaredField(field.getName()); - recordField.setAccessible(true); - Object value = recordField.get(record); - - if (field instanceof VectorStoreRecordKeyField) { - statement.setObject(i + 1, (String) value); - } else if (field instanceof VectorStoreRecordVectorField) { - Class vectorType = record.getClass().getDeclaredField(field.getName()) - .getType(); - - // If the vector field is other than String, serialize it to JSON - if (vectorType.equals(String.class)) { - statement.setObject(i + 1, value); - } else { - // Serialize the vector to JSON - statement.setObject(i + 1, new ObjectMapper().writeValueAsString(value)); + JsonNode valueNode = jsonNode.get(field.getEffectiveStorageName()); + + if (field instanceof VectorStoreRecordVectorField) { + // Convert the vector field to a string + if (!field.getFieldType().equals(String.class)) { + statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); + continue; } - } else { - statement.setObject(i + 1, value); } - } catch (NoSuchFieldException | IllegalAccessException | SQLException e) { - throw new SKException("Failed to set statement values", e); - } catch (JsonProcessingException e) { + + statement.setObject(i + 1, + objectMapper.convertValue(valueNode, field.getFieldType())); + } catch (SQLException | JsonProcessingException e) { throw new RuntimeException(e); } } @@ -96,8 +89,8 @@ public void upsertRecords(String collectionName, List records, onDuplicateKeyUpdate.append(", "); } - onDuplicateKeyUpdate.append(field.getName()).append(" = VALUES(") - .append(field.getName()).append(")"); + onDuplicateKeyUpdate.append(field.getEffectiveStorageName()).append(" = VALUES(") + .append(field.getEffectiveStorageName()).append(")"); } String query = "INSERT INTO " + getCollectionTableName(collectionName) @@ -123,6 +116,7 @@ public static class Builder private DataSource dataSource; private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; + private ObjectMapper objectMapper = new ObjectMapper(); @SuppressFBWarnings("EI_EXPOSE_REP2") public Builder withDataSource(DataSource dataSource) { @@ -150,13 +144,24 @@ public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { return this; } + /** + * Sets the object mapper. + * @param objectMapper the object mapper + * @return the builder + */ + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withObjectMapper(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + return this; + } + public MySQLVectorStoreQueryProvider build() { if (dataSource == null) { throw new SKException("DataSource is required"); } return new MySQLVectorStoreQueryProvider(dataSource, collectionsTable, - prefixForCollectionTables); + prefixForCollectionTables, objectMapper); } } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index d9d5deff..f8f7f8cc 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -2,6 +2,7 @@ package com.microsoft.semantickernel.connectors.data.postgres; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreDefaultQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; @@ -14,37 +15,35 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.sql.DataSource; -import java.lang.reflect.Field; import java.sql.Connection; import java.sql.PreparedStatement; -import java.sql.ResultSet; import java.sql.SQLException; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; public class PostgreSQLVectorStoreQueryProvider extends JDBCVectorStoreDefaultQueryProvider implements JDBCVectorStoreQueryProvider { - - private Map, String> supportedKeyTypes; - private Map, String> supportedDataTypes; - private Map, String> supportedVectorTypes; + private final Map, String> supportedKeyTypes; + private final Map, String> supportedDataTypes; + private final Map, String> supportedVectorTypes; private final DataSource dataSource; private final String collectionsTable; private final String prefixForCollectionTables; + private final ObjectMapper objectMapper; @SuppressFBWarnings("EI_EXPOSE_REP2") private PostgreSQLVectorStoreQueryProvider(DataSource dataSource, String collectionsTable, - String prefixForCollectionTables) { + String prefixForCollectionTables, ObjectMapper objectMapper) { super(dataSource, collectionsTable, prefixForCollectionTables); this.dataSource = dataSource; this.collectionsTable = collectionsTable; this.prefixForCollectionTables = prefixForCollectionTables; + this.objectMapper = objectMapper; supportedKeyTypes = new HashMap<>(); supportedKeyTypes.put(String.class, "VARCHAR(255)"); @@ -128,27 +127,22 @@ public void prepareVectorStore() { } } - private String getColumnNamesAndTypesForVectorFields(List fields, - Class recordClass) { + private String getColumnNamesAndTypesForVectorFields( + List fields) { StringBuilder columnNames = new StringBuilder(); for (VectorStoreRecordVectorField field : fields) { - try { - Field declaredField = recordClass.getDeclaredField(field.getName()); - if (columnNames.length() > 0) { - columnNames.append(", "); - } + if (columnNames.length() > 0) { + columnNames.append(", "); + } - if (declaredField.getType().equals(String.class)) { - columnNames.append(field.getName()).append(" ") - .append(supportedVectorTypes.get(String.class)); - } else { - // Get the vector type and dimensions - String type = String.format(supportedVectorTypes.get(declaredField.getType()), - field.getDimensions()); - columnNames.append(field.getName()).append(" ").append(type); - } - } catch (NoSuchFieldException e) { - throw new RuntimeException(e); + if (field.getFieldType().equals(String.class)) { + columnNames.append(field.getEffectiveStorageName()).append(" ") + .append(supportedVectorTypes.get(String.class)); + } else { + // Get the vector type and dimensions + String type = String.format(supportedVectorTypes.get(field.getFieldType()), + field.getDimensions()); + columnNames.append(field.getEffectiveStorageName()).append(" ").append(type); } } @@ -159,22 +153,21 @@ private String getColumnNamesAndTypesForVectorFields(List recordClass, + public void createCollection(String collectionName, VectorStoreRecordDefinition recordDefinition) { - Field keyDeclaredField = recordDefinition.getKeyDeclaredField(recordClass); - List dataDeclaredFields = recordDefinition.getDataDeclaredFields(recordClass); String createStorageTable = "CREATE TABLE IF NOT EXISTS " + getCollectionTableName(collectionName) - + " (" + keyDeclaredField.getName() + " VARCHAR(255) PRIMARY KEY, " - + getColumnNamesAndTypes(dataDeclaredFields, supportedDataTypes) + ", " - + getColumnNamesAndTypesForVectorFields(recordDefinition.getVectorFields(), recordClass) + + " (" + recordDefinition.getKeyField().getStorageName() + " VARCHAR(255) PRIMARY KEY, " + + getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), + supportedDataTypes) + + ", " + + getColumnNamesAndTypesForVectorFields(recordDefinition.getVectorFields()) + ");"; String insertCollectionQuery = "INSERT INTO " + validateSQLidentifier(collectionsTable) @@ -198,32 +191,24 @@ public void createCollection(String collectionName, Class recordClass, private void setStatementValues(PreparedStatement statement, Object record, List fields) { + JsonNode jsonNode = objectMapper.valueToTree(record); + for (int i = 0; i < fields.size(); ++i) { VectorStoreRecordField field = fields.get(i); try { - Field recordField = record.getClass().getDeclaredField(field.getName()); - recordField.setAccessible(true); - Object value = recordField.get(record); - - if (field instanceof VectorStoreRecordKeyField) { - statement.setObject(i + 1, (String) value); - } else if (field instanceof VectorStoreRecordVectorField) { - Class vectorType = record.getClass().getDeclaredField(field.getName()) - .getType(); - - // If the vector field is other than String, serialize it to JSON - if (vectorType.equals(String.class)) { - statement.setObject(i + 1, value); - } else { - // Serialize the vector to JSON - statement.setString(i + 1, new ObjectMapper().writeValueAsString(value)); + JsonNode valueNode = jsonNode.get(field.getEffectiveStorageName()); + + if (field instanceof VectorStoreRecordVectorField) { + // Convert the vector field to a string + if (!field.getFieldType().equals(String.class)) { + statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); + continue; } - } else { - statement.setObject(i + 1, value); } - } catch (NoSuchFieldException | IllegalAccessException | SQLException e) { - throw new SKException("Failed to set statement values", e); - } catch (JsonProcessingException e) { + + statement.setObject(i + 1, + objectMapper.convertValue(valueNode, field.getFieldType())); + } catch (SQLException | JsonProcessingException e) { throw new RuntimeException(e); } } @@ -268,15 +253,16 @@ public void upsertRecords(String collectionName, List records, if (onDuplicateKeyUpdate.length() > 0) { onDuplicateKeyUpdate.append(", "); } - onDuplicateKeyUpdate.append(field.getName()) + onDuplicateKeyUpdate.append(field.getEffectiveStorageName()) .append(" = EXCLUDED.") - .append(field.getName()); + .append(field.getEffectiveStorageName()); } String query = "INSERT INTO " + getCollectionTableName(collectionName) + " (" + getQueryColumnsFromFields(fields) + ")" + " VALUES (" + getWildcardStringWithCast(fields) + ")" - + " ON CONFLICT (" + recordDefinition.getKeyField().getName() + ") DO UPDATE SET " + + " ON CONFLICT (" + recordDefinition.getKeyField().getEffectiveStorageName() + + ") DO UPDATE SET " + onDuplicateKeyUpdate; try (Connection connection = dataSource.getConnection(); @@ -297,6 +283,7 @@ public static class Builder private DataSource dataSource; private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; + private ObjectMapper objectMapper = new ObjectMapper(); @SuppressFBWarnings("EI_EXPOSE_REP2") public PostgreSQLVectorStoreQueryProvider.Builder withDataSource(DataSource dataSource) { @@ -326,13 +313,26 @@ public PostgreSQLVectorStoreQueryProvider.Builder withPrefixForCollectionTables( return this; } + /** + * Sets the object mapper. + * + * @param objectMapper the object mapper + * @return the builder + */ + @SuppressFBWarnings("EI_EXPOSE_REP2") + public PostgreSQLVectorStoreQueryProvider.Builder withObjectMapper( + ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + return this; + } + public PostgreSQLVectorStoreQueryProvider build() { if (dataSource == null) { throw new SKException("DataSource is required"); } return new PostgreSQLVectorStoreQueryProvider(dataSource, collectionsTable, - prefixForCollectionTables); + prefixForCollectionTables, objectMapper); } } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java index 83b821c3..5e3fd188 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java @@ -2,13 +2,16 @@ package com.microsoft.semantickernel.connectors.data.postgres; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.exceptions.SKException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.postgresql.util.PGobject; import java.lang.reflect.Constructor; @@ -47,6 +50,7 @@ public static class Builder implements SemanticKernelBuilder> { private Class recordClass; private VectorStoreRecordDefinition vectorStoreRecordDefinition; + private ObjectMapper objectMapper = new ObjectMapper(); /** * Sets the record class. @@ -71,6 +75,18 @@ public Builder withVectorStoreRecordDefinition( return this; } + /** + * Sets the object mapper. + * + * @param objectMapper the object mapper + * @return the builder + */ + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withObjectMapper(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + return this; + } + /** * Builds the {@link PostgreSQLVectorStoreRecordMapper}. * @@ -87,12 +103,10 @@ public PostgreSQLVectorStoreRecordMapper build() { return new PostgreSQLVectorStoreRecordMapper<>( resultSet -> { try { - Constructor constructor = recordClass.getDeclaredConstructor(); - constructor.setAccessible(true); - Record record = (Record) constructor.newInstance(); + // Create an ObjectNode to hold the values + ObjectNode objectNode = objectMapper.createObjectNode(); // Select fields from the record definition. - // Check if vector fields are present in the result set. List fields; ResultSetMetaData metaData = resultSet.getMetaData(); if (metaData.getColumnCount() == vectorStoreRecordDefinition.getAllFields() @@ -103,42 +117,28 @@ public PostgreSQLVectorStoreRecordMapper build() { } for (VectorStoreRecordField field : fields) { - Object value = resultSet.getObject(field.getName()); - Field recordField = recordClass.getDeclaredField(field.getName()); - recordField.setAccessible(true); + Object value = resultSet.getObject(field.getEffectiveStorageName()); - // If the field is a vector field, deserialize the JSON string if (field instanceof VectorStoreRecordVectorField) { - Class vectorType = recordField.getType(); + Class vectorType = field.getFieldType(); - // If the vector type is a string, set the value directly - if (vectorType.equals(String.class)) { - recordField.set(record, value); - } else { + // If the vector field is other than String, deserialize it from the JSON string + if (!vectorType.equals(String.class)) { // Deserialize the pgvector string to the vector type - PGobject pgObject = (PGobject) value; - recordField.set(record, - new ObjectMapper().readValue(pgObject.getValue(), - vectorType)); + value = objectMapper.readValue(((PGobject) value).getValue(), + vectorType); } - } else { - recordField.set(record, value); } + + JsonNode genericNode = objectMapper.valueToTree(value); + objectNode.set(field.getEffectiveStorageName(), genericNode); } - return record; - } catch (NoSuchMethodException e) { - throw new SKException("Default constructor not found.", e); - } catch (InstantiationException | InvocationTargetException e) { - throw new SKException(String.format( - "SK cannot instantiate %s. A custom mapper is required.", - recordClass.getName()), e); - } catch (JsonProcessingException e) { - throw new SKException(String.format( - "SK cannot deserialize %s. A custom mapper is required.", - recordClass.getName()), e); - } catch (SQLException | NoSuchFieldException | IllegalAccessException e) { - throw new RuntimeException(e); + return objectMapper.treeToValue(objectNode, recordClass); + } catch (SQLException | JsonProcessingException e) { + throw new SKException( + "Failure to serialize object, by default the JDBC connector uses Jackson, ensure your model object can be serialized by Jackson, i.e the class is visible, has getters, constructor, annotations etc.", + e); } }); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java index 1c034774..1b8079e9 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java @@ -1,9 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; @@ -12,15 +10,12 @@ import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import org.json.JSONArray; -import org.json.JSONObject; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; import redis.clients.jedis.JedisPooled; import redis.clients.jedis.Pipeline; import redis.clients.jedis.Response; import redis.clients.jedis.exceptions.JedisDataException; -import redis.clients.jedis.json.Path2; import redis.clients.jedis.search.IndexDefinition; import redis.clients.jedis.search.IndexOptions; import redis.clients.jedis.search.Schema; @@ -83,11 +78,10 @@ public RedisHashSetVectorStoreRecordCollection( // Validate supported types VectorStoreRecordDefinition.validateSupportedTypes( - Collections - .singletonList(recordDefinition.getKeyDeclaredField(this.options.getRecordClass())), + Collections.singletonList(recordDefinition.getKeyField()), supportedKeyTypes); VectorStoreRecordDefinition.validateSupportedTypes( - recordDefinition.getVectorDeclaredFields(this.options.getRecordClass()), + new ArrayList<>(recordDefinition.getVectorFields()), supportedVectorTypes); // If mapper is not provided, set a default one @@ -103,7 +97,7 @@ public RedisHashSetVectorStoreRecordCollection( // Creates a list of paths to retrieve from Redis when no vectors are requested // Paths are in the format of $.field this.dataFields = recordDefinition.getDataFields().stream() - .map(VectorStoreRecordDataField::getName) + .map(VectorStoreRecordDataField::getEffectiveStorageName) .toArray(String[]::new); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java index 42d5967d..a61fed88 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java @@ -8,7 +8,6 @@ import com.microsoft.semantickernel.data.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.exceptions.SKException; @@ -94,8 +93,9 @@ public RedisHashSetVectorStoreRecordMapper build() { return new RedisHashSetVectorStoreRecordMapper<>(record -> { try { ObjectNode jsonNode = mapper.valueToTree(record); - String key = jsonNode.get(recordDefinition.getKeyField().getName()).asText(); - jsonNode.remove(recordDefinition.getKeyField().getName()); + String key = jsonNode + .get(recordDefinition.getKeyField().getEffectiveStorageName()).asText(); + jsonNode.remove(recordDefinition.getKeyField().getEffectiveStorageName()); Map resultMap = new HashMap<>(); Iterator> fields = jsonNode.fields(); @@ -123,29 +123,31 @@ public RedisHashSetVectorStoreRecordMapper build() { } ObjectNode jsonNode = mapper.createObjectNode(); - jsonNode.set(recordDefinition.getKeyField().getName(), + jsonNode.set(recordDefinition.getKeyField().getEffectiveStorageName(), mapper.valueToTree(storageModel.getKey())); for (VectorStoreRecordDataField field : recordDefinition.getDataFields()) { - jsonNode.put(field.getName(), storageModel.getValue().get(field.getName())); + jsonNode.put(field.getEffectiveStorageName(), + storageModel.getValue().get(field.getEffectiveStorageName())); } for (VectorStoreRecordVectorField field : recordDefinition.getVectorFields()) { - String value = storageModel.getValue().get(field.getName()); + String value = storageModel.getValue().get(field.getEffectiveStorageName()); // If vector fields were not requested, skip if (value == null) { continue; } - Class valueType = recordClass.getDeclaredField(field.getName()) + Class valueType = recordClass + .getDeclaredField(field.getEffectiveStorageName()) .getType(); if (valueType.equals(String.class)) { - jsonNode.put(field.getName(), value); + jsonNode.put(field.getEffectiveStorageName(), value); } else { // Convert the String stored in Redis back to the correct type and then put the JSON node - jsonNode.set(field.getName(), + jsonNode.set(field.getEffectiveStorageName(), mapper.valueToTree(mapper.readValue(value, valueType))); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java index 5fc9a73c..9beef7ff 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java @@ -81,17 +81,16 @@ public RedisJsonVectorStoreRecordCollection( // Validate supported types VectorStoreRecordDefinition.validateSupportedTypes( - Collections - .singletonList(recordDefinition.getKeyDeclaredField(this.options.getRecordClass())), + Collections.singletonList(recordDefinition.getKeyField()), supportedKeyTypes); VectorStoreRecordDefinition.validateSupportedTypes( - recordDefinition.getVectorDeclaredFields(this.options.getRecordClass()), + new ArrayList<>(recordDefinition.getVectorFields()), supportedVectorTypes); // If mapper is not provided, set a default one if (options.getVectorStoreRecordMapper() == null) { vectorStoreRecordMapper = new RedisJsonVectorStoreRecordMapper.Builder() - .withKeyFieldName(recordDefinition.getKeyField().getName()) + .withKeyFieldName(recordDefinition.getKeyField().getEffectiveStorageName()) .withRecordClass(options.getRecordClass()) .build(); } else { @@ -101,7 +100,7 @@ public RedisJsonVectorStoreRecordCollection( // Creates a list of paths to retrieve from Redis when no vectors are requested // Paths are in the format of $.field this.dataFields = recordDefinition.getDataFields().stream() - .map(VectorStoreRecordDataField::getName) + .map(VectorStoreRecordDataField::getEffectiveStorageName) .map(Path2::new) .toArray(Path2[]::new); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java index 42dabb5b..26241d26 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java @@ -6,6 +6,8 @@ import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; import com.microsoft.semantickernel.exceptions.SKException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + import java.util.AbstractMap; import java.util.Map.Entry; import java.util.function.Function; @@ -41,6 +43,7 @@ public static class Builder private String keyFieldName; @Nullable private Class recordClass; + private ObjectMapper objectMapper = new ObjectMapper(); /** * Sets the key field name in the record. @@ -64,6 +67,18 @@ public Builder withRecordClass(Class recordClass) { return this; } + /** + * Sets the object mapper. + * + * @param objectMapper the object mapper + * @return the builder + */ + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withObjectMapper(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + return this; + } + /** * Builds the {@link RedisJsonVectorStoreRecordMapper}. * @@ -77,11 +92,10 @@ public RedisJsonVectorStoreRecordMapper build() { if (recordClass == null) { throw new IllegalArgumentException("recordClass is required"); } - ObjectMapper mapper = new ObjectMapper(); return new RedisJsonVectorStoreRecordMapper<>(record -> { try { - ObjectNode jsonNode = mapper.valueToTree(record); + ObjectNode jsonNode = objectMapper.valueToTree(record); String key = jsonNode.get(keyFieldName).asText(); jsonNode.remove(keyFieldName); @@ -93,10 +107,10 @@ public RedisJsonVectorStoreRecordMapper build() { } }, storageModel -> { try { - ObjectNode jsonNode = mapper.valueToTree(storageModel.getValue()); + ObjectNode jsonNode = objectMapper.valueToTree(storageModel.getValue()); // Add the key back to the record jsonNode.put(keyFieldName, storageModel.getKey()); - return mapper.convertValue(jsonNode, recordClass); + return objectMapper.convertValue(jsonNode, recordClass); } catch (Exception e) { throw new SKException( "Failure to deserialize object, by default the Redis connector uses Jackson, ensure your model object can be serialized by Jackson, i.e the class is visible, has getters, constructor, annotations etc.", diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java index fe044c73..999daa15 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java @@ -82,17 +82,18 @@ public static Schema mapToSchema(List fields) { if (dataField.getFieldType() == null) { throw new IllegalArgumentException( - "Field type is required for filterable fields: " + dataField.getName()); + "Field type is required for filterable fields: " + + dataField.getEffectiveStorageName()); } if (dataField.getFieldType().equals(String.class)) { - schema.addTextField(getRedisPath(dataField.getName()), 1.0); + schema.addTextField(getRedisPath(dataField.getEffectiveStorageName()), 1.0); } else if (supportedFilterableNumericTypes.contains(dataField.getFieldType())) { - schema.addNumericField(getRedisPath(dataField.getName())); + schema.addNumericField(getRedisPath(dataField.getEffectiveStorageName())); } else { throw new IllegalArgumentException( "Unsupported field type for numeric filterable fields: " - + dataField.getName()); + + dataField.getEffectiveStorageName()); } } @@ -103,7 +104,7 @@ public static Schema mapToSchema(List fields) { if (vectorField.getDimensions() < 1) { throw new IllegalArgumentException( "Dimensions must be greater than 0 for vector fields: " - + vectorField.getName()); + + vectorField.getEffectiveStorageName()); } Schema.VectorField.VectorAlgo algorithm = getAlgorithmConfig(vectorField); @@ -114,7 +115,8 @@ public static Schema mapToSchema(List fields) { attributes.put(RedisIndexSchemaParams.DIMENSIONS, vectorField.getDimensions()); attributes.put(RedisIndexSchemaParams.DISTANCE_METRIC, metric); - schema.addVectorField(getRedisPath(vectorField.getName()), algorithm, attributes); + schema.addVectorField(getRedisPath(vectorField.getEffectiveStorageName()), + algorithm, attributes); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java index 07d55cdd..0d7c3ddd 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; @@ -43,8 +44,7 @@ public VolatileVectorStoreRecordCollection(String collectionName, // Validate the key type VectorStoreRecordDefinition.validateSupportedTypes( - Collections - .singletonList(recordDefinition.getKeyDeclaredField(options.getRecordClass())), + Collections.singletonList(recordDefinition.getKeyField()), supportedKeyTypes); } @@ -147,7 +147,8 @@ public Mono upsertAsync(Record data, UpsertRecordOptions options) { return Mono.fromCallable(() -> { try { ObjectNode objectNode = objectMapper.valueToTree(data); - String key = objectNode.get(recordDefinition.getKeyField().getName()).asText(); + String key = objectNode + .get(recordDefinition.getKeyField().getEffectiveStorageName()).asText(); getCollection().put(key, data); return key; @@ -173,7 +174,8 @@ public Mono> upsertBatchAsync(List data, UpsertRecordOption return data.stream().map(record -> { try { ObjectNode objectNode = objectMapper.valueToTree(record); - String key = objectNode.get(recordDefinition.getKeyField().getName()).asText(); + String key = objectNode + .get(recordDefinition.getKeyField().getEffectiveStorageName()).asText(); collection.put(key, record); return key; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordDataAttribute.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordDataAttribute.java index cc4d233d..10efe24f 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordDataAttribute.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordDataAttribute.java @@ -14,6 +14,8 @@ public @interface VectorStoreRecordDataAttribute { /** * Storage name of the field. + * This value is only used when JSON Serialization using Jackson is not supported in a VectorStore. + * When Jackson is supported, @JsonProperty should be used to specify an alternate field name in the storage database. */ String storageName() default ""; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDataField.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDataField.java index af43821a..83ae6c39 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDataField.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDataField.java @@ -8,8 +8,6 @@ public class VectorStoreRecordDataField extends VectorStoreRecordField { private final boolean hasEmbedding; @Nullable private final String embeddingFieldName; - @Nullable - private final Class fieldType; private final boolean isFilterable; public static Builder builder() { @@ -21,22 +19,21 @@ public static Builder builder() { * * @param name the name of the field * @param storageName the storage name of the field + * @param fieldType the field type * @param hasEmbedding a value indicating whether the field has an embedding * @param embeddingFieldName the name of the embedding - * @param fieldType the field type * @param isFilterable a value indicating whether the field is filterable */ public VectorStoreRecordDataField( @Nonnull String name, @Nullable String storageName, + @Nonnull Class fieldType, boolean hasEmbedding, @Nullable String embeddingFieldName, - @Nullable Class fieldType, boolean isFilterable) { - super(name, storageName); + super(name, storageName, fieldType); this.hasEmbedding = hasEmbedding; this.embeddingFieldName = embeddingFieldName; - this.fieldType = fieldType; this.isFilterable = isFilterable; } @@ -59,16 +56,6 @@ public String getEmbeddingFieldName() { return embeddingFieldName; } - /** - * Gets the field type. - * - * @return the field type - */ - @Nullable - public Class getFieldType() { - return fieldType; - } - /** * Gets a value indicating whether the field is filterable. * @@ -83,8 +70,6 @@ public static class Builder private boolean hasEmbedding; @Nullable private String embeddingFieldName; - @Nullable - private Class fieldType; private boolean isFilterable; /** @@ -109,17 +94,6 @@ public Builder withEmbeddingFieldName(String embeddingFieldName) { return this; } - /** - * Sets the field type. - * - * @param fieldType the field type - * @return the builder - */ - public Builder withFieldType(Class fieldType) { - this.fieldType = fieldType; - return this; - } - /** * Sets a value indicating whether the field is filterable. * @@ -141,6 +115,9 @@ public VectorStoreRecordDataField build() { if (name == null) { throw new IllegalArgumentException("name is required"); } + if (fieldType == null) { + throw new IllegalArgumentException("fieldType is required"); + } if (hasEmbedding && embeddingFieldName == null) { throw new IllegalArgumentException( "embeddingFieldName is required when hasEmbedding is true"); @@ -149,9 +126,9 @@ public VectorStoreRecordDataField build() { return new VectorStoreRecordDataField( name, storageName, + fieldType, hasEmbedding, embeddingFieldName, - fieldType, isFilterable); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java index 39e04a3f..f17f5c74 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.recorddefinition; +import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; @@ -56,49 +57,6 @@ public List getNonVectorFields() { return fields; } - private enum DeclaredFieldType { - KEY, DATA, VECTOR - } - - private List getDeclaredFields(Class recordClass, List fields, - DeclaredFieldType fieldType) { - List declaredFields = new ArrayList<>(); - for (VectorStoreRecordField field : fields) { - try { - Field declaredField = recordClass.getDeclaredField(field.getName()); - declaredFields.add(declaredField); - } catch (NoSuchFieldException e) { - throw new IllegalArgumentException( - String.format("%s field not found in record class: %s", fieldType, - field.getName())); - } - } - return declaredFields; - } - - public Field getKeyDeclaredField(Class recordClass) { - try { - return recordClass.getDeclaredField(keyField.getName()); - } catch (NoSuchFieldException e) { - throw new IllegalArgumentException( - "Key field not found in record class: " + keyField.getName()); - } - } - - public List getDataDeclaredFields(Class recordClass) { - return getDeclaredFields( - recordClass, - dataFields.stream().map(f -> (VectorStoreRecordField) f).collect(Collectors.toList()), - DeclaredFieldType.DATA); - } - - public List getVectorDeclaredFields(Class recordClass) { - return getDeclaredFields( - recordClass, - vectorFields.stream().map(f -> (VectorStoreRecordField) f).collect(Collectors.toList()), - DeclaredFieldType.VECTOR); - } - private VectorStoreRecordDefinition( VectorStoreRecordKeyField keyField, List dataFields, @@ -155,13 +113,23 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) List vectorFields = new ArrayList<>(); for (Field field : recordClass.getDeclaredFields()) { + String storageName = null; + if (field.isAnnotationPresent(JsonProperty.class)) { + storageName = field.getAnnotation(JsonProperty.class).value(); + } + if (field.isAnnotationPresent(VectorStoreRecordKeyAttribute.class)) { VectorStoreRecordKeyAttribute keyAttribute = field .getAnnotation(VectorStoreRecordKeyAttribute.class); + if (storageName == null) { + storageName = keyAttribute.storageName().isEmpty() ? field.getName() + : keyAttribute.storageName(); + } keyFields.add(VectorStoreRecordKeyField.builder() .withName(field.getName()) - .withStorageName(keyAttribute.storageName()) + .withStorageName(storageName) + .withFieldType(field.getType()) .build()); } @@ -169,12 +137,16 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) VectorStoreRecordDataAttribute dataAttribute = field .getAnnotation(VectorStoreRecordDataAttribute.class); + if (storageName == null) { + storageName = dataAttribute.storageName().isEmpty() ? field.getName() + : dataAttribute.storageName(); + } dataFields.add(VectorStoreRecordDataField.builder() .withName(field.getName()) - .withStorageName(dataAttribute.storageName()) + .withStorageName(storageName) + .withFieldType(field.getType()) .withHasEmbedding(dataAttribute.hasEmbedding()) .withEmbeddingFieldName(dataAttribute.embeddingFieldName()) - .withFieldType(field.getType()) .withIsFilterable(dataAttribute.isFilterable()) .build()); } @@ -183,9 +155,14 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) VectorStoreRecordVectorAttribute vectorAttribute = field .getAnnotation(VectorStoreRecordVectorAttribute.class); + if (storageName == null) { + storageName = vectorAttribute.storageName().isEmpty() ? field.getName() + : vectorAttribute.storageName(); + } vectorFields.add(VectorStoreRecordVectorField.builder() .withName(field.getName()) - .withStorageName(vectorAttribute.storageName()) + .withStorageName(storageName) + .withFieldType(field.getType()) .withDimensions(vectorAttribute.dimensions()) .withIndexKind(IndexKind.fromString(vectorAttribute.indexKind())) .withDistanceFunction( @@ -197,12 +174,12 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) return checkFields(keyFields, dataFields, vectorFields); } - public static void validateSupportedTypes(List declaredFields, + public static void validateSupportedTypes(List fields, Set> supportedTypes) { Set> unsupportedTypes = new HashSet<>(); - for (Field declaredField : declaredFields) { - if (!supportedTypes.contains(declaredField.getType())) { - unsupportedTypes.add(declaredField.getType()); + for (VectorStoreRecordField field : fields) { + if (!supportedTypes.contains(field.getFieldType())) { + unsupportedTypes.add(field.getFieldType()); } } if (!unsupportedTypes.isEmpty()) { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordField.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordField.java index c27c7a06..6c8cd65a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordField.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordField.java @@ -13,16 +13,20 @@ public class VectorStoreRecordField { private final String name; @Nullable private final String storageName; + private final Class fieldType; /** * Creates a new instance of the VectorStoreRecordField class. * * @param name the name of the field */ - public VectorStoreRecordField(@Nonnull String name, - @Nullable String storageName) { + public VectorStoreRecordField( + @Nonnull String name, + @Nullable String storageName, + @Nonnull Class fieldType) { this.name = name; this.storageName = storageName; + this.fieldType = fieldType; } /** @@ -39,15 +43,38 @@ public String getName() { * * @return the storage name of the field */ - @Nullable public String getStorageName() { return storageName; } + /** + * Gets the effective storage name of the field. + *

+ * If the storage name is not set, the name of the field is returned. + * @return the effective storage name of the field + */ + public String getEffectiveStorageName() { + return storageName != null ? storageName : name; + } + + /** + * Gets the field type. + * + * @return the field type + */ + public Class getFieldType() { + return fieldType; + } + public abstract static class Builder> implements SemanticKernelBuilder { - protected String name = ""; - protected String storageName = ""; + + @Nullable + protected String name; + @Nullable + protected String storageName; + @Nullable + protected Class fieldType; /** * Sets the name of the field. @@ -71,6 +98,17 @@ public U withStorageName(String storageName) { return (U) this; } + /** + * Sets the field type. + * + * @param fieldType the field type + * @return the builder + */ + public U withFieldType(Class fieldType) { + this.fieldType = fieldType; + return (U) this; + } + /** * Builds the field. * diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordKeyField.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordKeyField.java index 4502aa3b..8cbcbe00 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordKeyField.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordKeyField.java @@ -13,16 +13,24 @@ public static Builder builder() { * Creates a new instance of the VectorStoreRecordKeyField class. * * @param name the name of the field + * @param storageName the storage name of the field + * @param type the field type */ - public VectorStoreRecordKeyField(String name, String storageName) { - super(name, storageName); + public VectorStoreRecordKeyField(String name, String storageName, Class type) { + super(name, storageName, type); } public static class Builder extends VectorStoreRecordField.Builder { @Override public VectorStoreRecordKeyField build() { - return new VectorStoreRecordKeyField(name, storageName); + if (name == null) { + throw new IllegalArgumentException("name is required."); + } + if (fieldType == null) { + throw new IllegalArgumentException("fieldType is required."); + } + return new VectorStoreRecordKeyField(name, storageName, fieldType); } } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordVectorField.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordVectorField.java index 34653c84..c2d996aa 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordVectorField.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordVectorField.java @@ -23,6 +23,7 @@ public static Builder builder() { * * @param name the name of the field * @param storageName the storage name of the field + * @param fieldType the field type * @param dimensions the number of dimensions in the vector * @param indexKind the index kind * @param distanceFunction the distance function @@ -30,10 +31,11 @@ public static Builder builder() { public VectorStoreRecordVectorField( @Nonnull String name, @Nullable String storageName, + @Nonnull Class fieldType, int dimensions, @Nullable IndexKind indexKind, @Nullable DistanceFunction distanceFunction) { - super(name, storageName); + super(name, storageName, fieldType); this.dimensions = dimensions; this.indexKind = indexKind; this.distanceFunction = distanceFunction; @@ -119,11 +121,15 @@ public VectorStoreRecordVectorField build() { if (name == null) { throw new IllegalArgumentException("name is required"); } + if (fieldType == null) { + throw new IllegalArgumentException("fieldType is required"); + } if (dimensions <= 0) { throw new IllegalArgumentException("dimensions must be greater than 0"); } - return new VectorStoreRecordVectorField(name, storageName, dimensions, indexKind, + return new VectorStoreRecordVectorField(name, storageName, fieldType, dimensions, + indexKind, distanceFunction); } } From 8f12af5c34fd7ceac8198dd62805b23ddd83f385 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Wed, 21 Aug 2024 00:20:19 -0700 Subject: [PATCH 005/277] Fix Redis mapper and tests --- .../semantickernel/tests/connectors/memory/Hotel.java | 2 +- .../RedisHashSetVectorStoreRecordCollectionTest.java | 4 ++++ .../redis/RedisJsonVectorStoreRecordCollectionTest.java | 4 ++++ .../data/redis/RedisHashSetVectorStoreRecordMapper.java | 8 ++++---- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java index b5d81e97..24217240 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java @@ -18,7 +18,7 @@ public class Hotel { @JsonProperty("summary") @VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "descriptionEmbedding") private final String description; - @JsonProperty("summaryVector") + @JsonProperty("summaryEmbedding") @VectorStoreRecordVectorAttribute(dimensions = 3) private final List descriptionEmbedding; @VectorStoreRecordDataAttribute diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index d39eed92..300f45b4 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -53,6 +53,7 @@ static void setup() { List fields = new ArrayList<>(); fields.add(VectorStoreRecordKeyField.builder() .withName("id") + .withFieldType(String.class) .build()); fields.add(VectorStoreRecordDataField.builder() .withName("name") @@ -64,12 +65,15 @@ static void setup() { .build()); fields.add(VectorStoreRecordDataField.builder() .withName("description") + .withStorageName("summary") .withFieldType(String.class) .withHasEmbedding(true) .withEmbeddingFieldName("descriptionEmbedding") .build()); fields.add(VectorStoreRecordVectorField.builder() .withName("descriptionEmbedding") + .withStorageName("summaryEmbedding") + .withFieldType(List.class) .withDimensions(768) .build()); fields.add(VectorStoreRecordDataField.builder() diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index d5fb7cd7..74e9ba23 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -54,6 +54,7 @@ static void setup() { List fields = new ArrayList<>(); fields.add(VectorStoreRecordKeyField.builder() .withName("id") + .withFieldType(String.class) .build()); fields.add(VectorStoreRecordDataField.builder() .withName("name") @@ -65,12 +66,15 @@ static void setup() { .build()); fields.add(VectorStoreRecordDataField.builder() .withName("description") + .withStorageName("summary") .withFieldType(String.class) .withHasEmbedding(true) .withEmbeddingFieldName("descriptionEmbedding") .build()); fields.add(VectorStoreRecordVectorField.builder() .withName("descriptionEmbedding") + .withStorageName("summaryEmbedding") + .withFieldType(List.class) .withDimensions(768) .build()); fields.add(VectorStoreRecordDataField.builder() diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java index a61fed88..78ee1199 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -139,9 +141,7 @@ public RedisHashSetVectorStoreRecordMapper build() { continue; } - Class valueType = recordClass - .getDeclaredField(field.getEffectiveStorageName()) - .getType(); + Class valueType = field.getFieldType(); if (valueType.equals(String.class)) { jsonNode.put(field.getEffectiveStorageName(), value); @@ -153,7 +153,7 @@ public RedisHashSetVectorStoreRecordMapper build() { } return mapper.convertValue(jsonNode, recordClass); - } catch (Exception e) { + } catch (JsonProcessingException e) { throw new SKException( "Failure to deserialize object, by default the Redis connector uses Jackson, ensure your model object can be serialized by Jackson, i.e the class is visible, has getters, constructor, annotations etc.", e); From 07e1f5bf480e1d3d2b0a0cb96593007614090bd4 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Wed, 21 Aug 2024 08:41:44 -0700 Subject: [PATCH 006/277] Add explicit @Nonnull to query providers constructor arguments --- .../data/mysql/MySQLVectorStoreQueryProvider.java | 8 ++++++-- .../postgres/PostgreSQLVectorStoreQueryProvider.java | 9 +++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java index 5eb61805..6ea52c95 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java @@ -13,6 +13,7 @@ import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import javax.annotation.Nonnull; import javax.sql.DataSource; import java.sql.Connection; import java.sql.PreparedStatement; @@ -26,8 +27,11 @@ public class MySQLVectorStoreQueryProvider extends private final ObjectMapper objectMapper; @SuppressFBWarnings("EI_EXPOSE_REP2") - private MySQLVectorStoreQueryProvider(DataSource dataSource, String collectionsTable, - String prefixForCollectionTables, ObjectMapper objectMapper) { + private MySQLVectorStoreQueryProvider( + @Nonnull DataSource dataSource, + @Nonnull String collectionsTable, + @Nonnull String prefixForCollectionTables, + @Nonnull ObjectMapper objectMapper) { super(dataSource, collectionsTable, prefixForCollectionTables); this.dataSource = dataSource; this.objectMapper = objectMapper; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index f8f7f8cc..3d7a47a9 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -14,6 +14,7 @@ import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import javax.annotation.Nonnull; import javax.sql.DataSource; import java.sql.Connection; import java.sql.PreparedStatement; @@ -27,6 +28,7 @@ public class PostgreSQLVectorStoreQueryProvider extends JDBCVectorStoreDefaultQueryProvider implements JDBCVectorStoreQueryProvider { + private final Map, String> supportedKeyTypes; private final Map, String> supportedDataTypes; private final Map, String> supportedVectorTypes; @@ -37,8 +39,11 @@ public class PostgreSQLVectorStoreQueryProvider extends private final ObjectMapper objectMapper; @SuppressFBWarnings("EI_EXPOSE_REP2") - private PostgreSQLVectorStoreQueryProvider(DataSource dataSource, String collectionsTable, - String prefixForCollectionTables, ObjectMapper objectMapper) { + private PostgreSQLVectorStoreQueryProvider( + @Nonnull DataSource dataSource, + @Nonnull String collectionsTable, + @Nonnull String prefixForCollectionTables, + @Nonnull ObjectMapper objectMapper) { super(dataSource, collectionsTable, prefixForCollectionTables); this.dataSource = dataSource; this.collectionsTable = collectionsTable; From 4d25116960d79abce7399988a50f2fd891e95735 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Wed, 21 Aug 2024 13:58:49 -0700 Subject: [PATCH 007/277] Add suggestions --- .../JDBCVectorStoreDefaultQueryProvider.java | 25 ++++-- .../mysql/MySQLVectorStoreQueryProvider.java | 29 +++--- .../PostgreSQLVectorStoreQueryProvider.java | 90 ++++++++----------- 3 files changed, 67 insertions(+), 77 deletions(-) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java index 455aed49..e21146eb 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java @@ -87,13 +87,24 @@ protected String getWildcardString(int wildcards) { .collect(Collectors.joining(", ")); } + /** + * Gets the key column name from a key field. + * @param keyField the key field + * @return the key column name + */ + protected String getKeyColumnName(VectorStoreRecordField keyField) { + return validateSQLidentifier(keyField.getEffectiveStorageName()); + } + /** * Formats the query columns from a record definition. * @param fields the fields to get the columns from * @return the formatted query columns */ protected String getQueryColumnsFromFields(List fields) { - return fields.stream().map(VectorStoreRecordField::getEffectiveStorageName) + return fields.stream() + .map(VectorStoreRecordField::getEffectiveStorageName) + .map(JDBCVectorStoreDefaultQueryProvider::validateSQLidentifier) .collect(Collectors.joining(", ")); } @@ -106,7 +117,8 @@ protected String getQueryColumnsFromFields(List fields) protected String getColumnNamesAndTypes(List fields, Map, String> types) { List columns = fields.stream() - .map(field -> field.getEffectiveStorageName() + " " + types.get(field.getFieldType())) + .map(field -> validateSQLidentifier(field.getEffectiveStorageName()) + " " + + types.get(field.getFieldType())) .collect(Collectors.toList()); return String.join(", ", columns); @@ -221,9 +233,8 @@ public void createCollection(String collectionName, VectorStoreRecordDefinition recordDefinition) { String createStorageTable = "CREATE TABLE IF NOT EXISTS " - + getCollectionTableName(collectionName) - + " (" + recordDefinition.getKeyField().getEffectiveStorageName() - + " VARCHAR(255) PRIMARY KEY, " + + getCollectionTableName(collectionName) + " (" + + getKeyColumnName(recordDefinition.getKeyField()) + " VARCHAR(255) PRIMARY KEY, " + getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), getSupportedDataTypes()) + ", " @@ -330,7 +341,7 @@ public List getRecords(String collectionName, List keys String query = "SELECT " + getQueryColumnsFromFields(fields) + " FROM " + getCollectionTableName(collectionName) - + " WHERE " + recordDefinition.getKeyField().getEffectiveStorageName() + + " WHERE " + getKeyColumnName(recordDefinition.getKeyField()) + " IN (" + getWildcardString(keys.size()) + ")"; try (Connection connection = dataSource.getConnection(); @@ -372,7 +383,7 @@ public void upsertRecords(String collectionName, List records, public void deleteRecords(String collectionName, List keys, VectorStoreRecordDefinition recordDefinition, DeleteRecordOptions options) { String query = "DELETE FROM " + getCollectionTableName(collectionName) - + " WHERE " + recordDefinition.getKeyField().getEffectiveStorageName() + + " WHERE " + getKeyColumnName(recordDefinition.getKeyField()) + " IN (" + getWildcardString(keys.size()) + ")"; try (Connection connection = dataSource.getConnection(); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java index 6ea52c95..c6440bad 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java @@ -19,6 +19,7 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.List; +import java.util.stream.Collectors; public class MySQLVectorStoreQueryProvider extends JDBCVectorStoreDefaultQueryProvider implements JDBCVectorStoreQueryProvider { @@ -28,10 +29,10 @@ public class MySQLVectorStoreQueryProvider extends @SuppressFBWarnings("EI_EXPOSE_REP2") private MySQLVectorStoreQueryProvider( - @Nonnull DataSource dataSource, - @Nonnull String collectionsTable, - @Nonnull String prefixForCollectionTables, - @Nonnull ObjectMapper objectMapper) { + @Nonnull DataSource dataSource, + @Nonnull String collectionsTable, + @Nonnull String prefixForCollectionTables, + @Nonnull ObjectMapper objectMapper) { super(dataSource, collectionsTable, prefixForCollectionTables); this.dataSource = dataSource; this.objectMapper = objectMapper; @@ -45,7 +46,7 @@ public static Builder builder() { return new Builder(); } - private void setStatementValues(PreparedStatement statement, Object record, + private void setUpsertStatementValues(PreparedStatement statement, Object record, List fields) { JsonNode jsonNode = objectMapper.valueToTree(record); @@ -82,20 +83,12 @@ private void setStatementValues(PreparedStatement statement, Object record, @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") // SQL query is generated dynamically with valid identifiers public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { - validateSQLidentifier(getCollectionTableName(collectionName)); - List fields = recordDefinition.getAllFields(); - StringBuilder onDuplicateKeyUpdate = new StringBuilder(); - for (int i = 0; i < fields.size(); ++i) { - VectorStoreRecordField field = fields.get(i); - if (i > 0) { - onDuplicateKeyUpdate.append(", "); - } - - onDuplicateKeyUpdate.append(field.getEffectiveStorageName()).append(" = VALUES(") - .append(field.getEffectiveStorageName()).append(")"); - } + String onDuplicateKeyUpdate = fields.stream() + .map(field -> validateSQLidentifier(field.getEffectiveStorageName()) + + " = VALUES(" + validateSQLidentifier(field.getEffectiveStorageName()) + ")") + .collect(Collectors.joining(", ")); String query = "INSERT INTO " + getCollectionTableName(collectionName) + " (" + getQueryColumnsFromFields(fields) + ")" @@ -105,7 +98,7 @@ public void upsertRecords(String collectionName, List records, try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(query)) { for (Object record : records) { - setStatementValues(statement, record, recordDefinition.getAllFields()); + setUpsertStatementValues(statement, record, recordDefinition.getAllFields()); statement.addBatch(); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index 3d7a47a9..d061254d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -25,6 +25,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public class PostgreSQLVectorStoreQueryProvider extends JDBCVectorStoreDefaultQueryProvider implements JDBCVectorStoreQueryProvider { @@ -40,10 +41,10 @@ public class PostgreSQLVectorStoreQueryProvider extends @SuppressFBWarnings("EI_EXPOSE_REP2") private PostgreSQLVectorStoreQueryProvider( - @Nonnull DataSource dataSource, - @Nonnull String collectionsTable, - @Nonnull String prefixForCollectionTables, - @Nonnull ObjectMapper objectMapper) { + @Nonnull DataSource dataSource, + @Nonnull String collectionsTable, + @Nonnull String prefixForCollectionTables, + @Nonnull ObjectMapper objectMapper) { super(dataSource, collectionsTable, prefixForCollectionTables); this.dataSource = dataSource; this.collectionsTable = collectionsTable; @@ -134,24 +135,19 @@ public void prepareVectorStore() { private String getColumnNamesAndTypesForVectorFields( List fields) { - StringBuilder columnNames = new StringBuilder(); - for (VectorStoreRecordVectorField field : fields) { - if (columnNames.length() > 0) { - columnNames.append(", "); - } - - if (field.getFieldType().equals(String.class)) { - columnNames.append(field.getEffectiveStorageName()).append(" ") - .append(supportedVectorTypes.get(String.class)); - } else { - // Get the vector type and dimensions - String type = String.format(supportedVectorTypes.get(field.getFieldType()), - field.getDimensions()); - columnNames.append(field.getEffectiveStorageName()).append(" ").append(type); - } - } - - return columnNames.toString(); + return fields.stream() + .map(field -> { + String columnType; + if (field.getFieldType().equals(String.class)) { + columnType = supportedVectorTypes.get(String.class); + } else { + // Get the vector type and dimensions + columnType = String.format(supportedVectorTypes.get(field.getFieldType()), + field.getDimensions()); + } + return validateSQLidentifier(field.getEffectiveStorageName()) + " " + columnType; + }) + .collect(Collectors.joining(", ")); } /** @@ -167,8 +163,8 @@ public void createCollection(String collectionName, VectorStoreRecordDefinition recordDefinition) { String createStorageTable = "CREATE TABLE IF NOT EXISTS " - + getCollectionTableName(collectionName) - + " (" + recordDefinition.getKeyField().getStorageName() + " VARCHAR(255) PRIMARY KEY, " + + getCollectionTableName(collectionName) + " (" + + getKeyColumnName(recordDefinition.getKeyField()) + " VARCHAR(255) PRIMARY KEY, " + getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), supportedDataTypes) + ", " @@ -194,7 +190,7 @@ public void createCollection(String collectionName, } } - private void setStatementValues(PreparedStatement statement, Object record, + private void setUpsertStatementValues(PreparedStatement statement, Object record, List fields) { JsonNode jsonNode = objectMapper.valueToTree(record); @@ -220,19 +216,16 @@ private void setStatementValues(PreparedStatement statement, Object record, } private String getWildcardStringWithCast(List fields) { - StringBuilder wildcardString = new StringBuilder(); - int wildcards = fields.size(); - for (int i = 0; i < wildcards; ++i) { - if (i > 0) { - wildcardString.append(", "); - } - wildcardString.append("?"); - // Add casting for vector fields - if (fields.get(i) instanceof VectorStoreRecordVectorField) { - wildcardString.append("::vector"); - } - } - return wildcardString.toString(); + return fields.stream() + .map(field -> { + String wildcard = "?"; + // Add casting for vector fields + if (field instanceof VectorStoreRecordVectorField) { + wildcard += "::vector"; + } + return wildcard; + }) + .collect(Collectors.joining(", ")); } /** @@ -250,30 +243,23 @@ public void upsertRecords(String collectionName, List records, validateSQLidentifier(getCollectionTableName(collectionName)); List fields = recordDefinition.getAllFields(); - StringBuilder onDuplicateKeyUpdate = new StringBuilder(); - for (VectorStoreRecordField field : fields) { - if (field instanceof VectorStoreRecordKeyField) { - continue; - } - if (onDuplicateKeyUpdate.length() > 0) { - onDuplicateKeyUpdate.append(", "); - } - onDuplicateKeyUpdate.append(field.getEffectiveStorageName()) - .append(" = EXCLUDED.") - .append(field.getEffectiveStorageName()); - } + String onDuplicateKeyUpdate = fields.stream() + .filter(field -> !(field instanceof VectorStoreRecordKeyField)) // Exclude key fields + .map(field -> validateSQLidentifier(field.getEffectiveStorageName()) + + " = EXCLUDED." + validateSQLidentifier(field.getEffectiveStorageName())) + .collect(Collectors.joining(", ")); String query = "INSERT INTO " + getCollectionTableName(collectionName) + " (" + getQueryColumnsFromFields(fields) + ")" + " VALUES (" + getWildcardStringWithCast(fields) + ")" - + " ON CONFLICT (" + recordDefinition.getKeyField().getEffectiveStorageName() + + " ON CONFLICT (" + getKeyColumnName(recordDefinition.getKeyField()) + ") DO UPDATE SET " + onDuplicateKeyUpdate; try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(query)) { for (Object record : records) { - setStatementValues(statement, record, recordDefinition.getAllFields()); + setUpsertStatementValues(statement, record, recordDefinition.getAllFields()); statement.addBatch(); } From 300a30ec9949eb1eb7e7ea96ce6a930a653b2781 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Wed, 21 Aug 2024 21:00:33 -0700 Subject: [PATCH 008/277] Add VectorStoreRecordCollectionOptions interface and make getCollection to receive it --- .../memory/jdbc/JDBCVectorStoreTest.java | 6 +- .../memory/redis/RedisVectorStoreTest.java | 17 +++++- .../memory/AzureAISearch_DataStorage.java | 6 +- .../memory/InMemory_DataStorage.java | 6 +- .../memory/JDBC_DataStorage.java | 6 +- .../memory/Redis_DataStorage.java | 6 +- .../AzureAISearchVectorStore.java | 40 ++++++------- ...rchVectorStoreRecordCollectionOptions.java | 16 +++++- .../connectors/data/jdbc/JDBCVectorStore.java | 55 ++++++++---------- ...DBCVectorStoreRecordCollectionOptions.java | 14 ++++- ...SetVectorStoreRecordCollectionOptions.java | 14 ++++- ...sonVectorStoreRecordCollectionOptions.java | 14 ++++- .../data/redis/RedisVectorStore.java | 56 +++++++------------ .../semantickernel/data/VectorStore.java | 7 +-- .../VectorStoreRecordCollectionOptions.java | 27 +++++++++ .../data/VolatileVectorStore.java | 35 ++++-------- ...ileVectorStoreRecordCollectionOptions.java | 13 ++++- .../data/VolatileVectorStoreTest.java | 6 +- 18 files changed, 208 insertions(+), 136 deletions(-) create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollectionOptions.java diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java index 8c2fbfd0..85fccb89 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java @@ -3,6 +3,7 @@ import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStore; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; import com.microsoft.semantickernel.tests.connectors.memory.Hotel; @@ -89,7 +90,10 @@ public void getCollectionNamesAsync(QueryProvider provider) { List collectionNames = Arrays.asList("collection1", "collection2", "collection3"); for (String collectionName : collectionNames) { - vectorStore.getCollection(collectionName, Hotel.class, null).createCollectionAsync().block(); + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()).createCollectionAsync().block(); } List retrievedCollectionNames = vectorStore.getCollectionNamesAsync().block(); diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java index a8082e9b..43fead85 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java @@ -1,8 +1,11 @@ package com.microsoft.semantickernel.tests.connectors.memory.redis; +import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.redis.RedisStorageType; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStore; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreOptions; +import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.tests.connectors.memory.Hotel; import com.redis.testcontainers.RedisContainer; import org.junit.jupiter.api.BeforeAll; @@ -36,6 +39,18 @@ public static JedisPooled buildClient(RedisStorageType storageType) { } } + private static VectorStoreRecordCollectionOptions getRecordCollectionOptions(RedisStorageType storageType) { + if (storageType == RedisStorageType.JSON) { + return RedisJsonVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build(); + } else { + return RedisHashSetVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build(); + } + } + @ParameterizedTest @EnumSource(RedisStorageType.class) public void getCollectionNamesAsync(RedisStorageType storageType) { @@ -46,7 +61,7 @@ public void getCollectionNamesAsync(RedisStorageType storageType) { List collectionNames = Arrays.asList("collection1", "collection2", "collection3"); for (String collectionName : collectionNames) { - vectorStore.getCollection(collectionName, Hotel.class, null).createCollectionAsync().block(); + vectorStore.getCollection(collectionName, getRecordCollectionOptions(storageType)).createCollectionAsync().block(); } List retrievedCollectionNames = vectorStore.getCollectionNamesAsync().block(); diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java index 03cc93e3..6746afdf 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java @@ -13,6 +13,7 @@ import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStore; import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreOptions; +import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; @@ -123,8 +124,9 @@ public static void dataStorageWithAzureAISearch( String collectionName = "skgithubfiles"; var collection = azureAISearchVectorStore.getCollection( collectionName, - GitHubFile.class, - null); + AzureAISearchVectorStoreRecordCollectionOptions.builder() + .withRecordClass(GitHubFile.class) + .build()); // Create collection if it does not exist and store data collection diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java index c74d16f8..53904d57 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java @@ -8,6 +8,7 @@ import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.VolatileVectorStore; +import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; @@ -101,7 +102,10 @@ public static void inMemoryDataStorage( var volatileVectorStore = new VolatileVectorStore(); String collectionName = "skgithubfiles"; - var collection = volatileVectorStore.getCollection(collectionName, GitHubFile.class, null); + var collection = volatileVectorStore.getCollection(collectionName, + VolatileVectorStoreRecordCollectionOptions.builder() + .withRecordClass(GitHubFile.class) + .build()); // Create collection if it does not exist and store data List ids = collection diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java index ed1d8bd5..90e38e26 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java @@ -8,6 +8,7 @@ import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStore; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; @@ -132,8 +133,9 @@ public static void dataStorageWithMySQL( String collectionName = "skgithubfiles"; var collection = jdbcVectorStore.getCollection(collectionName, - GitHubFile.class, - null); + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(GitHubFile.class) + .build()); // Create collection if it does not exist and store data List ids = collection diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java index 68db6e56..bb236788 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java @@ -9,6 +9,7 @@ import com.azure.core.util.MetricsOptions; import com.azure.core.util.TracingOptions; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; +import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStore; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreOptions; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; @@ -120,7 +121,10 @@ public static void dataStorageWithRedis( .build(); String collectionName = "skgithubfiles"; - var collection = vectorStore.getCollection(collectionName, GitHubFile.class, null); + var collection = vectorStore.getCollection(collectionName, + RedisJsonVectorStoreRecordCollectionOptions.builder() + .withRecordClass(GitHubFile.class) + .build()); // Create collection if it does not exist and store data List ids = collection diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java index b79616c5..5c1ec120 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java @@ -5,7 +5,9 @@ import com.azure.search.documents.indexes.models.SearchIndex; import com.microsoft.semantickernel.data.VectorStore; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.List; import javax.annotation.Nonnull; @@ -34,44 +36,34 @@ public AzureAISearchVectorStore(@Nonnull SearchIndexAsyncClient client, * Gets a new instance of {@link AzureAISearchVectorStoreRecordCollection} * * @param collectionName The name of the collection. - * @param recordClass The class type of the record. - * @param recordDefinition The record definition. + * @param options The options for the collection. * @return The collection. */ @Override public final VectorStoreRecordCollection getCollection( @Nonnull String collectionName, - @Nonnull Class keyClass, - @Nonnull Class recordClass, - @Nullable VectorStoreRecordDefinition recordDefinition) { - if (!keyClass.equals(String.class)) { - throw new IllegalArgumentException("Azure AI Search only supports string keys"); + @Nonnull VectorStoreRecordCollectionOptions options) { + if (!options.getKeyClass().equals(String.class)) { + throw new SKException("Azure AI Search only supports string keys"); + } + if (options.getRecordClass() == null) { + throw new SKException("Record class is required"); } - return (VectorStoreRecordCollection) getCollection( - collectionName, recordClass, recordDefinition); - } - - public AzureAISearchVectorStoreRecordCollection getCollection( - @Nonnull String collectionName, - @Nonnull Class recordClass, - @Nullable VectorStoreRecordDefinition recordDefinition) { - if (options.getVectorStoreRecordCollectionFactory() != null) { - return options.getVectorStoreRecordCollectionFactory() + if (this.options.getVectorStoreRecordCollectionFactory() != null) { + return (VectorStoreRecordCollection) this.options + .getVectorStoreRecordCollectionFactory() .createVectorStoreRecordCollection( client, collectionName, - recordClass, - recordDefinition); + options.getRecordClass(), + options.getRecordDefinition()); } - return new AzureAISearchVectorStoreRecordCollection<>( + return (VectorStoreRecordCollection) new AzureAISearchVectorStoreRecordCollection<>( client, collectionName, - AzureAISearchVectorStoreRecordCollectionOptions.builder() - .withRecordClass(recordClass) - .withRecordDefinition(recordDefinition) - .build()); + (AzureAISearchVectorStoreRecordCollectionOptions) options); } /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java index 7275dd43..97197630 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java @@ -2,6 +2,7 @@ package com.microsoft.semantickernel.connectors.data.azureaisearch; import com.azure.search.documents.SearchDocument; +import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import javax.annotation.Nonnull; @@ -12,12 +13,11 @@ * * @param the record type */ -public class AzureAISearchVectorStoreRecordCollectionOptions { - +public class AzureAISearchVectorStoreRecordCollectionOptions + implements VectorStoreRecordCollectionOptions { private final Class recordClass; @Nullable private final VectorStoreRecordMapper vectorStoreRecordMapper; - @Nullable private final VectorStoreRecordDefinition recordDefinition; @@ -31,6 +31,16 @@ public static Builder builder() { return new Builder<>(); } + /** + * Gets the key class. + * + * @return the key class + */ + @Override + public Class getKeyClass() { + return String.class; + } + /** * Gets the record class. * diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java index 51775cd6..c11ee109 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java @@ -2,7 +2,9 @@ package com.microsoft.semantickernel.connectors.data.jdbc; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; @@ -55,53 +57,42 @@ public static Builder builder() { * Gets a collection from the vector store. * * @param collectionName The name of the collection. - * @param recordClass The class type of the record. - * @param recordDefinition The record definition. + * @param options The options for the collection. * @return The collection. */ @Override public VectorStoreRecordCollection getCollection( - @Nonnull String collectionName, @Nonnull Class keyClass, - @Nonnull Class recordClass, - @Nullable VectorStoreRecordDefinition recordDefinition) { - if (keyClass != String.class) { - throw new IllegalArgumentException("Redis only supports string keys"); + @Nonnull String collectionName, + @Nonnull VectorStoreRecordCollectionOptions options) { + if (!options.getKeyClass().equals(String.class)) { + throw new SKException("JDBC only supports string keys"); + } + if (options.getRecordClass() == null) { + throw new SKException("Record class is required"); } - return (VectorStoreRecordCollection) getCollection( - collectionName, - recordClass, - recordDefinition); - } - - /** - * Gets a collection from the vector store. - * - * @param collectionName The name of the collection. - * @param recordClass The class type of the record. - * @param recordDefinition The record definition. - * @return The collection. - */ - public JDBCVectorStoreRecordCollection getCollection( - @Nonnull String collectionName, - @Nonnull Class recordClass, - @Nullable VectorStoreRecordDefinition recordDefinition) { if (this.options != null && this.options.getVectorStoreRecordCollectionFactory() != null) { - return this.options.getVectorStoreRecordCollectionFactory() + return (VectorStoreRecordCollection) this.options + .getVectorStoreRecordCollectionFactory() .createVectorStoreRecordCollection( dataSource, collectionName, - recordClass, - recordDefinition); + options.getRecordClass(), + options.getRecordDefinition()); } - return new JDBCVectorStoreRecordCollection<>( + JDBCVectorStoreRecordCollectionOptions jdbcOptions = (JDBCVectorStoreRecordCollectionOptions) options; + return (VectorStoreRecordCollection) new JDBCVectorStoreRecordCollection<>( dataSource, collectionName, JDBCVectorStoreRecordCollectionOptions.builder() - .withRecordClass(recordClass) - .withRecordDefinition(recordDefinition) - .withQueryProvider(this.queryProvider) + .withCollectionsTableName(jdbcOptions.getCollectionsTableName()) + .withPrefixForCollectionTables(jdbcOptions.getPrefixForCollectionTables()) + .withQueryProvider(jdbcOptions.getQueryProvider() == null ? queryProvider + : jdbcOptions.getQueryProvider()) + .withRecordClass(jdbcOptions.getRecordClass()) + .withRecordDefinition(jdbcOptions.getRecordDefinition()) + .withVectorStoreRecordMapper(jdbcOptions.getVectorStoreRecordMapper()) .build()); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java index f6aa871d..2429ead6 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; +import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -11,7 +12,8 @@ import static com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider.DEFAULT_COLLECTIONS_TABLE; import static com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider.DEFAULT_PREFIX_FOR_COLLECTION_TABLES; -public class JDBCVectorStoreRecordCollectionOptions { +public class JDBCVectorStoreRecordCollectionOptions + implements VectorStoreRecordCollectionOptions { private final Class recordClass; private final VectorStoreRecordMapper vectorStoreRecordMapper; private final VectorStoreRecordDefinition recordDefinition; @@ -43,6 +45,16 @@ public static Builder builder() { return new Builder<>(); } + /** + * Gets the key class. + * + * @return the key class + */ + @Override + public Class getKeyClass() { + return String.class; + } + /** * Gets the record class. * @return the record class diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java index c4130ccd..184d5723 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; +import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; @@ -9,7 +10,8 @@ import java.util.Map; import java.util.Map.Entry; -public class RedisHashSetVectorStoreRecordCollectionOptions { +public class RedisHashSetVectorStoreRecordCollectionOptions + implements VectorStoreRecordCollectionOptions { private final Class recordClass; @Nullable private final VectorStoreRecordMapper>> vectorStoreRecordMapper; @@ -38,6 +40,16 @@ public static Builder builder() { return new Builder<>(); } + /** + * Gets the key class. + * + * @return the key class + */ + @Override + public Class getKeyClass() { + return String.class; + } + /** * Gets the record class. * diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java index 8fbaf90b..51c61f1c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; +import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; @@ -8,7 +9,8 @@ import javax.annotation.Nullable; import java.util.Map.Entry; -public class RedisJsonVectorStoreRecordCollectionOptions { +public class RedisJsonVectorStoreRecordCollectionOptions + implements VectorStoreRecordCollectionOptions { private final Class recordClass; @Nullable private final VectorStoreRecordMapper> vectorStoreRecordMapper; @@ -37,6 +39,16 @@ public static Builder builder() { return new Builder<>(); } + /** + * Gets the key class. + * + * @return the key class + */ + @Override + public Class getKeyClass() { + return String.class; + } + /** * Gets the record class. * diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java index 4ff45f77..bfa0a4bb 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java @@ -4,7 +4,9 @@ import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.VectorStore; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.ArrayList; import java.util.List; @@ -31,60 +33,42 @@ public RedisVectorStore(@Nonnull JedisPooled client, this.options = options; } - @Override - public VectorStoreRecordCollection getCollection( - @Nonnull String collectionName, - @Nonnull Class keyClass, - @Nonnull Class recordClass, - @Nullable VectorStoreRecordDefinition recordDefinition) { - if (keyClass != String.class) { - throw new IllegalArgumentException("Redis only supports string keys"); - } - - return (VectorStoreRecordCollection) getCollection( - collectionName, - recordClass, - recordDefinition); - } - /** * Gets a collection from the vector store. * * @param collectionName The name of the collection. - * @param recordClass The class type of the record. - * @param recordDefinition The record definition. * @return The collection. */ - public VectorStoreRecordCollection getCollection( + public VectorStoreRecordCollection getCollection( @Nonnull String collectionName, - @Nonnull Class recordClass, - @Nullable VectorStoreRecordDefinition recordDefinition) { + @Nonnull VectorStoreRecordCollectionOptions options) { + if (!options.getKeyClass().equals(String.class)) { + throw new SKException("Redis only supports string keys"); + } + if (options.getRecordClass() == null) { + throw new SKException("Record class is required"); + } - if (options.getVectorStoreRecordCollectionFactory() != null) { - return options.getVectorStoreRecordCollectionFactory() + if (this.options.getVectorStoreRecordCollectionFactory() != null) { + return (VectorStoreRecordCollection) this.options + .getVectorStoreRecordCollectionFactory() .createVectorStoreRecordCollection( client, collectionName, - recordClass, - recordDefinition); + options.getRecordClass(), + options.getRecordDefinition()); } - if (options.getStorageType() == RedisStorageType.JSON) { - return new RedisJsonVectorStoreRecordCollection<>( + if (this.options.getStorageType() == RedisStorageType.JSON) { + return (VectorStoreRecordCollection) new RedisJsonVectorStoreRecordCollection<>( client, collectionName, - RedisJsonVectorStoreRecordCollectionOptions.builder() - .withRecordClass(recordClass) - .withRecordDefinition(recordDefinition) - .build()); + (RedisJsonVectorStoreRecordCollectionOptions) options); } else { - return new RedisHashSetVectorStoreRecordCollection<>( + return (VectorStoreRecordCollection) new RedisHashSetVectorStoreRecordCollection<>( client, collectionName, - RedisHashSetVectorStoreRecordCollectionOptions.builder() - .withRecordClass(recordClass) - .withRecordDefinition(recordDefinition) - .build()); + (RedisHashSetVectorStoreRecordCollectionOptions) options); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStore.java index a51f044c..664bb281 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStore.java @@ -16,15 +16,12 @@ public interface VectorStore { * Gets a collection from the vector store. * * @param collectionName The name of the collection. - * @param recordClass The class type of the record. - * @param recordDefinition The record definition. + * @param options The options for the collection. * @return The collection. */ VectorStoreRecordCollection getCollection( @Nonnull String collectionName, - @Nonnull Class keyClass, - @Nonnull Class recordClass, - @Nullable VectorStoreRecordDefinition recordDefinition); + @Nonnull VectorStoreRecordCollectionOptions options); /** * Gets the names of all collections in the vector store. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollectionOptions.java new file mode 100644 index 00000000..6b128a13 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollectionOptions.java @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data; + +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; + +public interface VectorStoreRecordCollectionOptions { + /** + * Gets the key class. + * + * @return the key class + */ + Class getKeyClass(); + + /** + * Gets the record class. + * + * @return the record class + */ + Class getRecordClass(); + + /** + * Gets the record definition. + * + * @return the record definition + */ + VectorStoreRecordDefinition getRecordDefinition(); +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java index 25e90ad9..8d13814c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java @@ -8,6 +8,8 @@ import java.util.concurrent.ConcurrentHashMap; import javax.annotation.Nonnull; import javax.annotation.Nullable; + +import com.microsoft.semantickernel.exceptions.SKException; import reactor.core.publisher.Mono; public class VolatileVectorStore implements VectorStore { @@ -21,37 +23,24 @@ public VolatileVectorStore() { /** * Gets a collection from the vector store. * - * @param collectionName The name of the collection. - * @param recordDefinition The record definition. + * @param collectionName The name of the collection. + * @param options The options for the collection. * @return The collection. */ @Override public VectorStoreRecordCollection getCollection( @Nonnull String collectionName, - @Nonnull Class keyClass, - @Nonnull Class recordClass, - @Nullable VectorStoreRecordDefinition recordDefinition) { - if (keyClass != String.class) { - throw new IllegalArgumentException("Volatile only supports string keys"); + @Nonnull VectorStoreRecordCollectionOptions options) { + if (options.getKeyClass() != String.class) { + throw new SKException("Volatile only supports string keys"); + } + if (options.getRecordClass() == null) { + throw new SKException("Record class is required"); } - return (VectorStoreRecordCollection) getCollection( - collectionName, - recordClass, - recordDefinition); - } - - public VectorStoreRecordCollection getCollection( - @Nonnull String collectionName, - @Nonnull Class recordClass, - @Nullable VectorStoreRecordDefinition recordDefinition) { - return new VolatileVectorStoreRecordCollection<>( + return (VectorStoreRecordCollection) new VolatileVectorStoreRecordCollection<>( collectionName, - collections, - VolatileVectorStoreRecordCollectionOptions.builder() - .withRecordClass(recordClass) - .withRecordDefinition(recordDefinition) - .build()); + (VolatileVectorStoreRecordCollectionOptions) options); } /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java index 85bda03b..f3e50a68 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java @@ -6,7 +6,8 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; -public class VolatileVectorStoreRecordCollectionOptions { +public class VolatileVectorStoreRecordCollectionOptions + implements VectorStoreRecordCollectionOptions { private final Class recordClass; @Nullable private final VectorStoreRecordDefinition recordDefinition; @@ -33,6 +34,16 @@ public static Builder builder() { return new Builder<>(); } + /** + * Gets the key class. + * + * @return the key class + */ + @Override + public Class getKeyClass() { + return String.class; + } + /** * Gets the record class. * diff --git a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreTest.java b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreTest.java index 99c643e5..4cee074d 100644 --- a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreTest.java +++ b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreTest.java @@ -24,7 +24,11 @@ public void collectionNamesAsync() { List collectionNames = Arrays.asList("hotels1", "hotels2", "hotels3"); for (String collectionName : collectionNames) { - vectorStore.getCollection(collectionName, Hotel.class, null).createCollectionAsync() + vectorStore.getCollection(collectionName, + VolatileVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()) + .createCollectionAsync() .block(); } From c213a086436119710d921fc05638aba267225434 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 22 Aug 2024 08:29:39 -0700 Subject: [PATCH 009/277] Fix VolatileVectorStore --- .../com/microsoft/semantickernel/data/VolatileVectorStore.java | 1 + 1 file changed, 1 insertion(+) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java index 8d13814c..3b5c3efd 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java @@ -40,6 +40,7 @@ public VectorStoreRecordCollection getCollection( return (VectorStoreRecordCollection) new VolatileVectorStoreRecordCollection<>( collectionName, + collections, (VolatileVectorStoreRecordCollectionOptions) options); } From d84027f7306c51405ccd0af89b98e18be68702a5 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 22 Aug 2024 13:10:52 -0700 Subject: [PATCH 010/277] Update to SKException --- ...reAISearchVectorStoreCollectionCreateMapping.java | 12 +++++++----- ...reAISearchVectorStoreRecordCollectionOptions.java | 4 +++- .../connectors/data/jdbc/JDBCVectorStore.java | 2 +- .../jdbc/JDBCVectorStoreDefaultQueryProvider.java | 8 ++++---- .../data/jdbc/JDBCVectorStoreRecordCollection.java | 6 +++--- .../jdbc/JDBCVectorStoreRecordCollectionOptions.java | 3 ++- .../data/jdbc/JDBCVectorStoreRecordMapper.java | 4 ++-- .../postgres/PostgreSQLVectorStoreRecordMapper.java | 7 ++----- ...disHashSetVectorStoreRecordCollectionOptions.java | 3 ++- .../redis/RedisHashSetVectorStoreRecordMapper.java | 4 ++-- .../RedisJsonVectorStoreRecordCollectionOptions.java | 3 ++- .../data/redis/RedisJsonVectorStoreRecordMapper.java | 4 ++-- .../connectors/data/redis/RedisVectorStore.java | 4 ++-- .../RedisVectorStoreCollectionCreateMapping.java | 12 +++++++----- .../data/redis/RedisVectorStoreOptions.java | 4 +++- 15 files changed, 44 insertions(+), 36 deletions(-) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java index af7c0d41..a833ef69 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java @@ -13,6 +13,8 @@ import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.exceptions.SKException; + import java.time.OffsetDateTime; import java.util.List; import javax.annotation.Nonnull; @@ -41,7 +43,7 @@ private static VectorSearchAlgorithmMetric getAlgorithmMetric( case EUCLIDEAN: return VectorSearchAlgorithmMetric.EUCLIDEAN; default: - throw new IllegalArgumentException( + throw new SKException( "Unsupported distance function: " + vectorField.getDistanceFunction()); } } @@ -62,7 +64,7 @@ private static VectorSearchAlgorithmConfiguration getAlgorithmConfig( .setParameters( new ExhaustiveKnnParameters().setMetric(getAlgorithmMetric(vectorField))); default: - throw new IllegalArgumentException( + throw new SKException( "Unsupported index kind: " + vectorField.getIndexKind()); } } @@ -75,7 +77,7 @@ public static SearchField mapKeyField(VectorStoreRecordKeyField keyField) { public static SearchField mapDataField(VectorStoreRecordDataField dataField) { if (dataField.getFieldType() == null) { - throw new IllegalArgumentException( + throw new SKException( "Field type is required: " + dataField.getEffectiveStorageName()); } @@ -97,7 +99,7 @@ public static void updateVectorSearchParameters( List profiles, VectorStoreRecordVectorField vectorField) { if (vectorField.getDimensions() <= 0) { - throw new IllegalArgumentException("Vector field dimensions must be greater than 0"); + throw new SKException("Vector field dimensions must be greater than 0"); } algorithms.add(getAlgorithmConfig(vectorField)); @@ -121,7 +123,7 @@ public static SearchFieldDataType getSearchFieldDataType(Class fieldType) { } else if (fieldType == OffsetDateTime.class) { return SearchFieldDataType.DATE_TIME_OFFSET; } else { - throw new IllegalArgumentException("Unsupported field type: " + fieldType.getName()); + throw new SKException("Unsupported field type: " + fieldType.getName()); } } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java index 97197630..0437bcfd 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java @@ -5,6 +5,8 @@ import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.exceptions.SKException; + import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -128,7 +130,7 @@ public Builder withRecordDefinition(VectorStoreRecordDefinition recordDe */ public AzureAISearchVectorStoreRecordCollectionOptions build() { if (recordClass == null) { - throw new IllegalArgumentException("recordClass must be provided"); + throw new SKException("recordClass must be provided"); } return new AzureAISearchVectorStoreRecordCollectionOptions<>( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java index c11ee109..3b0eb189 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java @@ -162,7 +162,7 @@ public JDBCVectorStore build() { */ public Mono buildAsync() { if (dataSource == null) { - throw new IllegalArgumentException("dataSource is required"); + throw new SKException("dataSource is required"); } JDBCVectorStore vectorStore = new JDBCVectorStore(dataSource, options); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java index e21146eb..fef57d65 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java @@ -182,7 +182,7 @@ public void prepareVectorStore() { * Checks if the types of the record class fields are supported. * * @param recordDefinition the record definition - * @throws IllegalArgumentException if the types are not supported + * @throws SKException if the types are not supported */ @Override public void validateSupportedTypes(VectorStoreRecordDefinition recordDefinition) { @@ -403,13 +403,13 @@ public void deleteRecords(String collectionName, List keys, * * @param identifier the identifier * @return the identifier if it is valid - * @throws IllegalArgumentException if the identifier is invalid + * @throws SKException if the identifier is invalid */ public static String validateSQLidentifier(String identifier) { if (identifier.matches("[a-zA-Z_][a-zA-Z0-9_]*")) { return identifier; } - throw new IllegalArgumentException("Invalid SQL identifier: " + identifier); + throw new SKException("Invalid SQL identifier: " + identifier); } /** @@ -455,7 +455,7 @@ public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { @Override public JDBCVectorStoreDefaultQueryProvider build() { if (dataSource == null) { - throw new IllegalArgumentException("DataSource is required"); + throw new SKException("DataSource is required"); } return new JDBCVectorStoreDefaultQueryProvider(dataSource, collectionsTable, diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java index f5bb7e44..91582106 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -329,13 +329,13 @@ public Builder withOptions(JDBCVectorStoreRecordCollectionOptions build() { if (dataSource == null) { - throw new IllegalArgumentException("dataSource is required"); + throw new SKException("dataSource is required"); } if (collectionName == null) { - throw new IllegalArgumentException("collectionName is required"); + throw new SKException("collectionName is required"); } if (options == null) { - throw new IllegalArgumentException("options is required"); + throw new SKException("options is required"); } return new JDBCVectorStoreRecordCollection<>(dataSource, collectionName, options); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java index 2429ead6..093e3186 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java @@ -4,6 +4,7 @@ import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.sql.ResultSet; @@ -180,7 +181,7 @@ public Builder withPrefixForCollectionTables(String prefixForCollectionT */ public JDBCVectorStoreRecordCollectionOptions build() { if (recordClass == null) { - throw new IllegalArgumentException("recordClass is required"); + throw new SKException("recordClass is required"); } return new JDBCVectorStoreRecordCollectionOptions<>( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java index da53a99e..c9398948 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java @@ -97,10 +97,10 @@ public Builder withObjectMapper(ObjectMapper objectMapper) { */ public JDBCVectorStoreRecordMapper build() { if (recordClass == null) { - throw new IllegalArgumentException("recordClass is required"); + throw new SKException("recordClass is required"); } if (vectorStoreRecordDefinition == null) { - throw new IllegalArgumentException("vectorStoreRecordDefinition is required"); + throw new SKException("vectorStoreRecordDefinition is required"); } return new JDBCVectorStoreRecordMapper<>( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java index 5e3fd188..1a074de2 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java @@ -14,9 +14,6 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.postgresql.util.PGobject; -import java.lang.reflect.Constructor; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; @@ -94,10 +91,10 @@ public Builder withObjectMapper(ObjectMapper objectMapper) { */ public PostgreSQLVectorStoreRecordMapper build() { if (recordClass == null) { - throw new IllegalArgumentException("recordClass is required"); + throw new SKException("recordClass is required"); } if (vectorStoreRecordDefinition == null) { - throw new IllegalArgumentException("vectorStoreRecordDefinition is required"); + throw new SKException("vectorStoreRecordDefinition is required"); } return new PostgreSQLVectorStoreRecordMapper<>( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java index 184d5723..74d42544 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java @@ -4,6 +4,7 @@ import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.exceptions.SKException; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -155,7 +156,7 @@ public Builder withPrefixCollectionName(boolean prefixCollectionName) { */ public RedisHashSetVectorStoreRecordCollectionOptions build() { if (recordClass == null) { - throw new IllegalArgumentException("recordClass must be provided"); + throw new SKException("recordClass must be provided"); } return new RedisHashSetVectorStoreRecordCollectionOptions<>( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java index 78ee1199..dba568b7 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java @@ -84,10 +84,10 @@ public Builder withVectorStoreRecordDefinition( @Override public RedisHashSetVectorStoreRecordMapper build() { if (recordClass == null) { - throw new IllegalArgumentException("recordClass is required"); + throw new SKException("recordClass is required"); } if (recordDefinition == null) { - throw new IllegalArgumentException("vectorStoreRecordDefinition is required"); + throw new SKException("vectorStoreRecordDefinition is required"); } ObjectMapper mapper = new ObjectMapper(); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java index 51c61f1c..5690e424 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java @@ -4,6 +4,7 @@ import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.exceptions.SKException; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -154,7 +155,7 @@ public Builder withPrefixCollectionName(boolean prefixCollectionName) { */ public RedisJsonVectorStoreRecordCollectionOptions build() { if (recordClass == null) { - throw new IllegalArgumentException("recordClass must be provided"); + throw new SKException("recordClass must be provided"); } return new RedisJsonVectorStoreRecordCollectionOptions<>( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java index 26241d26..d33f6d65 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java @@ -87,10 +87,10 @@ public Builder withObjectMapper(ObjectMapper objectMapper) { @Override public RedisJsonVectorStoreRecordMapper build() { if (keyFieldName == null) { - throw new IllegalArgumentException("keyFieldName is required"); + throw new SKException("keyFieldName is required"); } if (recordClass == null) { - throw new IllegalArgumentException("recordClass is required"); + throw new SKException("recordClass is required"); } return new RedisJsonVectorStoreRecordMapper<>(record -> { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java index bfa0a4bb..cd7cd27e 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java @@ -122,11 +122,11 @@ public Builder withOptions(RedisVectorStoreOptions options) { @Override public RedisVectorStore build() { if (client == null) { - throw new IllegalArgumentException("client is required"); + throw new SKException("client is required"); } if (options == null) { - throw new IllegalArgumentException("options is required"); + throw new SKException("options is required"); } return new RedisVectorStore(client, options); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java index 999daa15..ec9bd62a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java @@ -10,6 +10,8 @@ import java.util.HashSet; import java.util.List; import java.util.Map; + +import com.microsoft.semantickernel.exceptions.SKException; import redis.clients.jedis.search.Schema; public class RedisVectorStoreCollectionCreateMapping { @@ -42,7 +44,7 @@ private static String getAlgorithmMetric( case EUCLIDEAN: return RedisVectorDistanceMetric.EUCLIDEAN; default: - throw new IllegalArgumentException( + throw new SKException( "Unsupported distance function: " + vectorField.getDistanceFunction()); } } @@ -59,7 +61,7 @@ private static Schema.VectorField.VectorAlgo getAlgorithmConfig( case FLAT: return Schema.VectorField.VectorAlgo.FLAT; default: - throw new IllegalArgumentException( + throw new SKException( "Unsupported index kind: " + vectorField.getIndexKind()); } } @@ -81,7 +83,7 @@ public static Schema mapToSchema(List fields) { VectorStoreRecordDataField dataField = (VectorStoreRecordDataField) field; if (dataField.getFieldType() == null) { - throw new IllegalArgumentException( + throw new SKException( "Field type is required for filterable fields: " + dataField.getEffectiveStorageName()); } @@ -91,7 +93,7 @@ public static Schema mapToSchema(List fields) { } else if (supportedFilterableNumericTypes.contains(dataField.getFieldType())) { schema.addNumericField(getRedisPath(dataField.getEffectiveStorageName())); } else { - throw new IllegalArgumentException( + throw new SKException( "Unsupported field type for numeric filterable fields: " + dataField.getEffectiveStorageName()); } @@ -102,7 +104,7 @@ public static Schema mapToSchema(List fields) { VectorStoreRecordVectorField vectorField = (VectorStoreRecordVectorField) field; if (vectorField.getDimensions() < 1) { - throw new IllegalArgumentException( + throw new SKException( "Dimensions must be greater than 0 for vector fields: " + vectorField.getEffectiveStorageName()); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreOptions.java index d7ba95fa..2f09e930 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreOptions.java @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; +import com.microsoft.semantickernel.exceptions.SKException; + import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -98,7 +100,7 @@ public Builder withStorageType(RedisStorageType storageType) { */ public RedisVectorStoreOptions build() { if (storageType == null) { - throw new IllegalArgumentException("storageType is required"); + throw new SKException("storageType is required"); } return new RedisVectorStoreOptions(storageType, vectorStoreRecordCollectionFactory); From 1d828d2a0f120a5b32cb03ec719dc1b22d21e3a2 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Fri, 23 Aug 2024 18:01:42 +0100 Subject: [PATCH 011/277] Add TPN file. Code format --- pom.xml | 14 ++++++++++++-- .../memory/AzureAISearch_DataStorage.java | 4 ++-- .../memory/InMemory_DataStorage.java | 6 +++--- .../syntaxexamples/memory/JDBC_DataStorage.java | 4 ++-- .../syntaxexamples/memory/Redis_DataStorage.java | 6 +++--- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/pom.xml b/pom.xml index c301d7f9..f53e0e0d 100644 --- a/pom.xml +++ b/pom.xml @@ -1,6 +1,7 @@ - + 4.0.0 @@ -435,6 +436,15 @@ update-project-license + + add-third-party + + add-third-party + + + compile,runtime + + @@ -677,7 +687,7 @@ // Copyright (c) Microsoft. All rights reserved. - + diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java index 946cadc6..86666dc2 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java @@ -127,8 +127,8 @@ public static void dataStorageWithAzureAISearch( var collection = azureAISearchVectorStore.getCollection( collectionName, AzureAISearchVectorStoreRecordCollectionOptions.builder() - .withRecordClass(GitHubFile.class) - .build()); + .withRecordClass(GitHubFile.class) + .build()); // Create collection if it does not exist and store data collection diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java index 53904d57..80e6d490 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java @@ -103,9 +103,9 @@ public static void inMemoryDataStorage( String collectionName = "skgithubfiles"; var collection = volatileVectorStore.getCollection(collectionName, - VolatileVectorStoreRecordCollectionOptions.builder() - .withRecordClass(GitHubFile.class) - .build()); + VolatileVectorStoreRecordCollectionOptions.builder() + .withRecordClass(GitHubFile.class) + .build()); // Create collection if it does not exist and store data List ids = collection diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java index 90e38e26..4a9c6d10 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java @@ -134,8 +134,8 @@ public static void dataStorageWithMySQL( String collectionName = "skgithubfiles"; var collection = jdbcVectorStore.getCollection(collectionName, JDBCVectorStoreRecordCollectionOptions.builder() - .withRecordClass(GitHubFile.class) - .build()); + .withRecordClass(GitHubFile.class) + .build()); // Create collection if it does not exist and store data List ids = collection diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java index bb236788..10674365 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java @@ -122,9 +122,9 @@ public static void dataStorageWithRedis( String collectionName = "skgithubfiles"; var collection = vectorStore.getCollection(collectionName, - RedisJsonVectorStoreRecordCollectionOptions.builder() - .withRecordClass(GitHubFile.class) - .build()); + RedisJsonVectorStoreRecordCollectionOptions.builder() + .withRecordClass(GitHubFile.class) + .build()); // Create collection if it does not exist and store data List ids = collection From c81cfa2028ab4df58160b5df215fe2a74cdf0b40 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Tue, 27 Aug 2024 19:26:41 +0100 Subject: [PATCH 012/277] Implement ResourceBundle usage --- .../chatcompletion/GeminiChatCompletion.java | 7 ++- .../KernelPluginCollection.java | 4 +- .../ContextVariableTypeConverter.java | 4 +- .../exceptions/AIException.java | 27 +++++---- .../exceptions/ConfigurationException.java | 15 +++-- .../EmbeddedResourceLoader.java | 7 ++- .../implementation/ServiceLoadUtil.java | 7 ++- .../chatcompletion/ChatXMLPromptParser.java | 3 +- .../tokenizer/blocks/CodeBlock.java | 20 +++++-- .../tokenizer/blocks/NamedArgBlock.java | 11 ++-- .../tokenizer/blocks/ValBlock.java | 11 ++-- .../tokenizer/blocks/VarBlock.java | 19 ++++--- .../localization/SemanticKernelResources.java | 47 +++++++++++++++ .../orchestration/FunctionInvocation.java | 14 +++-- .../plugin/KernelPluginFactory.java | 25 +++++--- .../KernelFunctionFromMethod.java | 24 +++----- .../KernelFunctionFromPrompt.java | 20 +++---- .../services/OrderedAIServiceSelector.java | 6 +- .../semantickernel/TemplateException.java | 16 ++++-- .../localization/ResourceBundle.properties | 57 +++++++++++++++++++ .../SemanticKernelResourcesTest.java | 45 +++++++++++++++ .../HandlebarsPromptTemplateTest.java | 7 +++ .../localization/ResourceBundle.properties | 1 + .../localization/ResourceBundle_en.properties | 1 + .../ResourceBundle_en_GB.properties | 1 + 25 files changed, 304 insertions(+), 95 deletions(-) create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java create mode 100644 semantickernel-api/src/main/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties create mode 100644 semantickernel-api/src/test/java/com/microsoft/semantickernel/localization/SemanticKernelResourcesTest.java create mode 100644 semantickernel-api/src/test/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties create mode 100644 semantickernel-api/src/test/resources/com/microsoft/semantickernel/localization/ResourceBundle_en.properties create mode 100644 semantickernel-api/src/test/resources/com/microsoft/semantickernel/localization/ResourceBundle_en_GB.properties diff --git a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java index 8aa669f0..bb2c820b 100644 --- a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java +++ b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java @@ -21,6 +21,7 @@ import com.microsoft.semantickernel.exceptions.AIException; import com.microsoft.semantickernel.exceptions.SKCheckedException; import com.microsoft.semantickernel.exceptions.SKException; +import com.microsoft.semantickernel.localization.SemanticKernelResources; import com.microsoft.semantickernel.orchestration.FunctionResult; import com.microsoft.semantickernel.orchestration.FunctionResultMetadata; import com.microsoft.semantickernel.orchestration.InvocationContext; @@ -97,7 +98,8 @@ private Mono>> internalChatMessageContentsAsync( try { GenerativeModel model = getGenerativeModel(kernel, invocationContext); return MonoConverter.fromApiFuture(model.generateContentAsync(contents)) - .doOnError(e -> LOGGER.error("Error generating chat completion", e)) + .doOnError(e -> LOGGER.error( + SemanticKernelResources.getString("error.generating.chat.completion"), e)) .flatMap(result -> { // Get ChatMessageContent from the response GeminiChatMessageContent response = getGeminiChatMessageContentFromResponse( @@ -253,7 +255,8 @@ private GenerativeModel getGenerativeModel(@Nullable Kernel kernel, if (settings.getResultsPerPrompt() < 1 || settings.getResultsPerPrompt() > MAX_RESULTS_PER_PROMPT) { - throw SKCheckedException.build("Error building generative model.", + throw SKCheckedException.build( + SemanticKernelResources.getString("error.building.generative.model"), new AIException(AIException.ErrorCodes.INVALID_REQUEST, String.format( "Results per prompt must be in range between 1 and %d, inclusive.", diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/KernelPluginCollection.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/KernelPluginCollection.java index 851934d4..12d4e50a 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/KernelPluginCollection.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/KernelPluginCollection.java @@ -3,6 +3,7 @@ import com.microsoft.semantickernel.contextvariables.CaseInsensitiveMap; import com.microsoft.semantickernel.exceptions.SKException; +import com.microsoft.semantickernel.localization.SemanticKernelResources; import com.microsoft.semantickernel.plugin.KernelPlugin; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionMetadata; @@ -122,7 +123,8 @@ KernelPlugin getPlugin(String pluginName) { */ void add(KernelPlugin plugin) { if (plugins.containsKey(plugin.getName())) { - LOGGER.warn("Plugin {} already exists, overwriting existing plugin", plugin.getName()); + LOGGER.warn(SemanticKernelResources.getString( + "plugin.already.exists.overwriting.existing.plugin"), plugin.getName()); } plugins.put(plugin.getName(), plugin); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypeConverter.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypeConverter.java index 687ef302..f2e9944d 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypeConverter.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypeConverter.java @@ -2,6 +2,7 @@ package com.microsoft.semantickernel.contextvariables; import com.microsoft.semantickernel.exceptions.SKException; +import com.microsoft.semantickernel.localization.SemanticKernelResources; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.ArrayList; import java.util.Collections; @@ -144,7 +145,8 @@ public U toObject(ContextVariableTypes types, @Nullable Object t, Class c return (U) converter.get().toObject((T) t); } - LOGGER.warn("No converter found for {} to {}", t.getClass(), clazz); + LOGGER.warn(SemanticKernelResources.getString("no.converter.found.for.to"), t.getClass(), + clazz); return null; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/AIException.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/AIException.java index 17416a5b..ac6290d6 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/AIException.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/AIException.java @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.exceptions; +import com.microsoft.semantickernel.localization.SemanticKernelResources; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -65,53 +66,57 @@ public enum ErrorCodes { /** * Unknown error. */ - UNKNOWN_ERROR("Unknown error"), + UNKNOWN_ERROR(SemanticKernelResources.getString("unknown.error")), /** * No response. */ - NO_RESPONSE("No response"), + NO_RESPONSE(SemanticKernelResources.getString("no.response")), /** * Access denied. */ - ACCESS_DENIED("Access is denied"), + ACCESS_DENIED(SemanticKernelResources.getString("access.is.denied")), /** * Invalid request. */ - INVALID_REQUEST("The request was invalid"), + INVALID_REQUEST(SemanticKernelResources.getString("the.request.was.invalid")), /** * Invalid response. */ - INVALID_RESPONSE_CONTENT("The content of the response was invalid"), + INVALID_RESPONSE_CONTENT( + SemanticKernelResources.getString("the.content.of.the.response.was.invalid")), /** * Throttling. */ - THROTTLING("The request was throttled"), + THROTTLING(SemanticKernelResources.getString("the.request.was.throttled")), /** * Request timeout. */ - REQUEST_TIMEOUT("The request timed out"), + REQUEST_TIMEOUT(SemanticKernelResources.getString("the.request.timed.out")), /** * Service error. */ - SERVICE_ERROR("There was an error in the service"), + SERVICE_ERROR(SemanticKernelResources.getString("there.was.an.error.in.the.service")), /** * Model not available. */ - MODEL_NOT_AVAILABLE("The requested model is not available"), + MODEL_NOT_AVAILABLE( + SemanticKernelResources.getString("the.requested.model.is.not.available")), /** * Invalid configuration. */ - INVALID_CONFIGURATION("The supplied configuration was invalid"), + INVALID_CONFIGURATION( + SemanticKernelResources.getString("the.supplied.configuration.was.invalid")), /** * Function type not supported. */ - FUNCTION_TYPE_NOT_SUPPORTED("The function is not supported"); + FUNCTION_TYPE_NOT_SUPPORTED( + SemanticKernelResources.getString("the.function.is.not.supported")); private final String message; diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/ConfigurationException.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/ConfigurationException.java index 1784ee82..e716f1fb 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/ConfigurationException.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/ConfigurationException.java @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.exceptions; +import com.microsoft.semantickernel.localization.SemanticKernelResources; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -65,27 +66,31 @@ public enum ErrorCodes { /** * Unknown error */ - UNKNOWN_ERROR("Unknown error"), + UNKNOWN_ERROR(SemanticKernelResources.getString("unknown.error")), /** * Could not find configuration file */ - CONFIGURATION_NOT_FOUND("Could not find configuration file"), + CONFIGURATION_NOT_FOUND( + SemanticKernelResources.getString("could.not.find.configuration.file")), /** * Could not parse or load configuration file */ - COULD_NOT_READ_CONFIGURATION("Could not parse or load configuration file"), + COULD_NOT_READ_CONFIGURATION( + SemanticKernelResources.getString("could.not.parse.or.load.configuration.file")), /** * Could not find any valid configuration settings */ - NO_VALID_CONFIGURATIONS_FOUND("Could not find any valid configuration settings"), + NO_VALID_CONFIGURATIONS_FOUND( + SemanticKernelResources.getString("could.not.find.any.valid.configuration.settings")), /** * Could not find value for configuration key */ - VALUE_NOT_FOUND("Could not find value for configuration key"); + VALUE_NOT_FOUND( + SemanticKernelResources.getString("could.not.find.value.for.configuration.key")); private final String message; diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/EmbeddedResourceLoader.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/EmbeddedResourceLoader.java index 42933888..9bc9d661 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/EmbeddedResourceLoader.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/EmbeddedResourceLoader.java @@ -2,6 +2,7 @@ package com.microsoft.semantickernel.implementation; import com.microsoft.semantickernel.exceptions.SKException; +import com.microsoft.semantickernel.localization.SemanticKernelResources; import java.io.BufferedReader; import java.io.File; import java.io.FileNotFoundException; @@ -9,6 +10,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.file.Files; +import java.text.MessageFormat; import java.util.Arrays; import java.util.List; import java.util.Objects; @@ -126,7 +128,10 @@ private static String readInputStream(String fileName, InputStream inputStream) return bf.lines().collect(Collectors.joining("\n")); } catch (IOException e) { // IGNORE - LOGGER.trace("Failed to load file: " + fileName, e); + LOGGER.trace( + MessageFormat.format(SemanticKernelResources.getString("failed.to.load.file.0"), + fileName), + e); } return null; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/ServiceLoadUtil.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/ServiceLoadUtil.java index ede54246..fe4544df 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/ServiceLoadUtil.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/ServiceLoadUtil.java @@ -8,6 +8,7 @@ import java.util.List; import java.util.ServiceLoader; import java.util.function.Supplier; +import com.microsoft.semantickernel.localization.SemanticKernelResources; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,11 +55,13 @@ public static Supplier findServiceLoader(Class clazz, String alternati | IllegalAccessException | NoSuchMethodException | RuntimeException e) { - LOGGER.error("Unable to load service " + clazz.getName() + " ", e); + LOGGER.error(String.format( + SemanticKernelResources.getString("unable.to.load.service.s"), clazz.getName()), + e); } if (impl == null) { - throw new RuntimeException("Service not found: " + clazz.getName()); + throw new RuntimeException(String.format("Service not found: %s", clazz.getName())); } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/chatcompletion/ChatXMLPromptParser.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/chatcompletion/ChatXMLPromptParser.java index 1c8f78c6..c1c9592a 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/chatcompletion/ChatXMLPromptParser.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/chatcompletion/ChatXMLPromptParser.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.exceptions.SKException; +import com.microsoft.semantickernel.localization.SemanticKernelResources; import com.microsoft.semantickernel.orchestration.ToolCallBehavior; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -203,7 +204,7 @@ private static ChatPromptParseVisitor getFunctionDefinitions(String promp } } } catch (IOException | XMLStreamException | IllegalArgumentException e) { - LOGGER.error("Error parsing prompt", e); + LOGGER.error(SemanticKernelResources.getString("error.parsing.prompt"), e); } return chatPromptParseVisitor; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/CodeBlock.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/CodeBlock.java index 50d7c469..e294c925 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/CodeBlock.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/CodeBlock.java @@ -6,12 +6,14 @@ import com.microsoft.semantickernel.contextvariables.ContextVariableType; import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; import com.microsoft.semantickernel.exceptions.SKException; +import com.microsoft.semantickernel.localization.SemanticKernelResources; import com.microsoft.semantickernel.orchestration.FunctionResult; import com.microsoft.semantickernel.orchestration.InvocationContext; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionMetadata; import com.microsoft.semantickernel.templateengine.semantickernel.TemplateException; import com.microsoft.semantickernel.templateengine.semantickernel.TemplateException.ErrorCodes; +import java.text.MessageFormat; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -44,12 +46,14 @@ public CodeBlock(List tokens, String content) { public boolean isValid() { Optional invalid = tokens.stream().filter(token -> !token.isValid()).findFirst(); if (invalid.isPresent()) { - LOGGER.error("Invalid block" + invalid.get().getContent()); + LOGGER.error(MessageFormat.format(SemanticKernelResources.getString("invalid.block.0"), + invalid.get().getContent())); return false; } if (!this.tokens.isEmpty() && this.tokens.get(0).getType() == BlockTypes.NAMED_ARG) { - LOGGER.error("Unexpected named argument found. Expected function name first."); + LOGGER.error(SemanticKernelResources.getString( + "unexpected.named.argument.found.expected.function.name.first")); return false; } @@ -62,7 +66,9 @@ public boolean isValid() { private boolean isValidFunctionCall() { if (this.tokens.get(0).getType() != BlockTypes.FUNCTION_ID) { - LOGGER.error("Unexpected second token found: " + tokens.get(1).getContent()); + LOGGER.error(MessageFormat.format( + SemanticKernelResources.getString("unexpected.second.token.found.0"), + tokens.get(1).getContent())); return false; } @@ -70,15 +76,17 @@ private boolean isValidFunctionCall() { this.tokens.get(1).getType() != BlockTypes.VARIABLE && this.tokens.get(1).getType() != BlockTypes.NAMED_ARG) { LOGGER.error( - "The first arg of a function must be a quoted string, variable or named argument"); + SemanticKernelResources.getString( + "the.first.arg.of.a.function.must.be.a.quoted.string.variable.or.named.argument")); return false; } for (int i = 2; i < this.tokens.size(); i++) { if (this.tokens.get(i).getType() != BlockTypes.NAMED_ARG) { LOGGER.error( - "Functions only support named arguments after the first argument. Argument " + i - + " is not named."); + SemanticKernelResources.getString( + "functions.only.support.named.arguments.after.the.first.argument"), + i); return false; } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/NamedArgBlock.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/NamedArgBlock.java index 30403123..010ba44e 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/NamedArgBlock.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/NamedArgBlock.java @@ -6,6 +6,7 @@ import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; import com.microsoft.semantickernel.exceptions.SKException; import com.microsoft.semantickernel.implementation.Verify; +import com.microsoft.semantickernel.localization.SemanticKernelResources; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; import javax.annotation.Nullable; import org.slf4j.Logger; @@ -109,18 +110,20 @@ private static String splitAndGetPart(String text, int x) { @Override public boolean isValid() { if (Verify.isNullOrEmpty(this.name)) { - LOGGER.error("A named argument must have a name"); + LOGGER.error(SemanticKernelResources.getString("a.named.argument.must.have.a.name")); return false; } if (this.valBlock != null && !this.valBlock.isValid()) { - LOGGER.error("There was an issue with the named argument value for '" + name); + LOGGER.error(SemanticKernelResources.getString( + "there.was.an.issue.with.the.named.argument.value.for"), name); return false; } else if (this.varBlock != null && !this.varBlock.isValid()) { - LOGGER.error("There was an issue with the named argument value for '" + name); + LOGGER.error(SemanticKernelResources.getString( + "there.was.an.issue.with.the.named.argument.value.for"), name); return false; } else if (this.valBlock == null && this.varBlock == null) { - LOGGER.error("A named argument must have a value"); + LOGGER.error(SemanticKernelResources.getString("a.named.argument.must.have.a.value")); return false; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/ValBlock.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/ValBlock.java index bdd10f69..3d647840 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/ValBlock.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/ValBlock.java @@ -2,6 +2,7 @@ package com.microsoft.semantickernel.implementation.templateengine.tokenizer.blocks; import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; +import com.microsoft.semantickernel.localization.SemanticKernelResources; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,7 +23,8 @@ public ValBlock(String quotedValue) { super(quotedValue.trim(), BlockTypes.VALUE); if (this.getContent().length() < 2) { - LOGGER.error("A value must have single quotes or double quotes on both sides"); + LOGGER.error(SemanticKernelResources.getString( + "a.value.must.have.single.quotes.or.double.quotes.on.both.sides")); return; } @@ -47,15 +49,16 @@ public String render(ContextVariableTypes types, @Nullable KernelFunctionArgumen public boolean isValid() { // Content includes the quotes, so it must be at least 2 chars long if (this.getContent().length() < 2) { - LOGGER.error("A value must have single quotes or double quotes on both sides"); + LOGGER.error(SemanticKernelResources.getString( + "a.value.must.have.single.quotes.or.double.quotes.on.both.sides")); return false; } // Check if delimiting chars are consistent if (first != last) { LOGGER.error( - "A value must be defined using either single quotes or double quotes, not" - + " both"); + SemanticKernelResources.getString( + "a.value.must.be.defined.using.either.single.quotes.or.double.quotes.not.both")); return false; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/VarBlock.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/VarBlock.java index 8514e8ab..1aec228c 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/VarBlock.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/VarBlock.java @@ -3,6 +3,7 @@ import com.microsoft.semantickernel.contextvariables.ContextVariable; import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; +import com.microsoft.semantickernel.localization.SemanticKernelResources; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; import com.microsoft.semantickernel.templateengine.semantickernel.TemplateException; import javax.annotation.Nullable; @@ -18,7 +19,7 @@ public VarBlock(String content) { super(content, BlockTypes.VARIABLE); if (content.length() < 2) { - LOGGER.error("The variable name is empty"); + LOGGER.error(SemanticKernelResources.getString("the.variable.name.is.empty")); } this.name = content.substring(1); @@ -40,7 +41,8 @@ public String render(ContextVariableTypes types, @Nullable KernelFunctionArgumen name); if (value == null) { - LOGGER.warn("Variable `{}{}` not found", Symbols.VarPrefix, name); + LOGGER.warn(SemanticKernelResources.getString("variable.not.found"), Symbols.VarPrefix, + name); } return value != null ? value.toPromptString(types) : ""; @@ -50,24 +52,27 @@ public String render(ContextVariableTypes types, @Nullable KernelFunctionArgumen public boolean isValid() { if (getContent() == null || getContent().isEmpty()) { LOGGER.error( - "A variable must start with the symbol {} and have a name", Symbols.VarPrefix); + SemanticKernelResources.getString( + "a.variable.must.start.with.the.symbol.and.have.a.name"), + Symbols.VarPrefix); return false; } if (getContent().charAt(0) != Symbols.VarPrefix) { - LOGGER.error("A variable must start with the symbol {}", Symbols.VarPrefix); + LOGGER.error(SemanticKernelResources.getString("a.variable.must.start.with.the.symbol"), + Symbols.VarPrefix); return false; } if (getContent().length() < 2) { - LOGGER.error("The variable name is empty"); + LOGGER.error(SemanticKernelResources.getString("the.variable.name.is.empty")); return false; } if (!name.matches("^[a-zA-Z0-9_]*$")) { LOGGER.error( - "The variable name '{}' contains invalid characters. " - + "Only alphanumeric chars and underscore are allowed.", + SemanticKernelResources.getString( + "the.variable.name.contains.invalid.characters.only.alphanumeric.chars.and.underscore.are.allowed"), name); return false; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java new file mode 100644 index 00000000..d6ca091f --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.localization; + +import java.util.Locale; +import java.util.MissingResourceException; +import java.util.PropertyResourceBundle; +import java.util.ResourceBundle; + +public class SemanticKernelResources { + + private static ResourceBundle RESOURCE_BUNDLE; + private static Locale LOCALE; + + static { + LOCALE = new Locale( + System.getProperty("semantickernel.locale", + String.valueOf(Locale.getDefault().getLanguage()))); + + setLocale(LOCALE); + } + + public static void setLocale(Locale locale) { + LOCALE = locale; + ResourceBundle resourceBundle; + try { + resourceBundle = PropertyResourceBundle.getBundle( + "com.microsoft.semantickernel.localization.ResourceBundle", locale); + } catch (MissingResourceException e) { + resourceBundle = PropertyResourceBundle.getBundle( + "com.microsoft.semantickernel.localization.ResourceBundle"); + } + RESOURCE_BUNDLE = resourceBundle; + } + + public static String localize(String id, String defaultValue) { + if (RESOURCE_BUNDLE.containsKey(id)) { + return RESOURCE_BUNDLE.getString(id); + } else { + return defaultValue; + } + } + + public static String getString(String s) { + return localize(s, s); + } + +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java index 4cd6ae2a..bcae014d 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java @@ -10,6 +10,7 @@ import com.microsoft.semantickernel.hooks.KernelHook; import com.microsoft.semantickernel.hooks.KernelHooks; import com.microsoft.semantickernel.hooks.KernelHooks.UnmodifiableKernelHooks; +import com.microsoft.semantickernel.localization.SemanticKernelResources; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -95,9 +96,10 @@ private static void performSubscribe( @Nullable InvocationContext context) { if (variableType == null) { LOGGER.debug( - "No variable type explicitly specified by calling 'withResultType' for function invocation: " - + function.getPluginName() + "." + function.getName() + "." - + " This may cause a runtime error (probably a ClassCastException) if the result type is not compatible with the expected type."); + SemanticKernelResources.getString( + "no.variable.type.explicitly.specified.by.calling.withresulttype.for.function"), + function.getPluginName(), + function.getName()); } function @@ -309,7 +311,8 @@ public FunctionInvocation withInvocationContext( private void logSubscribeWarning() { if (isSubscribed) { LOGGER.warn( - "Attempting to modify function {}.{} after it has already been subscribed to. This is not necessarily an error but may be an unusual pattern and indicate a potential bug.", + SemanticKernelResources.getString( + "attempting.to.modify.function.after.it.has.already.been.subscribed"), function.getPluginName(), function.getName()); } } @@ -324,7 +327,8 @@ public void subscribe(CoreSubscriber> coreSubscriber) if (isSubscribed) { LOGGER.warn( - "Function {}.{} has already been subscribed to. This is not necessarily an error but may be an unusual pattern.", + SemanticKernelResources.getString( + "function.has.already.been.subscribed.to.this.is.not.necessarily.an.error.but.may.be.an.unusual.pattern"), function.getPluginName(), function.getName()); } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/plugin/KernelPluginFactory.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/plugin/KernelPluginFactory.java index f6e8ad1e..a6a23c3b 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/plugin/KernelPluginFactory.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/plugin/KernelPluginFactory.java @@ -6,6 +6,7 @@ import com.microsoft.semantickernel.exceptions.SKException; import com.microsoft.semantickernel.implementation.EmbeddedResourceLoader; import com.microsoft.semantickernel.implementation.EmbeddedResourceLoader.ResourceLocation; +import com.microsoft.semantickernel.localization.SemanticKernelResources; import com.microsoft.semantickernel.semanticfunctions.InputVariable; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionFromMethod; @@ -143,8 +144,8 @@ public static KernelPlugin createFromObject(Class clazz, Object target, Strin if (plugin.getFunctions().isEmpty()) { LOGGER.warn( - "No functions found in class {}. This can be caused by DI frameworks that create proxies, or modules that are not making your methods visible. " - + "Try using: KernelPluginFactory.createFromObject(Class clazz, Object target, String pluginName).", + SemanticKernelResources.getString( + "no.functions.found.in.class.this.can.be.caused.by"), clazz.getName()); } return plugin; @@ -360,7 +361,7 @@ public static KernelPlugin importPluginFromDirectory( plugins.put(dir.getName(), plugin); } catch (IOException e) { - LOGGER.error("Failed to read file", e); + LOGGER.error(SemanticKernelResources.getString("failed.to.read.file"), e); } } @@ -386,7 +387,8 @@ private static KernelFunction getKernelFunction( return getKernelFunction(functionName, promptTemplateFactory, config, template); } catch (Exception e) { - LOGGER.error("Failed to read file " + configPath.getAbsolutePath(), e); + LOGGER.error(SemanticKernelResources.getString("failed.to.read.file1"), + configPath.getAbsolutePath(), e); throw new SKException("Failed to read function " + configPath.getAbsolutePath(), e); } @@ -474,8 +476,10 @@ public static KernelPlugin importPluginFromResourcesDirectory( pluginDirectoryName, functionName, clazz); if (promptTemplateConfig == null) { - LOGGER.warn("Unable to load prompt template config for " + functionName + " in " - + pluginDirectoryName); + LOGGER.warn( + SemanticKernelResources.getString("unable.to.load.prompt.template.config.for.in"), + functionName, + pluginDirectoryName); return null; } KernelFunction function = getKernelFunction(functionName, promptTemplateFactory, @@ -503,7 +507,8 @@ private static String getTemplatePrompt( try { return getFileContents(promptFileName, clazz); } catch (IOException e) { - LOGGER.error("Failed to read file " + promptFileName, e); + LOGGER.error(SemanticKernelResources.getString("failed.to.read.file1"), promptFileName, + e); throw new SKException("No plugins found in directory " + promptFileName); } @@ -535,11 +540,13 @@ private static PromptTemplateConfig getPromptTemplateConfig( return PromptTemplateConfig.parseFromJson(config); } catch (Exception e) { if (e instanceof SKException) { - LOGGER.error("Failed to parse config file " + configFileName, e); + LOGGER.error(SemanticKernelResources.getString("failed.to.parse.config.file"), + configFileName, e); throw new SKException("Failed to parse config file " + configFileName, e); } else { - LOGGER.debug("No config for " + functionName + " in " + pluginName); + LOGGER.debug(SemanticKernelResources.getString("no.config.for.in"), functionName, + pluginName); } return null; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java index 9a7b09dc..9a67b575 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java @@ -14,6 +14,7 @@ import com.microsoft.semantickernel.hooks.FunctionInvokedEvent; import com.microsoft.semantickernel.hooks.FunctionInvokingEvent; import com.microsoft.semantickernel.hooks.KernelHooks; +import com.microsoft.semantickernel.localization.SemanticKernelResources; import com.microsoft.semantickernel.orchestration.FunctionResult; import com.microsoft.semantickernel.orchestration.InvocationContext; import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; @@ -22,6 +23,7 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Parameter; +import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -400,7 +402,7 @@ private static Object getArgumentValue( if (requestedType == String.class && !targetArgType.isAssignableFrom(requestedType)) { LOGGER.warn( - "Annotation on method: {} is requesting a String which is not assignable to method type {}, possibly as the type argument has not been provided on the annotation.", + SemanticKernelResources.getString("annotation.on.method.is.requesting.a.string"), method.getName(), targetArgType.getName()); } @@ -632,21 +634,11 @@ private static String getGetVariableName(Parameter parameter) { private static String formErrorMessage(Method method, Parameter parameter) { Matcher matcher = Pattern.compile("arg(\\d)").matcher(parameter.getName()); matcher.find(); - return "For the function " - + method.getDeclaringClass().getName() - + "." - + method.getName() - + ", the unknown parameter" - + " name was detected as \"" - + parameter.getName() - + "\" this is argument" - + " number " - + matcher.group(1) - + " to the function, this indicates that the argument name for this function was" - + " removed during compilation and semantic-kernel is unable to determine the name" - + " of the parameter. To support this function the argument must be annotated with" - + " @SKFunctionParameters or @SKFunctionInputAttribute. Alternatively the function" - + " was invoked with a required context variable missing and no default value."; + return MessageFormat.format( + SemanticKernelResources.getString( + "for.the.function.0.1.the.unknown.parameter.name.was.detected"), + method.getDeclaringClass().getName(), method.getName(), parameter.getName(), + matcher.group(1)); } private static List getParameters(Method method) { diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java index 5d124d84..697da535 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java @@ -10,6 +10,7 @@ import com.microsoft.semantickernel.hooks.KernelHooks; import com.microsoft.semantickernel.hooks.PromptRenderedEvent; import com.microsoft.semantickernel.hooks.PromptRenderingEvent; +import com.microsoft.semantickernel.localization.SemanticKernelResources; import com.microsoft.semantickernel.orchestration.FunctionResult; import com.microsoft.semantickernel.orchestration.InvocationContext; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; @@ -126,7 +127,7 @@ private Flux> invokeInternalAsync( prompt = promptHookResult.getPrompt(); KernelFunctionArguments args = promptHookResult.getArguments(); - LOGGER.info("RENDERED PROMPT: \n{}", prompt); + LOGGER.info(SemanticKernelResources.getString("rendered.prompt"), prompt); FunctionInvokingEvent updateArguments = kernelHooks .executeHooks(new FunctionInvokingEvent(this, args)); @@ -246,9 +247,8 @@ private Flux> invokeInternalAsync( .doOnError( ex -> { LOGGER.warn( - "Something went wrong while rendering the semantic" - + " function or while executing the text" - + " completion. Function: {}.{}. Error: {}", + SemanticKernelResources.getString( + "something.went.wrong.while.rendering.the.semantic.function.or.while.executing.the.text.completion.function.error"), getPluginName(), getName(), ex.getMessage()); @@ -260,16 +260,10 @@ private Flux> invokeInternalAsync( if (ex instanceof HttpResponseException && ((HttpResponseException) ex).getResponse().getStatusCode() == 400 && ex.getMessage() != null - && ex.getMessage().contains("parameters are not available" + " on")) { + && ex.getMessage().contains("parameters are not available on")) { LOGGER.warn( - "This error indicates that you have attempted" - + " to use a chat completion model in a" - + " text completion service. Try using a" - + " chat completion service instead when" - + " building your kernel, for instance when" - + " building your service use" - + " SKBuilders.chatCompletion() rather than" - + " SKBuilders.textCompletionService()."); + SemanticKernelResources.getString( + "this.error.indicates.that.you.have.attempted.to.use.a.chat.completion.model")); } }); } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/OrderedAIServiceSelector.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/OrderedAIServiceSelector.java index 411ba987..8e607cdb 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/OrderedAIServiceSelector.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/OrderedAIServiceSelector.java @@ -2,6 +2,7 @@ package com.microsoft.semantickernel.services; import com.microsoft.semantickernel.implementation.Verify; +import com.microsoft.semantickernel.localization.SemanticKernelResources; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; @@ -155,7 +156,7 @@ public AIServiceSelection trySelectAIService( return castServiceSelection(new AIServiceSelection<>(service, settings)); } - LOGGER.warn("No service found meeting requirements"); + LOGGER.warn(SemanticKernelResources.getString("no.service.found.meeting.requirements")); return null; } @@ -211,7 +212,8 @@ public T getService(Class clazz) { (clazz.equals(TextGenerationService.class) || clazz.equals(ChatCompletionService.class))) { LOGGER.warn( - "Requested a non-existent service type of {}. Consider requesting a TextAIService instead.", + SemanticKernelResources.getString( + "requested.a.non.existent.service.type.of.consider.requesting.a.textaiservice.instead"), clazz.getName()); } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/templateengine/semantickernel/TemplateException.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/templateengine/semantickernel/TemplateException.java index 6c5779b4..e0fe492c 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/templateengine/semantickernel/TemplateException.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/templateengine/semantickernel/TemplateException.java @@ -2,6 +2,7 @@ package com.microsoft.semantickernel.templateengine.semantickernel; import com.microsoft.semantickernel.exceptions.SKException; +import com.microsoft.semantickernel.localization.SemanticKernelResources; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -78,34 +79,39 @@ public enum ErrorCodes { * Unknown error. */ //spotless:on - UNKNOWN_ERROR("Unknown error"), + UNKNOWN_ERROR(SemanticKernelResources.getString("unknown.error")), // spotless:off /** * Syntax error, the template syntax used is not valid. */ //spotless:on - SYNTAX_ERROR("Syntax error, the template syntax used is not valid"), + SYNTAX_ERROR( + SemanticKernelResources + .getString("syntax.error.the.template.syntax.used.is.not.valid")), // spotless:off /** * The block type produced be the tokenizer was not expected. */ - UNEXPECTED_BLOCK_TYPE("The block type produced be the tokenizer was not expected"), + UNEXPECTED_BLOCK_TYPE(SemanticKernelResources.getString( + "the.block.type.produced.be.the.tokenizer.was.not.expected")), //spotless:off /** * The template requires an unknown function. */ //spotless:on - FUNCTION_NOT_FOUND("The template requires an unknown function"), + FUNCTION_NOT_FOUND( + SemanticKernelResources.getString("the.template.requires.an.unknown.function")), // spotless:off /** * The template execution failed, e.g. a function call threw an exception. */ //spotless:on - RUNTIME_ERROR("The template execution failed, e.g. a function call threw an exception"); + RUNTIME_ERROR(SemanticKernelResources.getString( + "the.template.execution.failed.e.g.a.function.call.threw.an.exception")); private final String message; diff --git a/semantickernel-api/src/main/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties b/semantickernel-api/src/main/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties new file mode 100644 index 00000000..cf1d17b0 --- /dev/null +++ b/semantickernel-api/src/main/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties @@ -0,0 +1,57 @@ +a.named.argument.must.have.a.name=A named argument must have a name +a.named.argument.must.have.a.value=A named argument must have a value +a.value.must.be.defined.using.either.single.quotes.or.double.quotes.not.both=A value must be defined using either single quotes or double quotes, not both +a.value.must.have.single.quotes.or.double.quotes.on.both.sides=A value must have single quotes or double quotes on both sides +a.variable.must.start.with.the.symbol=A variable must start with the symbol {} +a.variable.must.start.with.the.symbol.and.have.a.name=A variable must start with the symbol {} and have a name +access.is.denied=Access is denied +annotation.on.method.is.requesting.a.string=Annotation on method: {} is requesting a String which is not assignable to method type {}, possibly as the type argument has not been provided on the annotation. +attempting.to.modify.function.after.it.has.already.been.subscribed=Attempting to modify function {}.{} after it has already been subscribed to. This is not necessarily an error but may be an unusual pattern and indicate a potential bug. +could.not.find.any.valid.configuration.settings=Could not find any valid configuration settings +could.not.find.configuration.file=Could not find configuration file +could.not.find.value.for.configuration.key=Could not find value for configuration key +could.not.parse.or.load.configuration.file=Could not parse or load configuration file +error.building.generative.model=Error building generative model. +error.generating.chat.completion=Error generating chat completion +error.parsing.prompt=Error parsing prompt +failed.to.load.file.0=Failed to load file: {0} +failed.to.parse.config.file=Failed to parse config file {} +failed.to.read.file=Failed to read file +failed.to.read.file1=Failed to read file {} +for.the.function.0.1.the.unknown.parameter.name.was.detected=For the function {0}.{1}, the unknown parameter name was detected as "{2}" this is argument number {3} to the function, this indicates that the argument name for this function was removed during compilation and semantic-kernel is unable to determine the name of the parameter. To support this function the argument must be annotated with @SKFunctionParameters or @SKFunctionInputAttribute. Alternatively the function was invoked with a required context variable missing and no default value. +function.has.already.been.subscribed.to.this.is.not.necessarily.an.error.but.may.be.an.unusual.pattern=Function {}.{} has already been subscribed to. This is not necessarily an error but may be an unusual pattern. +functions.only.support.named.arguments.after.the.first.argument=Functions only support named arguments after the first argument. Argument {} is not named. +invalid.block.0=Invalid block{0} +no.config.for.in=No config for {} in {} +no.converter.found.for.to=No converter found for {} to {} +no.functions.found.in.class.this.can.be.caused.by=No functions found in class {}. This can be caused by DI frameworks that create proxies, or modules that are not making your methods visible. Try using: KernelPluginFactory.createFromObject(Class clazz, Object target, String pluginName). +no.response=No response +no.service.found.meeting.requirements=No service found meeting requirements +no.variable.type.explicitly.specified.by.calling.withresulttype.for.function=No variable type explicitly specified by calling 'withResultType' for function invocation: {}.{}. This may cause a runtime error (probably a ClassCastException) if the result type is not compatible with the expected type. +plugin.already.exists.overwriting.existing.plugin=Plugin {} already exists, overwriting existing plugin +rendered.prompt=RENDERED PROMPT: \n{} +requested.a.non.existent.service.type.of.consider.requesting.a.textaiservice.instead=Requested a non-existent service type of {}. Consider requesting a TextAIService instead. +something.went.wrong.while.rendering.the.semantic.function.or.while.executing.the.text.completion.function.error=Something went wrong while rendering the semantic function or while executing the text completion. Function: {}.{}. Error: {} +syntax.error.the.template.syntax.used.is.not.valid=Syntax error, the template syntax used is not valid +the.block.type.produced.be.the.tokenizer.was.not.expected=The block type produced be the tokenizer was not expected +the.content.of.the.response.was.invalid=The content of the response was invalid +the.first.arg.of.a.function.must.be.a.quoted.string.variable.or.named.argument=The first arg of a function must be a quoted string, variable or named argument +the.function.is.not.supported=The function is not supported +the.request.timed.out=The request timed out +the.request.was.invalid=The request was invalid +the.request.was.throttled=The request was throttled +the.requested.model.is.not.available=The requested model is not available +the.supplied.configuration.was.invalid=The supplied configuration was invalid +the.template.execution.failed.e.g.a.function.call.threw.an.exception=The template execution failed, e.g. a function call threw an exception +the.template.requires.an.unknown.function=The template requires an unknown function +the.variable.name.contains.invalid.characters.only.alphanumeric.chars.and.underscore.are.allowed=The variable name '{}' contains invalid characters. Only alphanumeric chars and underscore are allowed. +the.variable.name.is.empty=The variable name is empty +there.was.an.error.in.the.service=There was an error in the service +there.was.an.issue.with.the.named.argument.value.for=There was an issue with the named argument value for {} +this.error.indicates.that.you.have.attempted.to.use.a.chat.completion.model=This error indicates that you have attempted to use a chat completion model in a text completion service. Try using a chat completion service instead when building your kernel, for instance when building your service use SKBuilders.chatCompletion() rather than SKBuilders.textCompletionService(). +unable.to.load.prompt.template.config.for.in=Unable to load prompt template config for {} in {} +unable.to.load.service.s=Unable to load service %s +unexpected.named.argument.found.expected.function.name.first=Unexpected named argument found. Expected function name first. +unexpected.second.token.found.0=Unexpected second token found: {0} +unknown.error=Unknown error +variable.not.found=Variable `{}{}` not found \ No newline at end of file diff --git a/semantickernel-api/src/test/java/com/microsoft/semantickernel/localization/SemanticKernelResourcesTest.java b/semantickernel-api/src/test/java/com/microsoft/semantickernel/localization/SemanticKernelResourcesTest.java new file mode 100644 index 00000000..9158255f --- /dev/null +++ b/semantickernel-api/src/test/java/com/microsoft/semantickernel/localization/SemanticKernelResourcesTest.java @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.localization; + +import java.util.Locale; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class SemanticKernelResourcesTest { + + @Test + public void languageAndCountry() { + SemanticKernelResources.setLocale(new Locale("en", "GB")); + + String result = SemanticKernelResources.localize("test_language_country", "a test string"); + + Assertions.assertEquals("GB english test value", result); + } + + @Test + public void valueAtTheLanguageLevel() { + SemanticKernelResources.setLocale(new Locale("en", "GB")); + + String result = SemanticKernelResources.localize("test_language", "a test string"); + + Assertions.assertEquals("English test value", result); + } + + @Test + public void topLevelValue() { + SemanticKernelResources.setLocale(new Locale("en", "GB")); + + String result = SemanticKernelResources.localize("test_top", "default value"); + + Assertions.assertEquals("Top level value", result); + } + + @Test + public void defaultValue() { + SemanticKernelResources.setLocale(new Locale("en", "GB")); + + String result = SemanticKernelResources.localize("not-there", "default value"); + + Assertions.assertEquals("default value", result); + } +} diff --git a/semantickernel-api/src/test/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplateTest.java b/semantickernel-api/src/test/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplateTest.java index eec1bfe7..beedb9b3 100644 --- a/semantickernel-api/src/test/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplateTest.java +++ b/semantickernel-api/src/test/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplateTest.java @@ -5,7 +5,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.Kernel; +import com.microsoft.semantickernel.contextvariables.ContextVariable; +import com.microsoft.semantickernel.contextvariables.ContextVariableTypeConverter; +import com.microsoft.semantickernel.contextvariables.converters.ContextVariableJacksonConverter; +import com.microsoft.semantickernel.orchestration.InvocationContext; import com.microsoft.semantickernel.plugin.KernelPlugin; import com.microsoft.semantickernel.plugin.KernelPluginFactory; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; @@ -17,6 +22,8 @@ import java.util.Arrays; import java.util.List; import java.util.Locale; +import org.apache.commons.text.StringEscapeUtils; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; /** diff --git a/semantickernel-api/src/test/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties b/semantickernel-api/src/test/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties new file mode 100644 index 00000000..6535223b --- /dev/null +++ b/semantickernel-api/src/test/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties @@ -0,0 +1 @@ +test_top=Top level value \ No newline at end of file diff --git a/semantickernel-api/src/test/resources/com/microsoft/semantickernel/localization/ResourceBundle_en.properties b/semantickernel-api/src/test/resources/com/microsoft/semantickernel/localization/ResourceBundle_en.properties new file mode 100644 index 00000000..8f7fc02e --- /dev/null +++ b/semantickernel-api/src/test/resources/com/microsoft/semantickernel/localization/ResourceBundle_en.properties @@ -0,0 +1 @@ +test_language=English test value \ No newline at end of file diff --git a/semantickernel-api/src/test/resources/com/microsoft/semantickernel/localization/ResourceBundle_en_GB.properties b/semantickernel-api/src/test/resources/com/microsoft/semantickernel/localization/ResourceBundle_en_GB.properties new file mode 100644 index 00000000..9acef168 --- /dev/null +++ b/semantickernel-api/src/test/resources/com/microsoft/semantickernel/localization/ResourceBundle_en_GB.properties @@ -0,0 +1 @@ +test_language_country=GB english test value \ No newline at end of file From d3a5e0b3975e02023956f5f3d1c1eee38b882bd6 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 29 Aug 2024 00:16:42 -0700 Subject: [PATCH 013/277] Add api classes for vector search Add full text support for azure ai search Re-org packages --- .../tests/connectors/memory/Hotel.java | 8 +- .../JDBCVectorStoreRecordCollectionTest.java | 2 +- ...ashSetVectorStoreRecordCollectionTest.java | 12 +- ...isJsonVectorStoreRecordCollectionTest.java | 12 +- .../samples/syntaxexamples/RunAll.java | 4 +- ...age.java => AzureAISearchVectorStore.java} | 48 +++--- .../memory/InMemory_DataStorage.java | 10 +- .../memory/JDBC_DataStorage.java | 8 +- .../memory/Redis_DataStorage.java | 10 +- .../AzureAISearchVectorStore.java | 32 ++-- ...rchVectorStoreCollectionCreateMapping.java | 14 +- ...rchVectorStoreCollectionSearchMapping.java | 75 +++++++++ ...reAISearchVectorStoreRecordCollection.java | 144 +++++++++++++----- ...rchVectorStoreRecordCollectionFactory.java | 2 +- ...rchVectorStoreRecordCollectionOptions.java | 2 +- .../connectors/data/jdbc/JDBCVectorStore.java | 1 - .../JDBCVectorStoreDefaultQueryProvider.java | 10 +- .../jdbc/JDBCVectorStoreQueryProvider.java | 8 +- .../jdbc/JDBCVectorStoreRecordCollection.java | 8 +- ...DBCVectorStoreRecordCollectionFactory.java | 2 +- ...DBCVectorStoreRecordCollectionOptions.java | 2 +- .../jdbc/JDBCVectorStoreRecordMapper.java | 6 +- .../mysql/MySQLVectorStoreQueryProvider.java | 8 +- .../PostgreSQLVectorStoreQueryProvider.java | 10 +- .../PostgreSQLVectorStoreRecordMapper.java | 6 +- ...disHashSetVectorStoreRecordCollection.java | 10 +- ...SetVectorStoreRecordCollectionOptions.java | 2 +- .../RedisHashSetVectorStoreRecordMapper.java | 7 +- .../RedisJsonVectorStoreRecordCollection.java | 10 +- ...sonVectorStoreRecordCollectionOptions.java | 2 +- .../data/redis/RedisVectorStore.java | 1 - ...disVectorStoreCollectionCreateMapping.java | 8 +- ...disVectorStoreRecordCollectionFactory.java | 2 +- .../semantickernel/data/VectorSearch.java | 29 ++++ .../data/VectorSearchResult.java | 36 +++++ .../semantickernel/data/VectorStore.java | 3 +- .../data/VectorStoreRecordCollection.java | 6 +- .../VectorStoreRecordCollectionOptions.java | 2 +- .../data/VolatileVectorStore.java | 2 - .../VolatileVectorStoreRecordCollection.java | 9 +- ...ileVectorStoreRecordCollectionOptions.java | 2 +- .../VectorStoreRecordDataAttribute.java | 15 +- .../VectorStoreRecordKeyAttribute.java | 2 +- .../VectorStoreRecordVectorAttribute.java | 2 +- .../definition}/DistanceFunction.java | 2 +- .../definition}/IndexKind.java | 2 +- .../VectorStoreRecordDataField.java | 82 +++------- .../VectorStoreRecordDefinition.java | 19 ++- .../definition}/VectorStoreRecordField.java | 2 +- .../VectorStoreRecordKeyField.java | 2 +- .../VectorStoreRecordVectorField.java | 2 +- .../options}/DeleteRecordOptions.java | 2 +- .../options}/GetRecordOptions.java | 2 +- .../options}/UpsertRecordOptions.java | 2 +- .../options/BasicVectorSearchFilter.java | 57 +++++++ .../options/EqualityFilterClause.java | 22 +++ .../vectorsearch/options/FilterClause.java | 28 ++++ .../options/FilterClauseType.java | 13 ++ .../options/TagListContainsFilterClause.java | 37 +++++ .../options/VectorSearchOptions.java | 139 +++++++++++++++++ .../queries/VectorSearchQuery.java | 33 ++++ .../queries/VectorSearchQueryType.java | 20 +++ .../queries/VectorizableTextSearchQuery.java | 30 ++++ .../queries/VectorizedSearchQuery.java | 29 ++++ .../microsoft/semantickernel/data/Hotel.java | 8 +- 65 files changed, 859 insertions(+), 268 deletions(-) rename samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/{AzureAISearch_DataStorage.java => AzureAISearchVectorStore.java} (81%) create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearch.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearchResult.java rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{recordattributes => record/attributes}/VectorStoreRecordDataAttribute.java (74%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{recordattributes => record/attributes}/VectorStoreRecordKeyAttribute.java (88%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{recordattributes => record/attributes}/VectorStoreRecordVectorAttribute.java (92%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{recorddefinition => record/definition}/DistanceFunction.java (94%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{recorddefinition => record/definition}/IndexKind.java (93%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{recorddefinition => record/definition}/VectorStoreRecordDataField.java (53%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{recorddefinition => record/definition}/VectorStoreRecordDefinition.java (92%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{recorddefinition => record/definition}/VectorStoreRecordField.java (97%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{recorddefinition => record/definition}/VectorStoreRecordKeyField.java (94%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{recorddefinition => record/definition}/VectorStoreRecordVectorField.java (98%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{recordoptions => record/options}/DeleteRecordOptions.java (68%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{recordoptions => record/options}/GetRecordOptions.java (95%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{recordoptions => record/options}/UpsertRecordOptions.java (69%) create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/BasicVectorSearchFilter.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/EqualityFilterClause.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/FilterClause.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/FilterClauseType.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/TagListContainsFilterClause.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/VectorSearchOptions.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQueryType.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizableTextSearchQuery.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java index 24217240..9e5a5699 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java @@ -2,9 +2,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordVectorAttribute; import java.util.List; @@ -16,7 +16,7 @@ public class Hotel { @VectorStoreRecordDataAttribute private final int code; @JsonProperty("summary") - @VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "descriptionEmbedding") + @VectorStoreRecordDataAttribute() private final String description; @JsonProperty("summaryEmbedding") @VectorStoreRecordVectorAttribute(dimensions = 3) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java index 8bee5a76..3f61b1d8 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -5,7 +5,7 @@ import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; +import com.microsoft.semantickernel.data.record.options.GetRecordOptions; import com.microsoft.semantickernel.tests.connectors.memory.Hotel; import com.mysql.cj.jdbc.MysqlDataSource; import org.junit.jupiter.params.ParameterizedTest; diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index 300f45b4..deaf9957 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -2,12 +2,12 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.record.options.GetRecordOptions; import com.microsoft.semantickernel.tests.connectors.memory.Hotel; import com.redis.testcontainers.RedisContainer; import org.junit.jupiter.api.BeforeAll; diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index 74e9ba23..9942c4ac 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -2,12 +2,12 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.record.options.GetRecordOptions; import com.microsoft.semantickernel.tests.connectors.memory.Hotel; import com.redis.testcontainers.RedisContainer; import org.junit.jupiter.api.BeforeAll; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/RunAll.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/RunAll.java index a5b19fe8..1a86b95d 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/RunAll.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/RunAll.java @@ -17,7 +17,7 @@ import com.microsoft.semantickernel.samples.syntaxexamples.functions.Example59_OpenAIFunctionCalling; import com.microsoft.semantickernel.samples.syntaxexamples.functions.Example60_AdvancedMethodFunctions; import com.microsoft.semantickernel.samples.syntaxexamples.java.KernelFunctionYaml_Example; -import com.microsoft.semantickernel.samples.syntaxexamples.memory.AzureAISearch_DataStorage; +import com.microsoft.semantickernel.samples.syntaxexamples.memory.AzureAISearchVectorStore; import com.microsoft.semantickernel.samples.syntaxexamples.plugins.Example10_DescribeAllPluginsAndFunctions; import com.microsoft.semantickernel.samples.syntaxexamples.plugins.Example13_ConversationSummaryPlugin; import com.microsoft.semantickernel.samples.syntaxexamples.template.Example06_TemplateLanguage; @@ -38,7 +38,7 @@ public class RunAll { public static void main(String[] args) { List mains = Arrays.asList( - AzureAISearch_DataStorage::main, + AzureAISearchVectorStore::main, Example01_NativeFunctions::main, Example03_Arguments::main, Example05_InlineFunctionDefinition::main, diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java similarity index 81% rename from samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java rename to samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java index 946cadc6..1b5da48e 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java @@ -10,16 +10,15 @@ import com.azure.core.util.TracingOptions; import com.azure.search.documents.indexes.SearchIndexAsyncClient; import com.azure.search.documents.indexes.SearchIndexClientBuilder; -import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; -import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStore; import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreOptions; +import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.VectorSearchResult; +import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordVectorAttribute; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Base64; @@ -27,10 +26,11 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; + import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -public class AzureAISearch_DataStorage { +public class AzureAISearchVectorStore { private static final String CLIENT_KEY = System.getenv("CLIENT_KEY"); private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY"); @@ -51,7 +51,7 @@ static class GitHubFile { @JsonProperty("fileId") // Set a different name for the storage field if needed @VectorStoreRecordKeyAttribute() private final String id; - @VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "embedding") + @VectorStoreRecordDataAttribute() private final String description; @VectorStoreRecordDataAttribute private final String link; @@ -118,13 +118,13 @@ public static void dataStorageWithAzureAISearch( OpenAITextEmbeddingGenerationService embeddingGeneration) { // Create a new Azure AI Search vector store - var azureAISearchVectorStore = AzureAISearchVectorStore.builder() - .withClient(searchClient) + var azureAISearchVectorStore = com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStore.builder() + .withSearchIndexAsyncClient(searchClient) .withOptions(new AzureAISearchVectorStoreOptions()) .build(); String collectionName = "skgithubfiles"; - var collection = azureAISearchVectorStore.getCollection( + var collection = (AzureAISearchVectorStoreRecordCollection) azureAISearchVectorStore.getCollection( collectionName, AzureAISearchVectorStoreRecordCollectionOptions.builder() .withRecordClass(GitHubFile.class) @@ -136,18 +136,26 @@ public static void dataStorageWithAzureAISearch( .then(storeData(collection, embeddingGeneration, sampleData())) .block(); - // Query the Azure AI Search client for results - // This might take a few seconds to return the best result - var result = searchClient.getSearchAsyncClient(collectionName) - .search("How to get started with the Semantic Kernel?") - .blockFirst(); + // Search for results + // Might need to wait for the data to be indexed + var results = search("How to get started", collection, embeddingGeneration).block(); + var searchResult = results.get(0); + System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", + searchResult.getScore(), searchResult.getRecord().link, searchResult.getRecord().description); + } + + + private static Mono>> search( + String searchText, + AzureAISearchVectorStoreRecordCollection recordCollection, + OpenAITextEmbeddingGenerationService embeddingGeneration) { - GitHubFile gitHubFile = result.getDocument(GitHubFile.class); - System.out.println("Best result: " + gitHubFile.description + ". Link: " + gitHubFile.link); + return embeddingGeneration.generateEmbeddingsAsync(Collections.singletonList(searchText)) + .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); } private static Mono> storeData( - VectorStoreRecordCollection recordStore, + AzureAISearchVectorStoreRecordCollection recordCollection, OpenAITextEmbeddingGenerationService embeddingGeneration, Map data) { @@ -163,7 +171,7 @@ private static Mono> storeData( entry.getValue(), entry.getKey(), embeddings.get(0).getVector()); - return recordStore.upsertAsync(gitHubFile, null); + return recordCollection.upsertAsync(gitHubFile, null); }); }) .collectList(); diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java index 53904d57..415503ca 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java @@ -9,9 +9,9 @@ import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.VolatileVectorStore; import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordVectorAttribute; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -37,7 +37,7 @@ static class GitHubFile { @VectorStoreRecordKeyAttribute() private final String id; - @VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "embedding") + @VectorStoreRecordDataAttribute() private final String description; @VectorStoreRecordDataAttribute private final String link; @@ -64,7 +64,7 @@ public String getDescription() { } static String encodeId(String realId) { - return AzureAISearch_DataStorage.GitHubFile.encodeId(realId); + return AzureAISearchVectorStore.GitHubFile.encodeId(realId); } } diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java index 90e38e26..5e2ffdc1 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java @@ -11,9 +11,9 @@ import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordVectorAttribute; import com.mysql.cj.jdbc.MysqlDataSource; import java.nio.charset.StandardCharsets; import java.sql.SQLException; @@ -45,7 +45,7 @@ static class GitHubFile { @VectorStoreRecordKeyAttribute() private final String id; - @VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "embedding") + @VectorStoreRecordDataAttribute() private final String description; @VectorStoreRecordDataAttribute private final String link; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java index bb236788..cf6f30ab 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java @@ -13,9 +13,9 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStore; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreOptions; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordVectorAttribute; import java.util.Arrays; import java.util.Collections; @@ -47,7 +47,7 @@ public static class GitHubFile { @VectorStoreRecordKeyAttribute() private final String id; - @VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "embedding") + @VectorStoreRecordDataAttribute() private final String description; @VectorStoreRecordDataAttribute private final String link; @@ -78,7 +78,7 @@ public String getDescription() { } static String encodeId(String realId) { - return AzureAISearch_DataStorage.GitHubFile.encodeId(realId); + return AzureAISearchVectorStore.GitHubFile.encodeId(realId); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java index 5c1ec120..e2f084bf 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java @@ -6,7 +6,6 @@ import com.microsoft.semantickernel.data.VectorStore; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.List; @@ -16,19 +15,19 @@ public class AzureAISearchVectorStore implements VectorStore { - private final SearchIndexAsyncClient client; + private final SearchIndexAsyncClient searchIndexAsyncClient; private final AzureAISearchVectorStoreOptions options; /** * Creates a new instance of {@link AzureAISearchVectorStore}. * - * @param client The Azure AI Search client. + * @param searchIndexAsyncClient The Azure AI Search client. * @param options The options for the vector store. */ @SuppressFBWarnings("EI_EXPOSE_REP2") - public AzureAISearchVectorStore(@Nonnull SearchIndexAsyncClient client, + public AzureAISearchVectorStore(@Nonnull SearchIndexAsyncClient searchIndexAsyncClient, @Nonnull AzureAISearchVectorStoreOptions options) { - this.client = client; + this.searchIndexAsyncClient = searchIndexAsyncClient; this.options = options; } @@ -54,14 +53,14 @@ public final VectorStoreRecordCollection getCollectio return (VectorStoreRecordCollection) this.options .getVectorStoreRecordCollectionFactory() .createVectorStoreRecordCollection( - client, + searchIndexAsyncClient, collectionName, options.getRecordClass(), options.getRecordDefinition()); } return (VectorStoreRecordCollection) new AzureAISearchVectorStoreRecordCollection<>( - client, + searchIndexAsyncClient, collectionName, (AzureAISearchVectorStoreRecordCollectionOptions) options); } @@ -73,7 +72,7 @@ public final VectorStoreRecordCollection getCollectio */ @Override public Mono> getCollectionNamesAsync() { - return client.listIndexes().map(SearchIndex::getName).collectList(); + return searchIndexAsyncClient.listIndexes().map(SearchIndex::getName).collectList(); } /** @@ -91,19 +90,20 @@ public static Builder builder() { public static class Builder { @Nullable - private SearchIndexAsyncClient client; + private SearchIndexAsyncClient searchIndexAsyncClient; @Nullable private AzureAISearchVectorStoreOptions options; /** - * Sets the Azure AI Search client. + * Sets the Azure AI Search searchIndexClient. * - * @param client The Azure AI Search client. + * @param searchIndexAsyncClient The Azure AI Search searchIndexClient. * @return The updated builder instance. */ @SuppressFBWarnings("EI_EXPOSE_REP2") - public Builder withClient(@Nonnull SearchIndexAsyncClient client) { - this.client = client; + public Builder withSearchIndexAsyncClient( + @Nonnull SearchIndexAsyncClient searchIndexAsyncClient) { + this.searchIndexAsyncClient = searchIndexAsyncClient; return this; } @@ -125,14 +125,14 @@ public Builder withOptions( * @return The Azure AI Search vector store. */ public AzureAISearchVectorStore build() { - if (client == null) { - throw new IllegalStateException("client is required"); + if (searchIndexAsyncClient == null) { + throw new IllegalStateException("searchIndexAsyncClient is required"); } if (options == null) { throw new IllegalStateException("options is required"); } - return new AzureAISearchVectorStore(client, options); + return new AzureAISearchVectorStore(searchIndexAsyncClient, options); } } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java index a833ef69..a989179f 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java @@ -10,9 +10,10 @@ import com.azure.search.documents.indexes.models.VectorSearchAlgorithmConfiguration; import com.azure.search.documents.indexes.models.VectorSearchAlgorithmMetric; import com.azure.search.documents.indexes.models.VectorSearchProfile; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; +import com.azure.search.documents.indexes.models.VectorSearchVectorizerKind; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.exceptions.SKException; import java.time.OffsetDateTime; @@ -83,7 +84,8 @@ public static SearchField mapDataField(VectorStoreRecordDataField dataField) { return new SearchField(dataField.getEffectiveStorageName(), getSearchFieldDataType(dataField.getFieldType())) - .setFilterable(dataField.isFilterable()); + .setFilterable(dataField.isFilterable()) + .setSearchable(dataField.isFullTextSearchable()); } public static SearchField mapVectorField(VectorStoreRecordVectorField vectorField) { @@ -103,8 +105,8 @@ public static void updateVectorSearchParameters( } algorithms.add(getAlgorithmConfig(vectorField)); - profiles.add(new VectorSearchProfile(getVectorSearchProfileName(vectorField), - getAlgorithmConfigName(vectorField))); + profiles.add(new VectorSearchProfile( + getVectorSearchProfileName(vectorField), getAlgorithmConfigName(vectorField))); } public static SearchFieldDataType getSearchFieldDataType(Class fieldType) { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java new file mode 100644 index 00000000..06856e40 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.azureaisearch; + +import com.microsoft.semantickernel.data.vectorsearch.options.BasicVectorSearchFilter; +import com.microsoft.semantickernel.data.vectorsearch.options.EqualityFilterClause; +import com.microsoft.semantickernel.data.vectorsearch.options.TagListContainsFilterClause; +import com.microsoft.semantickernel.exceptions.SKException; + +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class AzureAISearchVectorStoreCollectionSearchMapping { + public static String buildFilterString(BasicVectorSearchFilter basicVectorSearchFilter, + Map storageNames) { + if (basicVectorSearchFilter == null) { + return ""; + } + + String filter = ""; + + if (basicVectorSearchFilter.getFilterClauses() != null) { + + List filterStrings = basicVectorSearchFilter.getFilterClauses().stream() + .filter(filterClause -> filterClause instanceof EqualityFilterClause) + .map(filterClause -> { + String fieldName = ((EqualityFilterClause) filterClause).getFieldName(); + String storageFieldName = storageNames.get(fieldName); + Object value = ((EqualityFilterClause) filterClause).getValue(); + + if (value instanceof String) { + return String.format("%s eq '%s'", storageFieldName, value); + } else if (value instanceof Boolean) { + return String.format("%s eq %s", storageFieldName, + value.toString().toLowerCase()); + } else if (value instanceof Integer) { + return String.format("%s eq %d", storageFieldName, (Integer) value); + } else if (value instanceof Long) { + return String.format("%s eq %d", storageFieldName, (Long) value); + } else if (value instanceof Float) { + return String.format("%s eq %f", storageFieldName, (Float) value); + } else if (value instanceof Double) { + return String.format("%s eq %f", storageFieldName, (Double) value); + } else if (value instanceof OffsetDateTime) { + return String.format("%s eq %s", storageFieldName, ((OffsetDateTime) value) + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + } else if (value == null) { + return String.format("%s eq null", storageFieldName); + } else { + throw new SKException("Unsupported filter value type '" + + value.getClass().getSimpleName() + "'."); + } + }).collect(Collectors.toList()); + + List tagListContainsStrings = basicVectorSearchFilter.getFilterClauses() + .stream() + .filter(filterClause -> filterClause instanceof TagListContainsFilterClause) + .map(filterClause -> { + String fieldName = ((TagListContainsFilterClause) filterClause).getFieldName(); + String storageFieldName = storageNames.get(fieldName); + + TagListContainsFilterClause tagListClause = (TagListContainsFilterClause) filterClause; + return String.format("%s/any(t: t eq '%s')", storageFieldName, + tagListClause.getValue()); + }).collect(Collectors.toList()); + + filterStrings.addAll(tagListContainsStrings); + filter = String.join(" and ", filterStrings); + } + + return filter; + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index 84a91f2b..5d5fb5a0 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -6,21 +6,30 @@ import com.azure.search.documents.indexes.SearchIndexAsyncClient; import com.azure.search.documents.indexes.models.SearchField; import com.azure.search.documents.indexes.models.SearchIndex; -import com.azure.search.documents.indexes.models.VectorSearch; import com.azure.search.documents.indexes.models.VectorSearchAlgorithmConfiguration; import com.azure.search.documents.indexes.models.VectorSearchProfile; import com.azure.search.documents.models.IndexDocumentsResult; import com.azure.search.documents.models.IndexingResult; +import com.azure.search.documents.models.SearchOptions; +import com.azure.search.documents.models.VectorQuery; +import com.azure.search.documents.models.VectorizableTextQuery; +import com.azure.search.documents.models.VectorizedQuery; +import com.microsoft.semantickernel.data.VectorSearch; +import com.microsoft.semantickernel.data.VectorSearchResult; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; -import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.record.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.record.options.GetRecordOptions; +import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorsearch.options.VectorSearchOptions; +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizableTextSearchQuery; +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizedSearchQuery; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.time.OffsetDateTime; @@ -32,14 +41,13 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import javax.annotation.Nonnull; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; public class AzureAISearchVectorStoreRecordCollection implements - VectorStoreRecordCollection { + VectorStoreRecordCollection, VectorSearch { private static final HashSet> supportedKeyTypes = new HashSet<>( Collections.singletonList( @@ -65,22 +73,25 @@ public class AzureAISearchVectorStoreRecordCollection implements List.class, Collection.class)); - private final SearchIndexAsyncClient client; + private final SearchIndexAsyncClient searchIndexAsyncClient; + private final SearchAsyncClient searchAsyncClient; private final String collectionName; - private final Map clientsByIndex = new ConcurrentHashMap<>(); private final AzureAISearchVectorStoreRecordCollectionOptions options; private final VectorStoreRecordDefinition recordDefinition; // List of non-vector fields. Used to fetch only non-vector fields when vectors are not requested private final List nonVectorFields = new ArrayList<>(); + private final Map storageNames; + private final String firstVectorFieldName; @SuppressFBWarnings("EI_EXPOSE_REP2") public AzureAISearchVectorStoreRecordCollection( - @Nonnull SearchIndexAsyncClient client, + @Nonnull SearchIndexAsyncClient searchIndexAsyncClient, @Nonnull String collectionName, @Nonnull AzureAISearchVectorStoreRecordCollectionOptions options) { - this.client = client; + this.searchIndexAsyncClient = searchIndexAsyncClient; this.collectionName = collectionName; + this.searchAsyncClient = searchIndexAsyncClient.getSearchAsyncClient(collectionName); this.options = options; // If record definition is not provided, create one from the record class @@ -104,6 +115,10 @@ public AzureAISearchVectorStoreRecordCollection( nonVectorFields.addAll(this.recordDefinition.getDataFields().stream() .map(VectorStoreRecordDataField::getEffectiveStorageName) .collect(Collectors.toList())); + + storageNames = recordDefinition.getStorageNames(); + firstVectorFieldName = recordDefinition.getVectorFields().isEmpty() ? null + : recordDefinition.getVectorFields().get(0).getName(); } @Override @@ -112,7 +127,8 @@ public String getCollectionName() { } private Mono> getIndexesAsync() { - return client.listIndexes().map(SearchIndex::getName).collect(Collectors.toList()); + return searchIndexAsyncClient.listIndexes().map(SearchIndex::getName) + .collect(Collectors.toList()); } @Override @@ -145,11 +161,11 @@ public Mono> createCollectionAsync() SearchIndex newIndex = new SearchIndex(collectionName) .setFields(searchFields) - .setVectorSearch(new VectorSearch() + .setVectorSearch(new com.azure.search.documents.indexes.models.VectorSearch() .setAlgorithms(algorithms) .setProfiles(profiles)); - return client.createIndex(newIndex).then(Mono.just(this)); + return searchIndexAsyncClient.createIndex(newIndex).then(Mono.just(this)); } @Override @@ -166,14 +182,12 @@ public Mono> createCollectionIfNotEx @Override public Mono deleteCollectionAsync() { - return client.deleteIndex(this.collectionName).then(); + return searchIndexAsyncClient.deleteIndex(this.collectionName).then(); } @Override public Mono getAsync( @Nonnull String key, GetRecordOptions options) { - SearchAsyncClient client = this.getSearchClient(this.collectionName); - // If vectors are not requested, only fetch non-vector fields List selectedFields = null; if (options != null && !options.includeVectors()) { @@ -185,11 +199,12 @@ public Mono getAsync( // Use custom mapper if available if (mapper != null && mapper.getStorageModelToRecordMapper() != null) { - return client.getDocument(key, SearchDocument.class) + return searchAsyncClient.getDocument(key, SearchDocument.class) .map(this.options.getVectorStoreRecordMapper()::mapStorageModeltoRecord); } - return client.getDocumentWithResponse(key, this.options.getRecordClass(), selectedFields) + return searchAsyncClient + .getDocumentWithResponse(key, this.options.getRecordClass(), selectedFields) .flatMap(response -> { if (response.getStatusCode() == 404) { return Mono.error(new SKException("Record not found: " + key)); @@ -221,7 +236,6 @@ public Mono> upsertBatchAsync( if (records.isEmpty()) { return Mono.just(Collections.emptyList()); } - SearchAsyncClient client = this.getSearchClient(this.collectionName); VectorStoreRecordMapper mapper = this.options .getVectorStoreRecordMapper(); @@ -236,7 +250,7 @@ public Mono> upsertBatchAsync( documents = records; } - return client.uploadDocuments(documents) + return searchAsyncClient.uploadDocuments(documents) .map(IndexDocumentsResult::getResults) .map( results -> results.stream() @@ -251,24 +265,84 @@ public Mono deleteAsync(String key, DeleteRecordOptions options) { @Override public Mono deleteBatchAsync(List keys, DeleteRecordOptions options) { - SearchAsyncClient client = this.getSearchClient(this.collectionName); - - return client.deleteDocuments(keys.stream().map(key -> { + return searchAsyncClient.deleteDocuments(keys.stream().map(key -> { SearchDocument document = new SearchDocument(); document.put(this.recordDefinition.getKeyField().getEffectiveStorageName(), key); return document; }).collect(Collectors.toList())).then(); } + private Mono>> searchAndMapAsync(String searchText, + SearchOptions searchOptions) { + VectorStoreRecordMapper mapper = this.options + .getVectorStoreRecordMapper(); + + return this.searchAsyncClient.search(searchText, searchOptions) + .flatMap(response -> { + Record record; + + // Use custom mapper if available + if (mapper != null && mapper.getStorageModelToRecordMapper() != null) { + record = mapper + .mapStorageModeltoRecord(response.getDocument(SearchDocument.class)); + } else { + record = response.getDocument(this.options.getRecordClass()); + } + + return Mono.just(new VectorSearchResult<>(record, response.getScore())); + }).collectList(); + } + /** - * Get a search client for the index specified. Note: the index might not exist, but we avoid - * checking everytime and the extra latency. + * Vectorized search. This method searches for records that are similar to the given vector. * - * @param indexName Index name - * @return Search client ready to read/write + * @param vector The vector to search with. + * @param options The options to use for the search. + * @return A list of search results. */ - protected SearchAsyncClient getSearchClient(@Nonnull String indexName) { - return clientsByIndex.computeIfAbsent( - indexName, client::getSearchAsyncClient); + @Override + public Mono>> searchAsync(Vector vector, + VectorSearchOptions options) { + if (firstVectorFieldName == null) { + throw new SKException("No vector fields defined. Cannot perform vector search"); + } + + if (options == null) { + options = VectorSearchOptions.createDefault(firstVectorFieldName); + } + + List vectorQueries = new ArrayList<>(); + vectorQueries.add(new VectorizedQuery((List) vector) + .setFields(options.getVectorFieldName() != null ? options.getVectorFieldName() + : firstVectorFieldName) + .setKNearestNeighborsCount(options.getLimit())); + + String filter = AzureAISearchVectorStoreCollectionSearchMapping + .buildFilterString(options.getBasicVectorSearchFilter(), storageNames); + SearchOptions searchOptions = new SearchOptions() + .setFilter(filter) + .setTop(options.getLimit()) + .setSkip(options.getOffset()) + .setVectorSearchOptions(new com.azure.search.documents.models.VectorSearchOptions() + .setQueries(vectorQueries)); + + if (!options.isIncludeVectors()) { + searchOptions.setSelect(nonVectorFields.toArray(new String[0])); + } + + return searchAndMapAsync(null, searchOptions); + } + + /** + * Vectorizable text search. This method searches for records that are similar to the given text. + * + * @param searchText The text to search with. + * @param options The options to use for the search. + * @return A list of search results. + */ + @Override + public Mono>> searchAsync(String searchText, + VectorSearchOptions options) { + throw new UnsupportedOperationException("Not implemented"); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java index 41cc1633..b5bfba51 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java @@ -2,7 +2,7 @@ package com.microsoft.semantickernel.connectors.data.azureaisearch; import com.azure.search.documents.indexes.SearchIndexAsyncClient; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; /** * Factory for creating Azure AI Search vector store record collections. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java index 0437bcfd..d9cc43a8 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java @@ -4,7 +4,7 @@ import com.azure.search.documents.SearchDocument; import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; import javax.annotation.Nonnull; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java index 3b0eb189..01baf6c7 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java @@ -3,7 +3,6 @@ import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import reactor.core.publisher.Mono; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java index fef57d65..5c640ae2 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java @@ -2,12 +2,12 @@ package com.microsoft.semantickernel.connectors.data.jdbc; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.record.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.record.options.GetRecordOptions; +import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.annotation.Nonnull; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index cdda7d9a..62a8ad5d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -3,10 +3,10 @@ import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.record.options.GetRecordOptions; +import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; import java.sql.ResultSet; import java.util.List; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java index 91582106..90d041e1 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -7,10 +7,10 @@ import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreRecordMapper; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.record.options.GetRecordOptions; +import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.lang.reflect.Field; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java index 32deb0dc..d4d92111 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; import javax.sql.DataSource; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java index 093e3186..366e28ad 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java @@ -3,7 +3,7 @@ import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java index c9398948..f0974f2b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java @@ -7,9 +7,9 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java index c6440bad..bbd40e96 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java @@ -6,10 +6,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreDefaultQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; -import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index d061254d..0a37bec7 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -6,11 +6,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreDefaultQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; -import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java index 1a074de2..92434847 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java @@ -7,9 +7,9 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.postgresql.util.PGobject; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java index 1b8079e9..f7e6432b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java @@ -4,11 +4,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.record.options.GetRecordOptions; +import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java index 74d42544..1e2056fb 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java @@ -3,7 +3,7 @@ import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; import javax.annotation.Nonnull; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java index dba568b7..cfa7a80f 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java @@ -2,15 +2,14 @@ package com.microsoft.semantickernel.connectors.data.redis; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.exceptions.SKException; import javax.annotation.Nullable; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java index 9beef7ff..f37d9e14 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java @@ -6,11 +6,11 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.record.options.GetRecordOptions; +import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java index 5690e424..087c79ad 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java @@ -3,7 +3,7 @@ import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; import javax.annotation.Nonnull; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java index cd7cd27e..c7953da3 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java @@ -5,7 +5,6 @@ import com.microsoft.semantickernel.data.VectorStore; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.ArrayList; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java index ec9bd62a..d64afb94 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java @@ -1,10 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordVectorField; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java index 3834f163..4025b37b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java @@ -2,7 +2,7 @@ package com.microsoft.semantickernel.connectors.data.redis; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; import redis.clients.jedis.JedisPooled; /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearch.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearch.java new file mode 100644 index 00000000..a2b54dec --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearch.java @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data; + +import com.microsoft.semantickernel.data.vectorsearch.options.VectorSearchOptions; +import reactor.core.publisher.Mono; + +import java.util.List; + +public interface VectorSearch { + /** + * Vectorized search. This method searches for records that are similar to the given vector. + * + * @param vector The vector to search with. + * @param options The options to use for the search. + * @return A list of search results. + */ + Mono>> searchAsync(Vector vector, + VectorSearchOptions options); + + /** + * Vectorizable text search. This method searches for records that are similar to the given text. + * + * @param searchText The text to search with. + * @param options The options to use for the search. + * @return A list of search results. + */ + Mono>> searchAsync(String searchText, + VectorSearchOptions options); +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearchResult.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearchResult.java new file mode 100644 index 00000000..225fbf93 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearchResult.java @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data; + +public class VectorSearchResult { + private final Record record; + private final double score; + + /** + * Creates a new instance of VectorSearchResult. + * + * @param record The record. + * @param score The score. + */ + public VectorSearchResult(Record record, double score) { + this.record = record; + this.score = score; + } + + /** + * Gets the record. + * + * @return The record. + */ + public Record getRecord() { + return record; + } + + /** + * Gets the score. + * + * @return The score. + */ + public double getScore() { + return score; + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStore.java index 664bb281..8928e2af 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStore.java @@ -1,10 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import java.util.List; import javax.annotation.Nonnull; -import javax.annotation.Nullable; + import reactor.core.publisher.Mono; /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollection.java index 0b7319c0..04dde988 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollection.java @@ -1,9 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data; -import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.data.record.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.record.options.GetRecordOptions; +import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; import java.util.List; import reactor.core.publisher.Mono; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollectionOptions.java index 6b128a13..93635be2 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollectionOptions.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; public interface VectorStoreRecordCollectionOptions { /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java index 3b5c3efd..5b41477d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java @@ -1,13 +1,11 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import javax.annotation.Nonnull; -import javax.annotation.Nullable; import com.microsoft.semantickernel.exceptions.SKException; import reactor.core.publisher.Mono; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java index 0d7c3ddd..8a9c3f02 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java @@ -3,11 +3,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; -import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.record.options.GetRecordOptions; +import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import java.util.Collections; import java.util.HashSet; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java index f3e50a68..a9990c60 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; import javax.annotation.Nonnull; import javax.annotation.Nullable; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordDataAttribute.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/attributes/VectorStoreRecordDataAttribute.java similarity index 74% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordDataAttribute.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/attributes/VectorStoreRecordDataAttribute.java index 10efe24f..33ea5fcc 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordDataAttribute.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/attributes/VectorStoreRecordDataAttribute.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recordattributes; +package com.microsoft.semantickernel.data.record.attributes; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; @@ -20,17 +20,12 @@ String storageName() default ""; /** - * Whether the field has a vector representation. - */ - boolean hasEmbedding() default false; - - /** - * Name of the field that contains the vector representation. + * Whether the field is filterable. */ - String embeddingFieldName() default ""; + boolean isFilterable() default false; /** - * Whether the field is filterable. + * Whether the field is full text searchable. */ - boolean isFilterable() default false; + boolean isFullTextSearchable() default false; } \ No newline at end of file diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordKeyAttribute.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/attributes/VectorStoreRecordKeyAttribute.java similarity index 88% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordKeyAttribute.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/attributes/VectorStoreRecordKeyAttribute.java index ffe2451a..3791abbd 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordKeyAttribute.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/attributes/VectorStoreRecordKeyAttribute.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recordattributes; +package com.microsoft.semantickernel.data.record.attributes; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordVectorAttribute.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/attributes/VectorStoreRecordVectorAttribute.java similarity index 92% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordVectorAttribute.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/attributes/VectorStoreRecordVectorAttribute.java index 3f404652..ab1c5bd4 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordattributes/VectorStoreRecordVectorAttribute.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/attributes/VectorStoreRecordVectorAttribute.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recordattributes; +package com.microsoft.semantickernel.data.record.attributes; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/DistanceFunction.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/DistanceFunction.java similarity index 94% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/DistanceFunction.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/DistanceFunction.java index b41aa32e..13ed8ea8 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/DistanceFunction.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/DistanceFunction.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recorddefinition; +package com.microsoft.semantickernel.data.record.definition; public enum DistanceFunction { COSINE_SIMILARITY("cosineSimilarity"), DOT_PRODUCT("dotProduct"), EUCLIDEAN("euclidean"); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/IndexKind.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/IndexKind.java similarity index 93% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/IndexKind.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/IndexKind.java index 62832de2..33908f6e 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/IndexKind.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/IndexKind.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recorddefinition; +package com.microsoft.semantickernel.data.record.definition; public enum IndexKind { HNSW("Hnsw"), FLAT("Flat"); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDataField.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordDataField.java similarity index 53% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDataField.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordDataField.java index 83ae6c39..7aa21ba0 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDataField.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordDataField.java @@ -1,14 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recorddefinition; +package com.microsoft.semantickernel.data.record.definition; import javax.annotation.Nonnull; import javax.annotation.Nullable; public class VectorStoreRecordDataField extends VectorStoreRecordField { - private final boolean hasEmbedding; - @Nullable - private final String embeddingFieldName; private final boolean isFilterable; + private final boolean isFullTextSearchable; public static Builder builder() { return new Builder(); @@ -20,88 +18,61 @@ public static Builder builder() { * @param name the name of the field * @param storageName the storage name of the field * @param fieldType the field type - * @param hasEmbedding a value indicating whether the field has an embedding - * @param embeddingFieldName the name of the embedding * @param isFilterable a value indicating whether the field is filterable */ public VectorStoreRecordDataField( @Nonnull String name, @Nullable String storageName, @Nonnull Class fieldType, - boolean hasEmbedding, - @Nullable String embeddingFieldName, - boolean isFilterable) { + boolean isFilterable, + boolean isFullTextSearchable) { super(name, storageName, fieldType); - this.hasEmbedding = hasEmbedding; - this.embeddingFieldName = embeddingFieldName; this.isFilterable = isFilterable; + this.isFullTextSearchable = isFullTextSearchable; } /** - * Gets a value indicating whether the field has an embedding. - * - * @return a value indicating whether the field has an embedding - */ - public boolean hasEmbedding() { - return hasEmbedding; - } - - /** - * Gets the name of the embedding. + * Gets a value indicating whether the field is filterable. * - * @return the name of the embedding + * @return a value indicating whether the field is filterable */ - @Nullable - public String getEmbeddingFieldName() { - return embeddingFieldName; + public boolean isFilterable() { + return isFilterable; } /** - * Gets a value indicating whether the field is filterable. + * Gets a value indicating whether the field is full text searchable. * - * @return a value indicating whether the field is filterable + * @return a value indicating whether the field is full text searchable */ - public boolean isFilterable() { - return isFilterable; + public boolean isFullTextSearchable() { + return isFullTextSearchable; } public static class Builder extends VectorStoreRecordField.Builder { - private boolean hasEmbedding; - @Nullable - private String embeddingFieldName; private boolean isFilterable; + private boolean isFullTextSearchable; /** - * Sets a value indicating whether the field has an embedding. - * - * @param hasEmbedding a value indicating whether the field has an embedding - * @return the builder - */ - public Builder withHasEmbedding(boolean hasEmbedding) { - this.hasEmbedding = hasEmbedding; - return this; - } - - /** - * Sets the name of the embedding field. + * Sets a value indicating whether the field is filterable. * - * @param embeddingFieldName the name of the embedding + * @param isFilterable a value indicating whether the field is filterable * @return the builder */ - public Builder withEmbeddingFieldName(String embeddingFieldName) { - this.embeddingFieldName = embeddingFieldName; + public Builder isFilterable(boolean isFilterable) { + this.isFilterable = isFilterable; return this; } /** - * Sets a value indicating whether the field is filterable. + * Sets a value indicating whether the field is full text searchable. * - * @param isFilterable a value indicating whether the field is filterable + * @param isFullTextSearchable a value indicating whether the field is full text searchable * @return the builder */ - public Builder withIsFilterable(boolean isFilterable) { - this.isFilterable = isFilterable; + public Builder isFullTextSearchable(boolean isFullTextSearchable) { + this.isFullTextSearchable = isFullTextSearchable; return this; } @@ -118,18 +89,13 @@ public VectorStoreRecordDataField build() { if (fieldType == null) { throw new IllegalArgumentException("fieldType is required"); } - if (hasEmbedding && embeddingFieldName == null) { - throw new IllegalArgumentException( - "embeddingFieldName is required when hasEmbedding is true"); - } return new VectorStoreRecordDataField( name, storageName, fieldType, - hasEmbedding, - embeddingFieldName, - isFilterable); + isFilterable, + isFullTextSearchable); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordDefinition.java similarity index 92% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordDefinition.java index f17f5c74..5c16fe47 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordDefinition.java @@ -1,16 +1,17 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recorddefinition; +package com.microsoft.semantickernel.data.record.definition; import com.fasterxml.jackson.annotation.JsonProperty; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordVectorAttribute; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -57,6 +58,12 @@ public List getNonVectorFields() { return fields; } + public Map getStorageNames() { + return getAllFields().stream() + .collect(Collectors.toMap(VectorStoreRecordField::getName, + VectorStoreRecordField::getEffectiveStorageName)); + } + private VectorStoreRecordDefinition( VectorStoreRecordKeyField keyField, List dataFields, @@ -145,9 +152,7 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) .withName(field.getName()) .withStorageName(storageName) .withFieldType(field.getType()) - .withHasEmbedding(dataAttribute.hasEmbedding()) - .withEmbeddingFieldName(dataAttribute.embeddingFieldName()) - .withIsFilterable(dataAttribute.isFilterable()) + .isFilterable(dataAttribute.isFilterable()) .build()); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordField.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordField.java similarity index 97% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordField.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordField.java index 6c8cd65a..f80fffa1 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordField.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordField.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recorddefinition; +package com.microsoft.semantickernel.data.record.definition; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordKeyField.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordKeyField.java similarity index 94% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordKeyField.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordKeyField.java index 8cbcbe00..72236dd7 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordKeyField.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordKeyField.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recorddefinition; +package com.microsoft.semantickernel.data.record.definition; /** * Represents the key field in a record. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordVectorField.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordVectorField.java similarity index 98% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordVectorField.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordVectorField.java index c2d996aa..e79b5a7e 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordVectorField.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordVectorField.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recorddefinition; +package com.microsoft.semantickernel.data.record.definition; import javax.annotation.Nonnull; import javax.annotation.Nullable; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordoptions/DeleteRecordOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/DeleteRecordOptions.java similarity index 68% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordoptions/DeleteRecordOptions.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/DeleteRecordOptions.java index ad68a90f..a1405ec6 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordoptions/DeleteRecordOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/DeleteRecordOptions.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recordoptions; +package com.microsoft.semantickernel.data.record.options; /** * Options for deleting a record. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordoptions/GetRecordOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/GetRecordOptions.java similarity index 95% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordoptions/GetRecordOptions.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/GetRecordOptions.java index 6c81b5c9..c3ed0d8a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordoptions/GetRecordOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/GetRecordOptions.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recordoptions; +package com.microsoft.semantickernel.data.record.options; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordoptions/UpsertRecordOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/UpsertRecordOptions.java similarity index 69% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordoptions/UpsertRecordOptions.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/UpsertRecordOptions.java index 132eaf92..20105fb6 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recordoptions/UpsertRecordOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/UpsertRecordOptions.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.recordoptions; +package com.microsoft.semantickernel.data.record.options; /** * Options for upserting a record. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/BasicVectorSearchFilter.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/BasicVectorSearchFilter.java new file mode 100644 index 00000000..6151fc01 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/BasicVectorSearchFilter.java @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch.options; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class BasicVectorSearchFilter { + + /** + * Creates a new instance of the BasicVectorSearchFilter class. + * + * @return A new instance of the BasicVectorSearchFilter class. + */ + public static BasicVectorSearchFilter createDefault() { + return new BasicVectorSearchFilter(); + } + + private final List filterClauses; + + public BasicVectorSearchFilter() { + this.filterClauses = new ArrayList<>(); + } + + /** + * Adds an equality filter clause to the filter. + * + * @param field The field to filter on. + * @param value The value to filter on. + * @return The BasicVectorSearchFilter object. + */ + public BasicVectorSearchFilter equality(String field, Object value) { + filterClauses.add(new EqualityFilterClause(field, value)); + return this; + } + + /** + * Adds clause to the filter that checks if the tag list contains the specified value. + * + * @param field The field to filter on. + * @param value The value to filter on. + * @return The BasicVectorSearchFilter object. + */ + public BasicVectorSearchFilter tagListContains(String field, String value) { + filterClauses.add(new TagListContainsFilterClause(field, value)); + return this; + } + + /** + * Gets the filter clauses. + * + * @return The filter clauses. + */ + public List getFilterClauses() { + return Collections.unmodifiableList(filterClauses); + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/EqualityFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/EqualityFilterClause.java new file mode 100644 index 00000000..0589ddf4 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/EqualityFilterClause.java @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch.options; + +public class EqualityFilterClause extends FilterClause { + + private final String fieldName; + private final Object value; + + public EqualityFilterClause(String fieldName, Object value) { + super(FilterClauseType.EQUALITY); + this.fieldName = fieldName; + this.value = value; + } + + public String getFieldName() { + return fieldName; + } + + public Object getValue() { + return value; + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/FilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/FilterClause.java new file mode 100644 index 00000000..a7913264 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/FilterClause.java @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch.options; + +public class FilterClause { + + /** + * The type of the filter clause. + */ + private final FilterClauseType type; + + /** + * Creates a new instance of the FilterClause class. + * + * @param type The type of the filter clause. + */ + public FilterClause(FilterClauseType type) { + this.type = type; + } + + /** + * Gets the type of the filter clause. + * + * @return The type of the filter clause. + */ + public FilterClauseType getType() { + return type; + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/FilterClauseType.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/FilterClauseType.java new file mode 100644 index 00000000..92b33500 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/FilterClauseType.java @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch.options; + +public enum FilterClauseType { + /** + * An equality filter clause. + */ + EQUALITY, + /** + * A filter clause that checks if the tag list contains a value. + */ + TAG_LIST_CONTAINS +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/TagListContainsFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/TagListContainsFilterClause.java new file mode 100644 index 00000000..2a3d60e5 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/TagListContainsFilterClause.java @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch.options; + +public class TagListContainsFilterClause extends FilterClause { + private final String fieldName; + private final String value; + + /** + * Creates a new instance of the TagListContainsFilterClause class. + * + * @param fieldName The name of the field to filter on. + * @param value The value to filter on. + */ + public TagListContainsFilterClause(String fieldName, String value) { + super(FilterClauseType.TAG_LIST_CONTAINS); + this.fieldName = fieldName; + this.value = value; + } + + /** + * Gets the name of the field to filter on. + * + * @return The name of the field to filter on. + */ + public String getFieldName() { + return fieldName; + } + + /** + * Gets the value to filter on. + * + * @return The value to filter on. + */ + public String getValue() { + return value; + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/VectorSearchOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/VectorSearchOptions.java new file mode 100644 index 00000000..8cbbe1ec --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/VectorSearchOptions.java @@ -0,0 +1,139 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch.options; + +import com.microsoft.semantickernel.builders.SemanticKernelBuilder; + +import javax.annotation.Nullable; + +public class VectorSearchOptions { + + public static final int DEFAULT_RESULT_LIMIT = 3; + + /** + * Creates a new instance of the VectorSearchOptions class with default values. + * + * @param vectorFieldName The name of the vector field. + * @return A new instance of the VectorSearchOptions class with default values. + */ + public static VectorSearchOptions createDefault(String vectorFieldName) { + return VectorSearchOptions.builder() + .withVectorFieldName(vectorFieldName).build(); + } + + @Nullable + private final BasicVectorSearchFilter basicVectorSearchFilter; + @Nullable + private final String vectorFieldName; + private final int limit; + private final int offset; + private final boolean includeVectors; + + public VectorSearchOptions(BasicVectorSearchFilter basicVectorSearchFilter, + String vectorFieldName, int limit, int offset, boolean includeVectors) { + this.basicVectorSearchFilter = basicVectorSearchFilter; + this.vectorFieldName = vectorFieldName; + this.limit = limit; + this.offset = offset; + this.includeVectors = includeVectors; + } + + /** + * Gets the basic vector search filter. + * + * @return The basic vector search filter. + */ + @Nullable + public BasicVectorSearchFilter getBasicVectorSearchFilter() { + return basicVectorSearchFilter; + } + + /** + * Gets the name of the vector field. + * + * @return The name of the vector field. + */ + @Nullable + public String getVectorFieldName() { + return vectorFieldName; + } + + /** + * Gets the limit of the number of results to return. + * + * @return The limit of the number of results to return. + */ + public int getLimit() { + return limit; + } + + /** + * Gets the offset of the results to return. + * + * @return The offset of the results to return. + */ + public int getOffset() { + return offset; + } + + /** + * Gets a value indicating whether to include vectors in the results. + * + * @return A value indicating whether to include vectors in the results. + */ + public boolean isIncludeVectors() { + return includeVectors; + } + + /** + * Creates a new instance of the Builder class. + * + * @return A new instance of the Builder class. + */ + public static Builder builder() { + return new Builder(); + } + + public static class Builder implements SemanticKernelBuilder { + private BasicVectorSearchFilter basicVectorSearchFilter; + private String vectorFieldName; + private int limit; + private int offset; + private boolean includeVectors; + + public Builder() { + this.limit = DEFAULT_RESULT_LIMIT; + } + + public Builder withBasicVectorSearchFilter( + BasicVectorSearchFilter basicVectorSearchFilter) { + this.basicVectorSearchFilter = basicVectorSearchFilter; + return this; + } + + public Builder withVectorFieldName(String vectorFieldName) { + this.vectorFieldName = vectorFieldName; + return this; + } + + public Builder withLimit(int limit) { + this.limit = limit; + return this; + } + + public Builder withOffset(int offset) { + this.offset = offset; + return this; + } + + public Builder withIncludeVectors(boolean includeVectors) { + this.includeVectors = includeVectors; + return this; + } + + @Override + public VectorSearchOptions build() { + return new VectorSearchOptions(basicVectorSearchFilter, vectorFieldName, limit, offset, + includeVectors); + } + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java new file mode 100644 index 00000000..ad5b210d --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch.queries; + +import com.microsoft.semantickernel.data.vectorsearch.options.VectorSearchOptions; + +public class VectorSearchQuery { + + private final VectorSearchQueryType queryType; + private final Object searchOptions; + + public VectorSearchQuery(VectorSearchQueryType queryType, Object searchOptions) { + this.queryType = queryType; + this.searchOptions = searchOptions; + } + + public VectorSearchQueryType getQueryType() { + return queryType; + } + + public Object getSearchOptions() { + return searchOptions; + } + + public static VectorizedSearchQuery createQuery(Vector vector, + VectorSearchOptions options) { + return new VectorizedSearchQuery<>(vector, options); + } + + public static VectorizableTextSearchQuery createQuery(String searchText, + VectorSearchOptions options) { + return new VectorizableTextSearchQuery(searchText, options); + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQueryType.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQueryType.java new file mode 100644 index 00000000..3a64ad62 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQueryType.java @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch.queries; + +public enum VectorSearchQueryType { + + VECTORIZED_SEARCH_QUERY("VectorizedSearchQuery"), VECTORIZABLE_TEXT_SEARCH_QUERY( + "VectorizableTextSearchQuery"), HYBRID_TEXT_VECTORIZED_SEARCH_QUERY( + "HybridTextVectorizedSearchQuery"), HYBRID_VECTORIZABLE_TEXT_SEARCH_QUERY( + "HybridVectorizableTextSearchQuery"); + + private final String value; + + VectorSearchQueryType(String value) { + this.value = value; + } + + public String getValue() { + return value; + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizableTextSearchQuery.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizableTextSearchQuery.java new file mode 100644 index 00000000..78f4b298 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizableTextSearchQuery.java @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch.queries; + +import com.microsoft.semantickernel.data.vectorsearch.options.VectorSearchOptions; + +import javax.annotation.Nullable; + +public class VectorizableTextSearchQuery extends VectorSearchQuery { + + private final String queryText; + + @Nullable + private final VectorSearchOptions searchOptions; + + public VectorizableTextSearchQuery(String queryText, + @Nullable VectorSearchOptions searchOptions) { + super(VectorSearchQueryType.VECTORIZABLE_TEXT_SEARCH_QUERY, searchOptions); + this.queryText = queryText; + this.searchOptions = searchOptions; + } + + public String getQueryText() { + return queryText; + } + + @Nullable + public VectorSearchOptions getSearchOptions() { + return searchOptions; + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java new file mode 100644 index 00000000..5a5311bd --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch.queries; + +import com.microsoft.semantickernel.data.vectorsearch.options.VectorSearchOptions; +import com.microsoft.semantickernel.exceptions.SKException; + +import javax.annotation.Nullable; + +public class VectorizedSearchQuery extends VectorSearchQuery { + + private final Vector vector; + @Nullable + private final VectorSearchOptions searchOptions; + + public VectorizedSearchQuery(Vector vector, VectorSearchOptions searchOptions) { + super(VectorSearchQueryType.VECTORIZED_SEARCH_QUERY, null); + + this.vector = vector; + this.searchOptions = searchOptions; + } + + public Vector getVector() { + return vector; + } + + public VectorSearchOptions getSearchOptions() { + return searchOptions; + } +} diff --git a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java index 6e895714..3d7b5709 100644 --- a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java +++ b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java @@ -1,9 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordVectorAttribute; import java.util.List; @@ -14,7 +14,7 @@ public class Hotel { private final String name; @VectorStoreRecordDataAttribute private final int code; - @VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "descriptionEmbedding") + @VectorStoreRecordDataAttribute() private final String description; @VectorStoreRecordVectorAttribute(dimensions = 3) private final List descriptionEmbedding; From 43c028c1daab505343db8eff9c623543e079ab73 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 29 Aug 2024 00:28:33 -0700 Subject: [PATCH 014/277] Fix tests --- .../redis/RedisHashSetVectorStoreRecordCollectionTest.java | 2 -- .../memory/redis/RedisJsonVectorStoreRecordCollectionTest.java | 2 -- 2 files changed, 4 deletions(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index deaf9957..eaca6ad2 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -67,8 +67,6 @@ static void setup() { .withName("description") .withStorageName("summary") .withFieldType(String.class) - .withHasEmbedding(true) - .withEmbeddingFieldName("descriptionEmbedding") .build()); fields.add(VectorStoreRecordVectorField.builder() .withName("descriptionEmbedding") diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index 9942c4ac..30582bf6 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -68,8 +68,6 @@ static void setup() { .withName("description") .withStorageName("summary") .withFieldType(String.class) - .withHasEmbedding(true) - .withEmbeddingFieldName("descriptionEmbedding") .build()); fields.add(VectorStoreRecordVectorField.builder() .withName("descriptionEmbedding") From 95dbf63d19fd1feed1e1fbb5cb7bccb78bee6f11 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 5 Sep 2024 22:04:16 -0700 Subject: [PATCH 015/277] Add suggestions and re-organize --- .../AzureAISearchEqualityFilterClause.java | 54 ++++++++++++++ ...reAISearchTagListContainsFilterClause.java | 26 +++++++ ...rchVectorStoreCollectionCreateMapping.java | 2 +- ...rchVectorStoreCollectionSearchMapping.java | 74 ++++--------------- ...reAISearchVectorStoreRecordCollection.java | 7 +- .../semantickernel/data/VectorSearch.java | 2 +- .../options/VectorSearchOptions.java | 3 +- .../filtering/BasicVectorSearchFilter.java | 63 ++++++++++++++++ .../EqualityFilterClause.java | 5 +- .../vectorsearch/filtering/FilterClause.java | 12 +++ .../FilterClauseType.java | 2 +- .../TagListContainsFilterClause.java | 21 ++++++ .../options/BasicVectorSearchFilter.java | 57 -------------- .../vectorsearch/options/FilterClause.java | 28 ------- .../options/TagListContainsFilterClause.java | 37 ---------- .../queries/VectorSearchQuery.java | 2 +- .../queries/VectorizableTextSearchQuery.java | 2 +- .../queries/VectorizedSearchQuery.java | 3 +- 18 files changed, 204 insertions(+), 196 deletions(-) create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchEqualityFilterClause.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchTagListContainsFilterClause.java rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{vectorsearch => record}/options/VectorSearchOptions.java (96%) create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/BasicVectorSearchFilter.java rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/{options => filtering}/EqualityFilterClause.java (70%) create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/FilterClause.java rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/{options => filtering}/FilterClauseType.java (79%) create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/TagListContainsFilterClause.java delete mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/BasicVectorSearchFilter.java delete mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/FilterClause.java delete mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/TagListContainsFilterClause.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchEqualityFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchEqualityFilterClause.java new file mode 100644 index 00000000..dcb3efea --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchEqualityFilterClause.java @@ -0,0 +1,54 @@ +package com.microsoft.semantickernel.connectors.data.azureaisearch; + +import com.microsoft.semantickernel.data.vectorsearch.filtering.EqualityFilterClause; +import com.microsoft.semantickernel.exceptions.SKException; + +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; + +public class AzureAISearchEqualityFilterClause extends EqualityFilterClause { + + /** + * Initializes a new instance of the AzureAISearchEqualityFilterClause class. + * + * @param fieldName The field name to filter on. + * @param value The value. + */ + public AzureAISearchEqualityFilterClause(String fieldName, Object value) { + super(fieldName, value); + } + + /** + * Gets the filter string. + * + * @return The filter string. + */ + @Override + public String getFilter() { + String fieldName = getFieldName(); + Object value = getValue(); + + if (value instanceof String) { + return String.format("%s eq '%s'", fieldName, value); + } else if (value instanceof Boolean) { + return String.format("%s eq %s", fieldName, + value.toString().toLowerCase()); + } else if (value instanceof Integer) { + return String.format("%s eq %d", fieldName, (Integer) value); + } else if (value instanceof Long) { + return String.format("%s eq %d", fieldName, (Long) value); + } else if (value instanceof Float) { + return String.format("%s eq %f", fieldName, (Float) value); + } else if (value instanceof Double) { + return String.format("%s eq %f", fieldName, (Double) value); + } else if (value instanceof OffsetDateTime) { + return String.format("%s eq %s", fieldName, ((OffsetDateTime) value) + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + } else if (value == null) { + return String.format("%s eq null", fieldName); + } else { + throw new SKException("Unsupported filter value type '" + + value.getClass().getSimpleName() + "'."); + } + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchTagListContainsFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchTagListContainsFilterClause.java new file mode 100644 index 00000000..53fd90bd --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchTagListContainsFilterClause.java @@ -0,0 +1,26 @@ +package com.microsoft.semantickernel.connectors.data.azureaisearch; + +import com.microsoft.semantickernel.data.vectorsearch.filtering.TagListContainsFilterClause; + +public class AzureAISearchTagListContainsFilterClause extends TagListContainsFilterClause { + + /** + * Initializes a new instance of the AzureAISearchTagListContainsFilterClause class. + * + * @param fieldName The field name to filter on. + * @param value The value. + */ + public AzureAISearchTagListContainsFilterClause(String fieldName, Object value) { + super(fieldName, value); + } + + /** + * Gets the filter string. + * + * @return The filter string. + */ + @Override + public String getFilter() { + return String.format("%s/any(t: t eq '%s')", getFieldName(), getValue()); + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java index a989179f..09f4d188 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java @@ -20,7 +20,7 @@ import java.util.List; import javax.annotation.Nonnull; -public class AzureAISearchVectorStoreCollectionCreateMapping { +class AzureAISearchVectorStoreCollectionCreateMapping { private static String getVectorSearchProfileName(VectorStoreRecordVectorField vectorField) { return vectorField.getEffectiveStorageName() + "Profile"; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java index 06856e40..8ec2e9fa 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java @@ -1,75 +1,33 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.azureaisearch; -import com.microsoft.semantickernel.data.vectorsearch.options.BasicVectorSearchFilter; -import com.microsoft.semantickernel.data.vectorsearch.options.EqualityFilterClause; -import com.microsoft.semantickernel.data.vectorsearch.options.TagListContainsFilterClause; +import com.microsoft.semantickernel.data.vectorsearch.filtering.BasicVectorSearchFilter; import com.microsoft.semantickernel.exceptions.SKException; -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; -import java.util.List; import java.util.Map; import java.util.stream.Collectors; -public class AzureAISearchVectorStoreCollectionSearchMapping { +class AzureAISearchVectorStoreCollectionSearchMapping { public static String buildFilterString(BasicVectorSearchFilter basicVectorSearchFilter, Map storageNames) { - if (basicVectorSearchFilter == null) { + if (basicVectorSearchFilter == null || basicVectorSearchFilter.getFilterClauses().isEmpty()) { return ""; } - String filter = ""; - - if (basicVectorSearchFilter.getFilterClauses() != null) { - - List filterStrings = basicVectorSearchFilter.getFilterClauses().stream() - .filter(filterClause -> filterClause instanceof EqualityFilterClause) - .map(filterClause -> { - String fieldName = ((EqualityFilterClause) filterClause).getFieldName(); - String storageFieldName = storageNames.get(fieldName); - Object value = ((EqualityFilterClause) filterClause).getValue(); - - if (value instanceof String) { - return String.format("%s eq '%s'", storageFieldName, value); - } else if (value instanceof Boolean) { - return String.format("%s eq %s", storageFieldName, - value.toString().toLowerCase()); - } else if (value instanceof Integer) { - return String.format("%s eq %d", storageFieldName, (Integer) value); - } else if (value instanceof Long) { - return String.format("%s eq %d", storageFieldName, (Long) value); - } else if (value instanceof Float) { - return String.format("%s eq %f", storageFieldName, (Float) value); - } else if (value instanceof Double) { - return String.format("%s eq %f", storageFieldName, (Double) value); - } else if (value instanceof OffsetDateTime) { - return String.format("%s eq %s", storageFieldName, ((OffsetDateTime) value) - .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); - } else if (value == null) { - return String.format("%s eq null", storageFieldName); + return String.join(" and ", basicVectorSearchFilter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof AzureAISearchEqualityFilterClause) { + AzureAISearchEqualityFilterClause azureFilterClause = (AzureAISearchEqualityFilterClause) filterClause; + // Create new instance with the storage name of the field + return new AzureAISearchEqualityFilterClause(storageNames.get(azureFilterClause.getFieldName()), azureFilterClause.getValue()).getFilter(); + } else if (filterClause instanceof AzureAISearchTagListContainsFilterClause) { + AzureAISearchTagListContainsFilterClause azureFilterClause = (AzureAISearchTagListContainsFilterClause) filterClause; + // Create new instance with the storage name of the field + return new AzureAISearchTagListContainsFilterClause(storageNames.get(azureFilterClause.getFieldName()), azureFilterClause.getValue()).getFilter(); } else { - throw new SKException("Unsupported filter value type '" - + value.getClass().getSimpleName() + "'."); + throw new SKException("Unsupported filter clause type '" + + filterClause.getClass().getSimpleName() + "'."); } - }).collect(Collectors.toList()); - - List tagListContainsStrings = basicVectorSearchFilter.getFilterClauses() - .stream() - .filter(filterClause -> filterClause instanceof TagListContainsFilterClause) - .map(filterClause -> { - String fieldName = ((TagListContainsFilterClause) filterClause).getFieldName(); - String storageFieldName = storageNames.get(fieldName); - - TagListContainsFilterClause tagListClause = (TagListContainsFilterClause) filterClause; - return String.format("%s/any(t: t eq '%s')", storageFieldName, - tagListClause.getValue()); - }).collect(Collectors.toList()); - - filterStrings.addAll(tagListContainsStrings); - filter = String.join(" and ", filterStrings); - } - - return filter; + }) + .collect(Collectors.toList())); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index 5d5fb5a0..698272d0 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -12,7 +12,6 @@ import com.azure.search.documents.models.IndexingResult; import com.azure.search.documents.models.SearchOptions; import com.azure.search.documents.models.VectorQuery; -import com.azure.search.documents.models.VectorizableTextQuery; import com.azure.search.documents.models.VectorizedQuery; import com.microsoft.semantickernel.data.VectorSearch; import com.microsoft.semantickernel.data.VectorSearchResult; @@ -26,10 +25,7 @@ import com.microsoft.semantickernel.data.record.options.DeleteRecordOptions; import com.microsoft.semantickernel.data.record.options.GetRecordOptions; import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; -import com.microsoft.semantickernel.data.vectorsearch.options.VectorSearchOptions; -import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; -import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizableTextSearchQuery; -import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizedSearchQuery; +import com.microsoft.semantickernel.data.record.options.VectorSearchOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.time.OffsetDateTime; @@ -319,6 +315,7 @@ public Mono>> searchAsync(Vector vector String filter = AzureAISearchVectorStoreCollectionSearchMapping .buildFilterString(options.getBasicVectorSearchFilter(), storageNames); + SearchOptions searchOptions = new SearchOptions() .setFilter(filter) .setTop(options.getLimit()) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearch.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearch.java index a2b54dec..935a3dd6 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearch.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearch.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data; -import com.microsoft.semantickernel.data.vectorsearch.options.VectorSearchOptions; +import com.microsoft.semantickernel.data.record.options.VectorSearchOptions; import reactor.core.publisher.Mono; import java.util.List; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/VectorSearchOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/VectorSearchOptions.java similarity index 96% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/VectorSearchOptions.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/VectorSearchOptions.java index 8cbbe1ec..3ecb080b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/VectorSearchOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/VectorSearchOptions.java @@ -1,7 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorsearch.options; +package com.microsoft.semantickernel.data.record.options; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.data.vectorsearch.filtering.BasicVectorSearchFilter; import javax.annotation.Nullable; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/BasicVectorSearchFilter.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/BasicVectorSearchFilter.java new file mode 100644 index 00000000..057a7234 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/BasicVectorSearchFilter.java @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch.filtering; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class BasicVectorSearchFilter { + + /** + * Creates a new instance of the BasicVectorSearchFilter class. + * + * @return A new instance of the BasicVectorSearchFilter class. + */ + public static BasicVectorSearchFilter createDefault() { + return new BasicVectorSearchFilter(); + } + + private final List filterClauses; + + public BasicVectorSearchFilter() { + this(new ArrayList<>()); + } + + public BasicVectorSearchFilter(List filterClauses) { + this.filterClauses = new ArrayList<>(filterClauses); + } + + /** + * Gets the filter clauses. + * + * @return The filter clauses. + */ + public List getFilterClauses() { + return Collections.unmodifiableList(filterClauses); + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private final List filterClauses = new ArrayList<>(); + + public Builder equality(EqualityFilterClause equalityFilterClause) { + filterClauses.add(equalityFilterClause); + return this; + } + + public Builder tagListContains(TagListContainsFilterClause tagListContainsFilterClause) { + filterClauses.add(tagListContainsFilterClause); + return this; + } + + public BasicVectorSearchFilter build() { + BasicVectorSearchFilter basicVectorSearchFilter = new BasicVectorSearchFilter(); + basicVectorSearchFilter.filterClauses.addAll(filterClauses); + return basicVectorSearchFilter; + } + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/EqualityFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/EqualityFilterClause.java similarity index 70% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/EqualityFilterClause.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/EqualityFilterClause.java index 0589ddf4..1c662466 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/EqualityFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/EqualityFilterClause.java @@ -1,13 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorsearch.options; +package com.microsoft.semantickernel.data.vectorsearch.filtering; -public class EqualityFilterClause extends FilterClause { +public abstract class EqualityFilterClause implements FilterClause { private final String fieldName; private final Object value; public EqualityFilterClause(String fieldName, Object value) { - super(FilterClauseType.EQUALITY); this.fieldName = fieldName; this.value = value; } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/FilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/FilterClause.java new file mode 100644 index 00000000..d846164b --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/FilterClause.java @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch.filtering; + +public interface FilterClause { + + /** + * Gets the filter. + * + * @return The filter. + */ + String getFilter(); +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/FilterClauseType.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/FilterClauseType.java similarity index 79% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/FilterClauseType.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/FilterClauseType.java index 92b33500..9f597c81 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/FilterClauseType.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/FilterClauseType.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorsearch.options; +package com.microsoft.semantickernel.data.vectorsearch.filtering; public enum FilterClauseType { /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/TagListContainsFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/TagListContainsFilterClause.java new file mode 100644 index 00000000..7c0d0913 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/TagListContainsFilterClause.java @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch.filtering; + +public abstract class TagListContainsFilterClause implements FilterClause { + + private final String fieldName; + private final Object value; + + public TagListContainsFilterClause(String fieldName, Object value) { + this.fieldName = fieldName; + this.value = value; + } + + public String getFieldName() { + return fieldName; + } + + public Object getValue() { + return value; + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/BasicVectorSearchFilter.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/BasicVectorSearchFilter.java deleted file mode 100644 index 6151fc01..00000000 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/BasicVectorSearchFilter.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorsearch.options; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -public class BasicVectorSearchFilter { - - /** - * Creates a new instance of the BasicVectorSearchFilter class. - * - * @return A new instance of the BasicVectorSearchFilter class. - */ - public static BasicVectorSearchFilter createDefault() { - return new BasicVectorSearchFilter(); - } - - private final List filterClauses; - - public BasicVectorSearchFilter() { - this.filterClauses = new ArrayList<>(); - } - - /** - * Adds an equality filter clause to the filter. - * - * @param field The field to filter on. - * @param value The value to filter on. - * @return The BasicVectorSearchFilter object. - */ - public BasicVectorSearchFilter equality(String field, Object value) { - filterClauses.add(new EqualityFilterClause(field, value)); - return this; - } - - /** - * Adds clause to the filter that checks if the tag list contains the specified value. - * - * @param field The field to filter on. - * @param value The value to filter on. - * @return The BasicVectorSearchFilter object. - */ - public BasicVectorSearchFilter tagListContains(String field, String value) { - filterClauses.add(new TagListContainsFilterClause(field, value)); - return this; - } - - /** - * Gets the filter clauses. - * - * @return The filter clauses. - */ - public List getFilterClauses() { - return Collections.unmodifiableList(filterClauses); - } -} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/FilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/FilterClause.java deleted file mode 100644 index a7913264..00000000 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/FilterClause.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorsearch.options; - -public class FilterClause { - - /** - * The type of the filter clause. - */ - private final FilterClauseType type; - - /** - * Creates a new instance of the FilterClause class. - * - * @param type The type of the filter clause. - */ - public FilterClause(FilterClauseType type) { - this.type = type; - } - - /** - * Gets the type of the filter clause. - * - * @return The type of the filter clause. - */ - public FilterClauseType getType() { - return type; - } -} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/TagListContainsFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/TagListContainsFilterClause.java deleted file mode 100644 index 2a3d60e5..00000000 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/options/TagListContainsFilterClause.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorsearch.options; - -public class TagListContainsFilterClause extends FilterClause { - private final String fieldName; - private final String value; - - /** - * Creates a new instance of the TagListContainsFilterClause class. - * - * @param fieldName The name of the field to filter on. - * @param value The value to filter on. - */ - public TagListContainsFilterClause(String fieldName, String value) { - super(FilterClauseType.TAG_LIST_CONTAINS); - this.fieldName = fieldName; - this.value = value; - } - - /** - * Gets the name of the field to filter on. - * - * @return The name of the field to filter on. - */ - public String getFieldName() { - return fieldName; - } - - /** - * Gets the value to filter on. - * - * @return The value to filter on. - */ - public String getValue() { - return value; - } -} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java index ad5b210d..d52e0bdb 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.vectorsearch.queries; -import com.microsoft.semantickernel.data.vectorsearch.options.VectorSearchOptions; +import com.microsoft.semantickernel.data.record.options.VectorSearchOptions; public class VectorSearchQuery { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizableTextSearchQuery.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizableTextSearchQuery.java index 78f4b298..4373fb70 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizableTextSearchQuery.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizableTextSearchQuery.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.vectorsearch.queries; -import com.microsoft.semantickernel.data.vectorsearch.options.VectorSearchOptions; +import com.microsoft.semantickernel.data.record.options.VectorSearchOptions; import javax.annotation.Nullable; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java index 5a5311bd..b2b82e1c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java @@ -1,8 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.vectorsearch.queries; -import com.microsoft.semantickernel.data.vectorsearch.options.VectorSearchOptions; -import com.microsoft.semantickernel.exceptions.SKException; +import com.microsoft.semantickernel.data.record.options.VectorSearchOptions; import javax.annotation.Nullable; From 109a22793246ab548e4239b0af3afcb475c3d7de Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 5 Sep 2024 22:23:16 -0700 Subject: [PATCH 016/277] Format and fix storage name for vector --- .../AzureAISearchEqualityFilterClause.java | 7 ++-- ...reAISearchTagListContainsFilterClause.java | 1 + ...rchVectorStoreCollectionSearchMapping.java | 36 +++++++++++-------- ...reAISearchVectorStoreRecordCollection.java | 5 +-- .../filtering/BasicVectorSearchFilter.java | 21 +++++++++-- .../filtering/EqualityFilterClause.java | 10 ++++++ .../vectorsearch/filtering/FilterClause.java | 2 +- .../TagListContainsFilterClause.java | 10 ++++++ 8 files changed, 68 insertions(+), 24 deletions(-) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchEqualityFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchEqualityFilterClause.java index dcb3efea..59c4d0bb 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchEqualityFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchEqualityFilterClause.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.azureaisearch; import com.microsoft.semantickernel.data.vectorsearch.filtering.EqualityFilterClause; @@ -32,7 +33,7 @@ public String getFilter() { return String.format("%s eq '%s'", fieldName, value); } else if (value instanceof Boolean) { return String.format("%s eq %s", fieldName, - value.toString().toLowerCase()); + value.toString().toLowerCase()); } else if (value instanceof Integer) { return String.format("%s eq %d", fieldName, (Integer) value); } else if (value instanceof Long) { @@ -43,12 +44,12 @@ public String getFilter() { return String.format("%s eq %f", fieldName, (Double) value); } else if (value instanceof OffsetDateTime) { return String.format("%s eq %s", fieldName, ((OffsetDateTime) value) - .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); } else if (value == null) { return String.format("%s eq null", fieldName); } else { throw new SKException("Unsupported filter value type '" - + value.getClass().getSimpleName() + "'."); + + value.getClass().getSimpleName() + "'."); } } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchTagListContainsFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchTagListContainsFilterClause.java index 53fd90bd..204e81d2 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchTagListContainsFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchTagListContainsFilterClause.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.azureaisearch; import com.microsoft.semantickernel.data.vectorsearch.filtering.TagListContainsFilterClause; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java index 8ec2e9fa..195b4749 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java @@ -10,24 +10,30 @@ class AzureAISearchVectorStoreCollectionSearchMapping { public static String buildFilterString(BasicVectorSearchFilter basicVectorSearchFilter, Map storageNames) { - if (basicVectorSearchFilter == null || basicVectorSearchFilter.getFilterClauses().isEmpty()) { + if (basicVectorSearchFilter == null + || basicVectorSearchFilter.getFilterClauses().isEmpty()) { return ""; } - return String.join(" and ", basicVectorSearchFilter.getFilterClauses().stream().map(filterClause -> { - if (filterClause instanceof AzureAISearchEqualityFilterClause) { - AzureAISearchEqualityFilterClause azureFilterClause = (AzureAISearchEqualityFilterClause) filterClause; - // Create new instance with the storage name of the field - return new AzureAISearchEqualityFilterClause(storageNames.get(azureFilterClause.getFieldName()), azureFilterClause.getValue()).getFilter(); - } else if (filterClause instanceof AzureAISearchTagListContainsFilterClause) { - AzureAISearchTagListContainsFilterClause azureFilterClause = (AzureAISearchTagListContainsFilterClause) filterClause; - // Create new instance with the storage name of the field - return new AzureAISearchTagListContainsFilterClause(storageNames.get(azureFilterClause.getFieldName()), azureFilterClause.getValue()).getFilter(); - } else { - throw new SKException("Unsupported filter clause type '" - + filterClause.getClass().getSimpleName() + "'."); - } - }) + return String.join(" and ", + basicVectorSearchFilter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof AzureAISearchEqualityFilterClause) { + AzureAISearchEqualityFilterClause azureFilterClause = (AzureAISearchEqualityFilterClause) filterClause; + // Create new instance with the storage name of the field + return new AzureAISearchEqualityFilterClause( + storageNames.get(azureFilterClause.getFieldName()), + azureFilterClause.getValue()).getFilter(); + } else if (filterClause instanceof AzureAISearchTagListContainsFilterClause) { + AzureAISearchTagListContainsFilterClause azureFilterClause = (AzureAISearchTagListContainsFilterClause) filterClause; + // Create new instance with the storage name of the field + return new AzureAISearchTagListContainsFilterClause( + storageNames.get(azureFilterClause.getFieldName()), + azureFilterClause.getValue()).getFilter(); + } else { + throw new SKException("Unsupported filter clause type '" + + filterClause.getClass().getSimpleName() + "'."); + } + }) .collect(Collectors.toList())); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index 698272d0..0aedebf8 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -309,8 +309,9 @@ public Mono>> searchAsync(Vector vector List vectorQueries = new ArrayList<>(); vectorQueries.add(new VectorizedQuery((List) vector) - .setFields(options.getVectorFieldName() != null ? options.getVectorFieldName() - : firstVectorFieldName) + .setFields( + storageNames.get(options.getVectorFieldName() != null ? options.getVectorFieldName() + : firstVectorFieldName)) .setKNearestNeighborsCount(options.getLimit())); String filter = AzureAISearchVectorStoreCollectionSearchMapping diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/BasicVectorSearchFilter.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/BasicVectorSearchFilter.java index 057a7234..a853ab19 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/BasicVectorSearchFilter.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/BasicVectorSearchFilter.java @@ -24,6 +24,11 @@ public BasicVectorSearchFilter() { this(new ArrayList<>()); } + /** + * Creates a new instance of the BasicVectorSearchFilter class. + * + * @param filterClauses The filter clauses. + */ public BasicVectorSearchFilter(List filterClauses) { this.filterClauses = new ArrayList<>(filterClauses); } @@ -44,20 +49,30 @@ public static Builder builder() { public static class Builder { private final List filterClauses = new ArrayList<>(); + /** + * Adds an equality filter clause to the filter. + * + * @param equalityFilterClause The equality filter clause to add. + * @return The builder. + */ public Builder equality(EqualityFilterClause equalityFilterClause) { filterClauses.add(equalityFilterClause); return this; } + /** + * Adds a tag list contains filter clause to the filter. + * + * @param tagListContainsFilterClause The tag list contains filter clause to add. + * @return The builder. + */ public Builder tagListContains(TagListContainsFilterClause tagListContainsFilterClause) { filterClauses.add(tagListContainsFilterClause); return this; } public BasicVectorSearchFilter build() { - BasicVectorSearchFilter basicVectorSearchFilter = new BasicVectorSearchFilter(); - basicVectorSearchFilter.filterClauses.addAll(filterClauses); - return basicVectorSearchFilter; + return new BasicVectorSearchFilter(filterClauses); } } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/EqualityFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/EqualityFilterClause.java index 1c662466..3580de42 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/EqualityFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/EqualityFilterClause.java @@ -11,10 +11,20 @@ public EqualityFilterClause(String fieldName, Object value) { this.value = value; } + /** + * Gets the field name to filter on. + * + * @return The field name to filter on. + */ public String getFieldName() { return fieldName; } + /** + * Gets the value. + * + * @return The value. + */ public Object getValue() { return value; } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/FilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/FilterClause.java index d846164b..6d45e339 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/FilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/FilterClause.java @@ -4,7 +4,7 @@ public interface FilterClause { /** - * Gets the filter. + * Gets the filter string. * * @return The filter. */ diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/TagListContainsFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/TagListContainsFilterClause.java index 7c0d0913..6dd54329 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/TagListContainsFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/TagListContainsFilterClause.java @@ -11,10 +11,20 @@ public TagListContainsFilterClause(String fieldName, Object value) { this.value = value; } + /** + * Gets the field name to filter on. + * + * @return The field name to filter on. + */ public String getFieldName() { return fieldName; } + /** + * Gets the value. + * + * @return The value. + */ public Object getValue() { return value; } From bb99af2fd30d7ff0a098de0bfaaf144b617b9f21 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Sun, 8 Sep 2024 22:59:36 -0700 Subject: [PATCH 017/277] Remove vectorizable search --- .../AzureAISearchVectorStoreRecordCollection.java | 13 ------------- .../microsoft/semantickernel/data/VectorSearch.java | 10 ---------- 2 files changed, 23 deletions(-) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index 0aedebf8..47571d1a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -330,17 +330,4 @@ public Mono>> searchAsync(Vector vector return searchAndMapAsync(null, searchOptions); } - - /** - * Vectorizable text search. This method searches for records that are similar to the given text. - * - * @param searchText The text to search with. - * @param options The options to use for the search. - * @return A list of search results. - */ - @Override - public Mono>> searchAsync(String searchText, - VectorSearchOptions options) { - throw new UnsupportedOperationException("Not implemented"); - } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearch.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearch.java index 935a3dd6..d1aec34c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearch.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearch.java @@ -16,14 +16,4 @@ public interface VectorSearch { */ Mono>> searchAsync(Vector vector, VectorSearchOptions options); - - /** - * Vectorizable text search. This method searches for records that are similar to the given text. - * - * @param searchText The text to search with. - * @param options The options to use for the search. - * @return A list of search results. - */ - Mono>> searchAsync(String searchText, - VectorSearchOptions options); } From 3dbc1f2ce21f4eae8ba5b71d10d8663226635b71 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 10 Sep 2024 02:21:57 -0700 Subject: [PATCH 018/277] Add azure vectorizable search --- .../memory/AzureAISearchVectorStore.java | 2 +- ...reAISearchVectorStoreRecordCollection.java | 42 +++++++++++++------ .../semantickernel/data/VectorSearch.java | 30 +++++++++++-- .../queries/VectorSearchQuery.java | 7 ++-- .../queries/VectorizedSearchQuery.java | 12 +++--- 5 files changed, 69 insertions(+), 24 deletions(-) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java index 7e06c895..6e7e7a78 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java @@ -151,7 +151,7 @@ private static Mono>> search( OpenAITextEmbeddingGenerationService embeddingGeneration) { return embeddingGeneration.generateEmbeddingsAsync(Collections.singletonList(searchText)) - .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); + .flatMap(r -> recordCollection.searchAsync("How to get started", null)); } private static Mono> storeData( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index 47571d1a..46c7ba8f 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -12,6 +12,7 @@ import com.azure.search.documents.models.IndexingResult; import com.azure.search.documents.models.SearchOptions; import com.azure.search.documents.models.VectorQuery; +import com.azure.search.documents.models.VectorizableTextQuery; import com.azure.search.documents.models.VectorizedQuery; import com.microsoft.semantickernel.data.VectorSearch; import com.microsoft.semantickernel.data.VectorSearchResult; @@ -26,6 +27,9 @@ import com.microsoft.semantickernel.data.record.options.GetRecordOptions; import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; import com.microsoft.semantickernel.data.record.options.VectorSearchOptions; +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizableTextSearchQuery; +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizedSearchQuery; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.time.OffsetDateTime; @@ -42,8 +46,9 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -public class AzureAISearchVectorStoreRecordCollection implements - VectorStoreRecordCollection, VectorSearch { +public class AzureAISearchVectorStoreRecordCollection extends VectorSearch + implements + VectorStoreRecordCollection { private static final HashSet> supportedKeyTypes = new HashSet<>( Collections.singletonList( @@ -290,29 +295,42 @@ record = response.getDocument(this.options.getRecordClass()); } /** - * Vectorized search. This method searches for records that are similar to the given vector. + * Search the vector store for records that match the given embedding and filter. * - * @param vector The vector to search with. - * @param options The options to use for the search. + * @param query The vector search query. * @return A list of search results. */ @Override - public Mono>> searchAsync(Vector vector, - VectorSearchOptions options) { + public Mono>> searchAsync(VectorSearchQuery query) { if (firstVectorFieldName == null) { throw new SKException("No vector fields defined. Cannot perform vector search"); } + VectorSearchOptions options = (VectorSearchOptions) query.getSearchOptions(); if (options == null) { options = VectorSearchOptions.createDefault(firstVectorFieldName); } List vectorQueries = new ArrayList<>(); - vectorQueries.add(new VectorizedQuery((List) vector) - .setFields( - storageNames.get(options.getVectorFieldName() != null ? options.getVectorFieldName() - : firstVectorFieldName)) - .setKNearestNeighborsCount(options.getLimit())); + + if (query instanceof VectorizedSearchQuery) { + vectorQueries.add(new VectorizedQuery(((VectorizedSearchQuery) query).getVector()) + .setFields( + storageNames + .get(options.getVectorFieldName() != null ? options.getVectorFieldName() + : firstVectorFieldName)) + .setKNearestNeighborsCount(options.getLimit())); + } else if (query instanceof VectorizableTextSearchQuery) { + vectorQueries + .add(new VectorizableTextQuery(((VectorizableTextSearchQuery) query).getQueryText()) + .setFields( + storageNames + .get(options.getVectorFieldName() != null ? options.getVectorFieldName() + : firstVectorFieldName)) + .setKNearestNeighborsCount(options.getLimit())); + } else { + throw new SKException("Unsupported query type: " + query.getQueryType()); + } String filter = AzureAISearchVectorStoreCollectionSearchMapping .buildFilterString(options.getBasicVectorSearchFilter(), storageNames); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearch.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearch.java index d1aec34c..3be6679b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearch.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearch.java @@ -2,11 +2,21 @@ package com.microsoft.semantickernel.data; import com.microsoft.semantickernel.data.record.options.VectorSearchOptions; +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; import reactor.core.publisher.Mono; import java.util.List; -public interface VectorSearch { +public abstract class VectorSearch { + + /** + * Search the vector store for records that match the given embedding and filter. + * + * @param query The vector search query. + * @return A list of search results. + */ + public abstract Mono>> searchAsync(VectorSearchQuery query); + /** * Vectorized search. This method searches for records that are similar to the given vector. * @@ -14,6 +24,20 @@ public interface VectorSearch { * @param options The options to use for the search. * @return A list of search results. */ - Mono>> searchAsync(Vector vector, - VectorSearchOptions options); + public Mono>> searchAsync(List vector, + VectorSearchOptions options) { + return searchAsync(VectorSearchQuery.createQuery(vector, options)); + } + + /** + * Vectorizable text search. This method searches for records that are similar to the given text. + * + * @param searchText The text to search with. + * @param options The options to use for the search. + * @return A list of search results. + */ + public Mono>> searchAsync(String searchText, + VectorSearchOptions options) { + return searchAsync(VectorSearchQuery.createQuery(searchText, options)); + } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java index d52e0bdb..83511ce1 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java @@ -3,8 +3,9 @@ import com.microsoft.semantickernel.data.record.options.VectorSearchOptions; -public class VectorSearchQuery { +import java.util.List; +public class VectorSearchQuery { private final VectorSearchQueryType queryType; private final Object searchOptions; @@ -21,9 +22,9 @@ public Object getSearchOptions() { return searchOptions; } - public static VectorizedSearchQuery createQuery(Vector vector, + public static VectorizedSearchQuery createQuery(List vector, VectorSearchOptions options) { - return new VectorizedSearchQuery<>(vector, options); + return new VectorizedSearchQuery(vector, options); } public static VectorizableTextSearchQuery createQuery(String searchText, diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java index b2b82e1c..aeb429a6 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java @@ -4,21 +4,23 @@ import com.microsoft.semantickernel.data.record.options.VectorSearchOptions; import javax.annotation.Nullable; +import java.util.Collections; +import java.util.List; -public class VectorizedSearchQuery extends VectorSearchQuery { +public class VectorizedSearchQuery extends VectorSearchQuery { - private final Vector vector; + private final List vector; @Nullable private final VectorSearchOptions searchOptions; - public VectorizedSearchQuery(Vector vector, VectorSearchOptions searchOptions) { + public VectorizedSearchQuery(List vector, VectorSearchOptions searchOptions) { super(VectorSearchQueryType.VECTORIZED_SEARCH_QUERY, null); - this.vector = vector; + this.vector = Collections.unmodifiableList(vector); this.searchOptions = searchOptions; } - public Vector getVector() { + public List getVector() { return vector; } From c71e22c133d027674fb142ac3f2215ca156ab9de Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 10 Sep 2024 08:43:27 -0700 Subject: [PATCH 019/277] Add "Prelease" to typos exclude --- .github/_typos.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/_typos.toml b/.github/_typos.toml index a92576ba..89dbcf79 100644 --- a/.github/_typos.toml +++ b/.github/_typos.toml @@ -42,3 +42,8 @@ extend-ignore-re = [ extend-ignore-re = [ 'Version=".*"', # ignore package version numbers ] + +[exclude] +words = [ + "Prelease" +] From ab01be40be96ba53dfea189d2dd7094f112397e0 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 10 Sep 2024 08:47:29 -0700 Subject: [PATCH 020/277] Update --- .github/_typos.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/_typos.toml b/.github/_typos.toml index 89dbcf79..67732488 100644 --- a/.github/_typos.toml +++ b/.github/_typos.toml @@ -43,7 +43,7 @@ extend-ignore-re = [ 'Version=".*"', # ignore package version numbers ] -[exclude] -words = [ +[default] +ignore-words = [ "Prelease" ] From ca74c786d04652d295b4486051a7f75c8f763d05 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 10 Sep 2024 08:51:23 -0700 Subject: [PATCH 021/277] Update --- .github/_typos.toml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/_typos.toml b/.github/_typos.toml index 67732488..c101713b 100644 --- a/.github/_typos.toml +++ b/.github/_typos.toml @@ -29,6 +29,7 @@ ans = "ans" # Short for answers arange = "arange" # Method in Python numpy package prompty = "prompty" # prompty is a format name. ist = "ist" # German language +Prelease = "Prelease" # Prelease is a format name. [default.extend-identifiers] ags = "ags" # Azure Graph Service @@ -42,8 +43,3 @@ extend-ignore-re = [ extend-ignore-re = [ 'Version=".*"', # ignore package version numbers ] - -[default] -ignore-words = [ - "Prelease" -] From 0979ab05a5f10a4335c09d0e337b35865876552f Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 10 Sep 2024 08:53:20 -0700 Subject: [PATCH 022/277] Update actions/upload-artifact --- .github/workflows/java-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/java-build.yml b/.github/workflows/java-build.yml index d4b487cf..f9da894f 100644 --- a/.github/workflows/java-build.yml +++ b/.github/workflows/java-build.yml @@ -56,7 +56,7 @@ jobs: run: ./mvnw -B -Pbug-check -Pcompile-jdk${{ matrix.java-versions }} test --file pom.xml # Uploads test artifacts for each JDK version - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 if: always() with: name: test_output_sk_jdk${{ matrix.java-versions }}u From f92a0ed1fc6b01f219b5a5a41d8dd2004813e611 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 10 Sep 2024 08:55:38 -0700 Subject: [PATCH 023/277] Fix example --- .../samples/syntaxexamples/memory/AzureAISearchVectorStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java index 6e7e7a78..7e06c895 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java @@ -151,7 +151,7 @@ private static Mono>> search( OpenAITextEmbeddingGenerationService embeddingGeneration) { return embeddingGeneration.generateEmbeddingsAsync(Collections.singletonList(searchText)) - .flatMap(r -> recordCollection.searchAsync("How to get started", null)); + .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); } private static Mono> storeData( From 99e67c8645e00d88e0150b6317e28d26dea2349e Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 10 Sep 2024 09:31:59 -0700 Subject: [PATCH 024/277] Format --- .github/_typos.toml | 1 + .github/workflows/java-build.yml | 2 +- .../postgres/PostgreSQLVectorStoreQueryProvider.java | 12 ++++++------ 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/_typos.toml b/.github/_typos.toml index a92576ba..c101713b 100644 --- a/.github/_typos.toml +++ b/.github/_typos.toml @@ -29,6 +29,7 @@ ans = "ans" # Short for answers arange = "arange" # Method in Python numpy package prompty = "prompty" # prompty is a format name. ist = "ist" # German language +Prelease = "Prelease" # Prelease is a format name. [default.extend-identifiers] ags = "ags" # Azure Graph Service diff --git a/.github/workflows/java-build.yml b/.github/workflows/java-build.yml index d4b487cf..f9da894f 100644 --- a/.github/workflows/java-build.yml +++ b/.github/workflows/java-build.yml @@ -56,7 +56,7 @@ jobs: run: ./mvnw -B -Pbug-check -Pcompile-jdk${{ matrix.java-versions }} test --file pom.xml # Uploads test artifacts for each JDK version - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 if: always() with: name: test_output_sk_jdk${{ matrix.java-versions }}u diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index 95b34698..80729b20 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -195,13 +195,13 @@ public void createCollection(String collectionName, Statement createTableAndIndexes = connection.createStatement()) { String createStorageTable = "CREATE TABLE IF NOT EXISTS " - + getCollectionTableName(collectionName) + " (" - + getKeyColumnName(recordDefinition.getKeyField()) + " VARCHAR(255) PRIMARY KEY, " - + getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), + + getCollectionTableName(collectionName) + " (" + + getKeyColumnName(recordDefinition.getKeyField()) + " VARCHAR(255) PRIMARY KEY, " + + getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), supportedDataTypes) - + ", " - + getColumnNamesAndTypesForVectorFields(recordDefinition.getVectorFields()) - + ");"; + + ", " + + getColumnNamesAndTypesForVectorFields(recordDefinition.getVectorFields()) + + ");"; createTableAndIndexes.addBatch(createStorageTable); for (VectorStoreRecordVectorField vectorField : vectorFields) { From b5ad055c8386e2faa3d7f69ac582aa2afe09a1d5 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 10 Sep 2024 09:35:57 -0700 Subject: [PATCH 025/277] Updates --- .../data/azureaisearch/AzureAISearchVectorStore.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java index e2f084bf..4716c300 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java @@ -126,10 +126,10 @@ public Builder withOptions( */ public AzureAISearchVectorStore build() { if (searchIndexAsyncClient == null) { - throw new IllegalStateException("searchIndexAsyncClient is required"); + throw new SKException("searchIndexAsyncClient is required"); } if (options == null) { - throw new IllegalStateException("options is required"); + throw new SKException("options is required"); } return new AzureAISearchVectorStore(searchIndexAsyncClient, options); From 881ddd2e8e57da7a521094bd3f80be0e45847976 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 10 Sep 2024 18:40:44 -0700 Subject: [PATCH 026/277] Add suggestions and update packages --- .../tests/connectors/memory/Hotel.java | 6 +- .../JDBCVectorStoreRecordCollectionTest.java | 2 +- ...ashSetVectorStoreRecordCollectionTest.java | 12 +-- ...isJsonVectorStoreRecordCollectionTest.java | 12 +-- .../memory/redis/RedisVectorStoreTest.java | 5 +- .../memory/AzureAISearchVectorStore.java | 13 +++- .../memory/InMemory_DataStorage.java | 8 +- .../memory/JDBC_DataStorage.java | 8 +- .../memory/Redis_DataStorage.java | 8 +- .../AzureAISearchEqualityFilterClause.java | 2 +- ...reAISearchTagListContainsFilterClause.java | 2 +- .../AzureAISearchVectorStore.java | 6 +- ...rchVectorStoreCollectionCreateMapping.java | 7 +- ...rchVectorStoreCollectionSearchMapping.java | 2 +- ...reAISearchVectorStoreRecordCollection.java | 76 ++++++++++++++----- ...rchVectorStoreRecordCollectionFactory.java | 2 +- ...rchVectorStoreRecordCollectionOptions.java | 6 +- .../connectors/data/jdbc/JDBCVectorStore.java | 4 +- .../JDBCVectorStoreDefaultQueryProvider.java | 12 +-- .../jdbc/JDBCVectorStoreQueryProvider.java | 10 +-- .../jdbc/JDBCVectorStoreRecordCollection.java | 12 +-- ...DBCVectorStoreRecordCollectionFactory.java | 2 +- ...DBCVectorStoreRecordCollectionOptions.java | 6 +- .../jdbc/JDBCVectorStoreRecordMapper.java | 8 +- .../connectors/data/jdbc/SQLVectorStore.java | 2 +- .../jdbc/SQLVectorStoreRecordCollection.java | 2 +- .../mysql/MySQLVectorStoreQueryProvider.java | 8 +- .../PostgreSQLVectorStoreQueryProvider.java | 10 +-- .../PostgreSQLVectorStoreRecordMapper.java | 8 +- ...disHashSetVectorStoreRecordCollection.java | 14 ++-- ...SetVectorStoreRecordCollectionOptions.java | 6 +- .../RedisHashSetVectorStoreRecordMapper.java | 8 +- .../RedisJsonVectorStoreRecordCollection.java | 14 ++-- ...sonVectorStoreRecordCollectionOptions.java | 6 +- .../RedisJsonVectorStoreRecordMapper.java | 2 +- .../data/redis/RedisVectorStore.java | 6 +- ...disVectorStoreCollectionCreateMapping.java | 8 +- ...disVectorStoreRecordCollectionFactory.java | 4 +- .../semantickernel/data/VectorSearch.java | 43 ----------- .../data/VolatileVectorStore.java | 3 + .../VolatileVectorStoreRecordCollection.java | 9 ++- ...ileVectorStoreRecordCollectionOptions.java | 3 +- .../filtering/BasicVectorSearchFilter.java | 8 +- .../filtering/EqualityFilterClause.java | 2 +- .../filtering/FilterClause.java | 2 +- .../TagListContainsFilterClause.java | 2 +- .../data/vectorsearch/VectorSearch.java | 12 +++ .../VectorSearchResult.java | 2 +- .../data/vectorsearch/VectorizableSearch.java | 19 +++++ .../data/vectorsearch/VectorizedSearch.java | 20 +++++ .../filtering/FilterClauseType.java | 13 ---- .../queries/VectorSearchQuery.java | 2 +- .../queries/VectorizableTextSearchQuery.java | 2 +- .../queries/VectorizedSearchQuery.java | 2 +- .../data/{ => vectorstorage}/VectorStore.java | 2 +- .../VectorStoreRecordCollection.java | 8 +- .../VectorStoreRecordCollectionOptions.java | 4 +- .../VectorStoreRecordMapper.java | 2 +- .../VectorStoreRecordDataAttribute.java | 2 +- .../VectorStoreRecordKeyAttribute.java | 2 +- .../VectorStoreRecordVectorAttribute.java | 2 +- .../definition/DistanceFunction.java | 2 +- .../definition/IndexKind.java | 2 +- .../VectorStoreRecordDataField.java | 2 +- .../VectorStoreRecordDefinition.java | 8 +- .../definition/VectorStoreRecordField.java | 2 +- .../definition/VectorStoreRecordKeyField.java | 2 +- .../VectorStoreRecordVectorField.java | 2 +- .../options/DeleteRecordOptions.java | 2 +- .../options/GetRecordOptions.java | 2 +- .../options/UpsertRecordOptions.java | 2 +- .../options/VectorSearchOptions.java | 9 ++- .../microsoft/semantickernel/data/Hotel.java | 6 +- 73 files changed, 287 insertions(+), 249 deletions(-) delete mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearch.java rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{vectorsearch => }/filtering/BasicVectorSearchFilter.java (89%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{vectorsearch => }/filtering/EqualityFilterClause.java (90%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{vectorsearch => }/filtering/FilterClause.java (74%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{vectorsearch => }/filtering/TagListContainsFilterClause.java (90%) create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearch.java rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{ => vectorsearch}/VectorSearchResult.java (92%) create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableSearch.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java delete mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/FilterClauseType.java rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{ => vectorstorage}/VectorStore.java (93%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{ => vectorstorage}/VectorStoreRecordCollection.java (90%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{ => vectorstorage}/VectorStoreRecordCollectionOptions.java (76%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{ => vectorstorage}/VectorStoreRecordMapper.java (97%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{record => vectorstorage}/attributes/VectorStoreRecordDataAttribute.java (93%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{record => vectorstorage}/attributes/VectorStoreRecordKeyAttribute.java (87%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{record => vectorstorage}/attributes/VectorStoreRecordVectorAttribute.java (92%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{record => vectorstorage}/definition/DistanceFunction.java (93%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{record => vectorstorage}/definition/IndexKind.java (92%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{record => vectorstorage}/definition/VectorStoreRecordDataField.java (97%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{record => vectorstorage}/definition/VectorStoreRecordDefinition.java (95%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{record => vectorstorage}/definition/VectorStoreRecordField.java (97%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{record => vectorstorage}/definition/VectorStoreRecordKeyField.java (94%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{record => vectorstorage}/definition/VectorStoreRecordVectorField.java (98%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{record => vectorstorage}/options/DeleteRecordOptions.java (66%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{record => vectorstorage}/options/GetRecordOptions.java (95%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{record => vectorstorage}/options/UpsertRecordOptions.java (66%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{record => vectorstorage}/options/VectorSearchOptions.java (92%) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java index 9e5a5699..e69585ed 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java @@ -2,9 +2,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; import java.util.List; diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java index 3f61b1d8..cb1bb2de 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -5,7 +5,7 @@ import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.record.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.tests.connectors.memory.Hotel; import com.mysql.cj.jdbc.MysqlDataSource; import org.junit.jupiter.params.ParameterizedTest; diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index eaca6ad2..24945df6 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -2,12 +2,12 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordKeyField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordVectorField; -import com.microsoft.semantickernel.data.record.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.tests.connectors.memory.Hotel; import com.redis.testcontainers.RedisContainer; import org.junit.jupiter.api.BeforeAll; diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index 30582bf6..9d833217 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -2,12 +2,12 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordKeyField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordVectorField; -import com.microsoft.semantickernel.data.record.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.tests.connectors.memory.Hotel; import com.redis.testcontainers.RedisContainer; import org.junit.jupiter.api.BeforeAll; diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java index 43fead85..fd63b9e5 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java @@ -5,12 +5,9 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisStorageType; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStore; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreOptions; -import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.tests.connectors.memory.Hotel; import com.redis.testcontainers.RedisContainer; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.testcontainers.junit.jupiter.Container; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java index 7e06c895..2a1a49ab 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java @@ -15,10 +15,10 @@ import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreOptions; import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.VectorSearchResult; -import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Base64; @@ -139,6 +139,11 @@ public static void dataStorageWithAzureAISearch( // Search for results // Might need to wait for the data to be indexed var results = search("How to get started", collection, embeddingGeneration).block(); + + if (results == null || results.isEmpty()) { + System.out.println("No search results found."); + return; + } var searchResult = results.get(0); System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", searchResult.getScore(), searchResult.getRecord().link, searchResult.getRecord().description); diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java index 55f497e7..70c7297d 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java @@ -6,12 +6,12 @@ import com.azure.core.credential.AzureKeyCredential; import com.azure.core.credential.KeyCredential; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.VolatileVectorStore; import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; import java.util.Arrays; import java.util.Collections; import java.util.List; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java index 281edd1a..9e2e4ea4 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java @@ -10,10 +10,10 @@ import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; import com.mysql.cj.jdbc.MysqlDataSource; import java.nio.charset.StandardCharsets; import java.sql.SQLException; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java index cc7711c0..90acae83 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java @@ -12,10 +12,10 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStore; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreOptions; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; import java.util.Arrays; import java.util.Collections; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchEqualityFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchEqualityFilterClause.java index 59c4d0bb..98da48d1 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchEqualityFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchEqualityFilterClause.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.azureaisearch; -import com.microsoft.semantickernel.data.vectorsearch.filtering.EqualityFilterClause; +import com.microsoft.semantickernel.data.filtering.EqualityFilterClause; import com.microsoft.semantickernel.exceptions.SKException; import java.time.OffsetDateTime; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchTagListContainsFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchTagListContainsFilterClause.java index 204e81d2..8e2ba0bf 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchTagListContainsFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchTagListContainsFilterClause.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.azureaisearch; -import com.microsoft.semantickernel.data.vectorsearch.filtering.TagListContainsFilterClause; +import com.microsoft.semantickernel.data.filtering.TagListContainsFilterClause; public class AzureAISearchTagListContainsFilterClause extends TagListContainsFilterClause { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java index 4716c300..e0e763c1 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java @@ -3,9 +3,9 @@ import com.azure.search.documents.indexes.SearchIndexAsyncClient; import com.azure.search.documents.indexes.models.SearchIndex; -import com.microsoft.semantickernel.data.VectorStore; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStore; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.List; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java index 09f4d188..3876631a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java @@ -10,10 +10,9 @@ import com.azure.search.documents.indexes.models.VectorSearchAlgorithmConfiguration; import com.azure.search.documents.indexes.models.VectorSearchAlgorithmMetric; import com.azure.search.documents.indexes.models.VectorSearchProfile; -import com.azure.search.documents.indexes.models.VectorSearchVectorizerKind; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordKeyField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.exceptions.SKException; import java.time.OffsetDateTime; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java index 195b4749..016568f6 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.azureaisearch; -import com.microsoft.semantickernel.data.vectorsearch.filtering.BasicVectorSearchFilter; +import com.microsoft.semantickernel.data.filtering.BasicVectorSearchFilter; import com.microsoft.semantickernel.exceptions.SKException; import java.util.Map; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index 46c7ba8f..e4632a36 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -14,19 +14,21 @@ import com.azure.search.documents.models.VectorQuery; import com.azure.search.documents.models.VectorizableTextQuery; import com.azure.search.documents.models.VectorizedQuery; -import com.microsoft.semantickernel.data.VectorSearch; -import com.microsoft.semantickernel.data.VectorSearchResult; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordKeyField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordVectorField; -import com.microsoft.semantickernel.data.record.options.DeleteRecordOptions; -import com.microsoft.semantickernel.data.record.options.GetRecordOptions; -import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; -import com.microsoft.semantickernel.data.record.options.VectorSearchOptions; +import com.microsoft.semantickernel.data.vectorsearch.VectorizableSearch; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearch; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.VectorizedSearch; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizableTextSearchQuery; import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizedSearchQuery; @@ -46,9 +48,10 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -public class AzureAISearchVectorStoreRecordCollection extends VectorSearch - implements - VectorStoreRecordCollection { +public class AzureAISearchVectorStoreRecordCollection implements + VectorStoreRecordCollection, + VectorizedSearch, + VectorizableSearch { private static final HashSet> supportedKeyTypes = new HashSet<>( Collections.singletonList( @@ -207,10 +210,15 @@ public Mono getAsync( return searchAsyncClient .getDocumentWithResponse(key, this.options.getRecordClass(), selectedFields) .flatMap(response -> { + int statusCode = response.getStatusCode(); + if (statusCode >= 200 && statusCode < 300) { + return Mono.just(response.getValue()); + } if (response.getStatusCode() == 404) { return Mono.error(new SKException("Record not found: " + key)); } - return Mono.just(response.getValue()); + return Mono.error(new SKException("Failed to get record: " + key + ". Status code: " + + statusCode)); }); } @@ -273,12 +281,11 @@ public Mono deleteBatchAsync(List keys, DeleteRecordOptions option }).collect(Collectors.toList())).then(); } - private Mono>> searchAndMapAsync(String searchText, - SearchOptions searchOptions) { + private Mono>> searchAndMapAsync(SearchOptions searchOptions) { VectorStoreRecordMapper mapper = this.options .getVectorStoreRecordMapper(); - return this.searchAsyncClient.search(searchText, searchOptions) + return this.searchAsyncClient.search(null, searchOptions) .flatMap(response -> { Record record; @@ -339,6 +346,7 @@ public Mono>> searchAsync(VectorSearchQuery quer .setFilter(filter) .setTop(options.getLimit()) .setSkip(options.getOffset()) + .setScoringParameters() .setVectorSearchOptions(new com.azure.search.documents.models.VectorSearchOptions() .setQueries(vectorQueries)); @@ -346,6 +354,32 @@ public Mono>> searchAsync(VectorSearchQuery quer searchOptions.setSelect(nonVectorFields.toArray(new String[0])); } - return searchAndMapAsync(null, searchOptions); + return searchAndMapAsync(searchOptions); + } + + /** + * Vectorizable text search. This method searches for records that are similar to the given text. + * + * @param searchText The text to search with. + * @param options The options to use for the search. + * @return A list of search results. + */ + @Override + public Mono>> searchAsync(String searchText, + VectorSearchOptions options) { + return searchAsync(VectorSearchQuery.createQuery(searchText, options)); + } + + /** + * Vectorized search. This method searches for records that are similar to the given vector. + * + * @param vector The vector to search with. + * @param options The options to use for the search. + * @return A list of search results. + */ + @Override + public Mono>> searchAsync(List vector, + VectorSearchOptions options) { + return searchAsync(VectorSearchQuery.createQuery(vector, options)); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java index b5bfba51..b0acf545 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java @@ -2,7 +2,7 @@ package com.microsoft.semantickernel.connectors.data.azureaisearch; import com.azure.search.documents.indexes.SearchIndexAsyncClient; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; /** * Factory for creating Azure AI Search vector store record collections. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java index d9cc43a8..d1fad192 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java @@ -2,9 +2,9 @@ package com.microsoft.semantickernel.connectors.data.azureaisearch; import com.azure.search.documents.SearchDocument; -import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; import javax.annotation.Nonnull; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java index 01baf6c7..1591918d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java @@ -1,8 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import reactor.core.publisher.Mono; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java index 5c640ae2..7f2f077b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java @@ -1,13 +1,13 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.record.options.DeleteRecordOptions; -import com.microsoft.semantickernel.data.record.options.GetRecordOptions; -import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.annotation.Nonnull; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index 62a8ad5d..757253c8 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -2,11 +2,11 @@ package com.microsoft.semantickernel.connectors.data.jdbc; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.record.options.DeleteRecordOptions; -import com.microsoft.semantickernel.data.record.options.GetRecordOptions; -import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import java.sql.ResultSet; import java.util.List; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java index 90d041e1..8db96c20 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -5,12 +5,12 @@ import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreRecordMapper; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.record.options.DeleteRecordOptions; -import com.microsoft.semantickernel.data.record.options.GetRecordOptions; -import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.lang.reflect.Field; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java index d4d92111..14cdcc27 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import javax.sql.DataSource; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java index 366e28ad..c9046f59 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java @@ -1,9 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; -import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java index f0974f2b..40e654f2 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java @@ -6,10 +6,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java index 046f9941..ee45f8be 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; -import com.microsoft.semantickernel.data.VectorStore; +import com.microsoft.semantickernel.data.vectorstorage.VectorStore; import reactor.core.publisher.Mono; public interface SQLVectorStore diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java index ff12c88b..711344fa 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import reactor.core.publisher.Mono; public interface SQLVectorStoreRecordCollection diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java index bbd40e96..cd6977cd 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java @@ -6,10 +6,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreDefaultQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordVectorField; -import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index 0a37bec7..d63c46c8 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -6,11 +6,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreDefaultQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordKeyField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordVectorField; -import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java index 92434847..5db2c112 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java @@ -6,10 +6,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.postgresql.util.PGobject; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java index f7e6432b..1c6c5e42 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java @@ -2,13 +2,13 @@ package com.microsoft.semantickernel.connectors.data.redis; import com.fasterxml.jackson.databind.ObjectMapper; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.record.options.DeleteRecordOptions; -import com.microsoft.semantickernel.data.record.options.GetRecordOptions; -import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java index 1e2056fb..ab062708 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java @@ -1,9 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; -import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; import javax.annotation.Nonnull; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java index cfa7a80f..e4f843ad 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java @@ -6,10 +6,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.exceptions.SKException; import javax.annotation.Nullable; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java index f37d9e14..f99e2bba 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java @@ -4,13 +4,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.record.options.DeleteRecordOptions; -import com.microsoft.semantickernel.data.record.options.GetRecordOptions; -import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java index 087c79ad..6c35fa82 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java @@ -1,9 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; -import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; import javax.annotation.Nonnull; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java index d33f6d65..76db6862 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java @@ -4,7 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.data.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java index c7953da3..b9bf20b9 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java @@ -2,9 +2,9 @@ package com.microsoft.semantickernel.connectors.data.redis; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.data.VectorStore; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.VectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStore; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.ArrayList; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java index d64afb94..0a172899 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java @@ -1,10 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordKeyField; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java index 4025b37b..0752493c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java @@ -1,8 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import redis.clients.jedis.JedisPooled; /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearch.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearch.java deleted file mode 100644 index 3be6679b..00000000 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearch.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data; - -import com.microsoft.semantickernel.data.record.options.VectorSearchOptions; -import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; -import reactor.core.publisher.Mono; - -import java.util.List; - -public abstract class VectorSearch { - - /** - * Search the vector store for records that match the given embedding and filter. - * - * @param query The vector search query. - * @return A list of search results. - */ - public abstract Mono>> searchAsync(VectorSearchQuery query); - - /** - * Vectorized search. This method searches for records that are similar to the given vector. - * - * @param vector The vector to search with. - * @param options The options to use for the search. - * @return A list of search results. - */ - public Mono>> searchAsync(List vector, - VectorSearchOptions options) { - return searchAsync(VectorSearchQuery.createQuery(vector, options)); - } - - /** - * Vectorizable text search. This method searches for records that are similar to the given text. - * - * @param searchText The text to search with. - * @param options The options to use for the search. - * @return A list of search results. - */ - public Mono>> searchAsync(String searchText, - VectorSearchOptions options) { - return searchAsync(VectorSearchQuery.createQuery(searchText, options)); - } -} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java index 5b41477d..0cf1044a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java @@ -7,6 +7,9 @@ import java.util.concurrent.ConcurrentHashMap; import javax.annotation.Nonnull; +import com.microsoft.semantickernel.data.vectorstorage.VectorStore; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.exceptions.SKException; import reactor.core.publisher.Mono; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java index 8a9c3f02..d8b23d39 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java @@ -3,10 +3,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.record.options.DeleteRecordOptions; -import com.microsoft.semantickernel.data.record.options.GetRecordOptions; -import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import java.util.Collections; import java.util.HashSet; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java index a9990c60..64e95db6 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java @@ -1,7 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import javax.annotation.Nonnull; import javax.annotation.Nullable; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/BasicVectorSearchFilter.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/BasicVectorSearchFilter.java similarity index 89% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/BasicVectorSearchFilter.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/BasicVectorSearchFilter.java index a853ab19..642f4589 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/BasicVectorSearchFilter.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/BasicVectorSearchFilter.java @@ -1,11 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorsearch.filtering; +package com.microsoft.semantickernel.data.filtering; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; public class BasicVectorSearchFilter { @@ -30,7 +28,7 @@ public BasicVectorSearchFilter() { * @param filterClauses The filter clauses. */ public BasicVectorSearchFilter(List filterClauses) { - this.filterClauses = new ArrayList<>(filterClauses); + this.filterClauses = Collections.unmodifiableList(filterClauses); } /** @@ -39,7 +37,7 @@ public BasicVectorSearchFilter(List filterClauses) { * @return The filter clauses. */ public List getFilterClauses() { - return Collections.unmodifiableList(filterClauses); + return filterClauses; } public static Builder builder() { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/EqualityFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/EqualityFilterClause.java similarity index 90% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/EqualityFilterClause.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/EqualityFilterClause.java index 3580de42..ec6072ef 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/EqualityFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/EqualityFilterClause.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorsearch.filtering; +package com.microsoft.semantickernel.data.filtering; public abstract class EqualityFilterClause implements FilterClause { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/FilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/FilterClause.java similarity index 74% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/FilterClause.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/FilterClause.java index 6d45e339..a63e85dd 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/FilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/FilterClause.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorsearch.filtering; +package com.microsoft.semantickernel.data.filtering; public interface FilterClause { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/TagListContainsFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/TagListContainsFilterClause.java similarity index 90% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/TagListContainsFilterClause.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/TagListContainsFilterClause.java index 6dd54329..bcf2e64f 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/TagListContainsFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/TagListContainsFilterClause.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorsearch.filtering; +package com.microsoft.semantickernel.data.filtering; public abstract class TagListContainsFilterClause implements FilterClause { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearch.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearch.java new file mode 100644 index 00000000..288b7768 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearch.java @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch; + +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; +import reactor.core.publisher.Mono; + +import java.util.List; + +public interface VectorSearch { + + Mono>> searchAsync(VectorSearchQuery query); +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearchResult.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java similarity index 92% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearchResult.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java index 225fbf93..6e508669 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorSearchResult.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data; +package com.microsoft.semantickernel.data.vectorsearch; public class VectorSearchResult { private final Record record; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableSearch.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableSearch.java new file mode 100644 index 00000000..37847b75 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableSearch.java @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch; + +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import reactor.core.publisher.Mono; + +import java.util.List; + +public interface VectorizableSearch extends VectorSearch { + /** + * Vectorizable text search. This method searches for records that are similar to the given text. + * + * @param searchText The text to search with. + * @param options The options to use for the search. + * @return A list of search results. + */ + Mono>> searchAsync(String searchText, + VectorSearchOptions options); +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java new file mode 100644 index 00000000..51ab175a --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch; + +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import reactor.core.publisher.Mono; + +import java.util.List; + +public interface VectorizedSearch extends VectorSearch { + + /** + * Vectorized search. This method searches for records that are similar to the given vector. + * + * @param vector The vector to search with. + * @param options The options to use for the search. + * @return A list of search results. + */ + Mono>> searchAsync(List vector, + VectorSearchOptions options); +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/FilterClauseType.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/FilterClauseType.java deleted file mode 100644 index 9f597c81..00000000 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/filtering/FilterClauseType.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorsearch.filtering; - -public enum FilterClauseType { - /** - * An equality filter clause. - */ - EQUALITY, - /** - * A filter clause that checks if the tag list contains a value. - */ - TAG_LIST_CONTAINS -} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java index 83511ce1..f62a9362 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.vectorsearch.queries; -import com.microsoft.semantickernel.data.record.options.VectorSearchOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import java.util.List; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizableTextSearchQuery.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizableTextSearchQuery.java index 4373fb70..1df9b6ec 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizableTextSearchQuery.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizableTextSearchQuery.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.vectorsearch.queries; -import com.microsoft.semantickernel.data.record.options.VectorSearchOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import javax.annotation.Nullable; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java index aeb429a6..2a33c07d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.vectorsearch.queries; -import com.microsoft.semantickernel.data.record.options.VectorSearchOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import javax.annotation.Nullable; import java.util.Collections; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java similarity index 93% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStore.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java index 8928e2af..26e20c15 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data; +package com.microsoft.semantickernel.data.vectorstorage; import java.util.List; import javax.annotation.Nonnull; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java similarity index 90% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollection.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java index 04dde988..7efe19bb 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java @@ -1,9 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data; +package com.microsoft.semantickernel.data.vectorstorage; -import com.microsoft.semantickernel.data.record.options.DeleteRecordOptions; -import com.microsoft.semantickernel.data.record.options.GetRecordOptions; -import com.microsoft.semantickernel.data.record.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import java.util.List; import reactor.core.publisher.Mono; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java similarity index 76% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollectionOptions.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java index 93635be2..5782b7b0 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data; +package com.microsoft.semantickernel.data.vectorstorage; -import com.microsoft.semantickernel.data.record.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; public interface VectorStoreRecordCollectionOptions { /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java similarity index 97% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordMapper.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java index 100c7e6d..d77e9123 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data; +package com.microsoft.semantickernel.data.vectorstorage; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/attributes/VectorStoreRecordDataAttribute.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordDataAttribute.java similarity index 93% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/attributes/VectorStoreRecordDataAttribute.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordDataAttribute.java index 33ea5fcc..a5103105 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/attributes/VectorStoreRecordDataAttribute.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordDataAttribute.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.record.attributes; +package com.microsoft.semantickernel.data.vectorstorage.attributes; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/attributes/VectorStoreRecordKeyAttribute.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKeyAttribute.java similarity index 87% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/attributes/VectorStoreRecordKeyAttribute.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKeyAttribute.java index 3791abbd..8d54d60b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/attributes/VectorStoreRecordKeyAttribute.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKeyAttribute.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.record.attributes; +package com.microsoft.semantickernel.data.vectorstorage.attributes; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/attributes/VectorStoreRecordVectorAttribute.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java similarity index 92% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/attributes/VectorStoreRecordVectorAttribute.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java index ab1c5bd4..efe56285 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/attributes/VectorStoreRecordVectorAttribute.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.record.attributes; +package com.microsoft.semantickernel.data.vectorstorage.attributes; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/DistanceFunction.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java similarity index 93% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/DistanceFunction.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java index 13ed8ea8..66aa3e3a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/DistanceFunction.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.record.definition; +package com.microsoft.semantickernel.data.vectorstorage.definition; public enum DistanceFunction { COSINE_SIMILARITY("cosineSimilarity"), DOT_PRODUCT("dotProduct"), EUCLIDEAN("euclidean"); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/IndexKind.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java similarity index 92% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/IndexKind.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java index 33908f6e..fac43ea9 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/IndexKind.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.record.definition; +package com.microsoft.semantickernel.data.vectorstorage.definition; public enum IndexKind { HNSW("Hnsw"), FLAT("Flat"); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordDataField.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java similarity index 97% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordDataField.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java index 7aa21ba0..76a47187 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordDataField.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.record.definition; +package com.microsoft.semantickernel.data.vectorstorage.definition; import javax.annotation.Nonnull; import javax.annotation.Nullable; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordDefinition.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java similarity index 95% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordDefinition.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java index 5c16fe47..1acf0a13 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordDefinition.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java @@ -1,10 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.record.definition; +package com.microsoft.semantickernel.data.vectorstorage.definition; import com.fasterxml.jackson.annotation.JsonProperty; -import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; import java.lang.reflect.Field; import java.util.ArrayList; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordField.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java similarity index 97% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordField.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java index f80fffa1..9d85f2c9 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordField.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.record.definition; +package com.microsoft.semantickernel.data.vectorstorage.definition; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordKeyField.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java similarity index 94% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordKeyField.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java index 72236dd7..b255e561 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordKeyField.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.record.definition; +package com.microsoft.semantickernel.data.vectorstorage.definition; /** * Represents the key field in a record. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordVectorField.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java similarity index 98% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordVectorField.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java index e79b5a7e..47f47fd4 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/definition/VectorStoreRecordVectorField.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.record.definition; +package com.microsoft.semantickernel.data.vectorstorage.definition; import javax.annotation.Nonnull; import javax.annotation.Nullable; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/DeleteRecordOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java similarity index 66% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/DeleteRecordOptions.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java index a1405ec6..ed0dbb92 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/DeleteRecordOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.record.options; +package com.microsoft.semantickernel.data.vectorstorage.options; /** * Options for deleting a record. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/GetRecordOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java similarity index 95% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/GetRecordOptions.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java index c3ed0d8a..bff9d6a1 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/GetRecordOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.record.options; +package com.microsoft.semantickernel.data.vectorstorage.options; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/UpsertRecordOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java similarity index 66% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/UpsertRecordOptions.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java index 20105fb6..a0a9325b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/UpsertRecordOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.record.options; +package com.microsoft.semantickernel.data.vectorstorage.options; /** * Options for upserting a record. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/VectorSearchOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java similarity index 92% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/VectorSearchOptions.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java index 3ecb080b..67194bd8 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/record/options/VectorSearchOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java @@ -1,8 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.record.options; +package com.microsoft.semantickernel.data.vectorstorage.options; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.data.vectorsearch.filtering.BasicVectorSearchFilter; +import com.microsoft.semantickernel.data.filtering.BasicVectorSearchFilter; +import com.microsoft.semantickernel.exceptions.SKException; import javax.annotation.Nullable; @@ -133,6 +134,10 @@ public Builder withIncludeVectors(boolean includeVectors) { @Override public VectorSearchOptions build() { + if (limit < 1) { + throw new SKException("The limit must be greater than 0."); + } + return new VectorSearchOptions(basicVectorSearchFilter, vectorFieldName, limit, offset, includeVectors); } diff --git a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java index 3d7b5709..9da92923 100644 --- a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java +++ b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java @@ -1,9 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data; -import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.record.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; import java.util.List; From fed1621ae2a76ad3492a2d33d05ab1c5c56bddcb Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 10 Sep 2024 18:45:29 -0700 Subject: [PATCH 027/277] Format --- .../data/postgres/PostgreSQLVectorDistanceFunction.java | 2 +- .../connectors/data/postgres/PostgreSQLVectorIndexKind.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java index 7c0ba30e..e36104c6 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.postgres; -import com.microsoft.semantickernel.data.recorddefinition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; public enum PostgreSQLVectorDistanceFunction { L2("vector_l2_ops", "<->"), COSINE("vector_cosine_ops", "<=>"), INNER_PRODUCT("vector_ip_ops", diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java index 9983df0f..939caaed 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.postgres; -import com.microsoft.semantickernel.data.recorddefinition.IndexKind; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; public enum PostgreSQLVectorIndexKind { HNSW("hnsw"), IVFFLAT("ivfflat"); From 9a04ada877bcc0e67ff5d49da78c56065bb67315 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 10 Sep 2024 21:24:13 -0700 Subject: [PATCH 028/277] Update SQL query formatting in Vector Stores --- .../JDBCVectorStoreDefaultQueryProvider.java | 70 ++++++++++++------- .../mysql/MySQLVectorStoreQueryProvider.java | 14 ++-- .../PostgreSQLVectorStoreQueryProvider.java | 49 +++++++------ 3 files changed, 78 insertions(+), 55 deletions(-) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java index 7f2f077b..47a6b431 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java @@ -166,9 +166,9 @@ public Map, String> getSupportedVectorTypes() { */ @Override public void prepareVectorStore() { - String createCollectionsTable = "CREATE TABLE IF NOT EXISTS " - + validateSQLidentifier(collectionsTable) - + " (collectionId VARCHAR(255) PRIMARY KEY);"; + String createCollectionsTable = formatQuery( + "CREATE TABLE IF NOT EXISTS %s (collectionId VARCHAR(255) PRIMARY KEY);", + validateSQLidentifier(collectionsTable)); try (Connection connection = dataSource.getConnection(); PreparedStatement createTable = connection.prepareStatement(createCollectionsTable)) { @@ -207,8 +207,8 @@ public void validateSupportedTypes(VectorStoreRecordDefinition recordDefinition) */ @Override public boolean collectionExists(String collectionName) { - String query = "SELECT 1 FROM " + validateSQLidentifier(collectionsTable) - + " WHERE collectionId = ?"; + String query = formatQuery("SELECT 1 FROM %s WHERE collectionId = ?", + validateSQLidentifier(collectionsTable)); try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(query)) { @@ -232,18 +232,19 @@ public boolean collectionExists(String collectionName) { public void createCollection(String collectionName, VectorStoreRecordDefinition recordDefinition) { - String createStorageTable = "CREATE TABLE IF NOT EXISTS " - + getCollectionTableName(collectionName) + " (" - + getKeyColumnName(recordDefinition.getKeyField()) + " VARCHAR(255) PRIMARY KEY, " - + getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), - getSupportedDataTypes()) - + ", " - + getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getVectorFields()), - getSupportedVectorTypes()) - + ");"; + String createStorageTable = formatQuery("CREATE TABLE IF NOT EXISTS %s (" + + "%s VARCHAR(255) PRIMARY KEY, " + + "%s, " + + "%s);", + getCollectionTableName(collectionName), + getKeyColumnName(recordDefinition.getKeyField()), + getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), + getSupportedDataTypes()), + getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getVectorFields()), + getSupportedVectorTypes())); - String insertCollectionQuery = "INSERT INTO " + validateSQLidentifier(collectionsTable) - + " (collectionId) VALUES (?)"; + String insertCollectionQuery = formatQuery("INSERT INTO %s (collectionId) VALUES (?)", + validateSQLidentifier(collectionsTable)); try (Connection connection = dataSource.getConnection(); PreparedStatement createTable = connection.prepareStatement(createStorageTable)) { @@ -269,9 +270,10 @@ public void createCollection(String collectionName, */ @Override public void deleteCollection(String collectionName) { - String deleteCollectionOperation = "DELETE FROM " + validateSQLidentifier(collectionsTable) - + " WHERE collectionId = ?"; - String dropTableOperation = "DROP TABLE " + getCollectionTableName(collectionName); + String deleteCollectionOperation = formatQuery("DELETE FROM %s WHERE collectionId = ?", + validateSQLidentifier(collectionsTable)); + String dropTableOperation = formatQuery("DROP TABLE %s", + getCollectionTableName(collectionName)); try (Connection connection = dataSource.getConnection(); PreparedStatement deleteCollection = connection @@ -298,7 +300,8 @@ public void deleteCollection(String collectionName) { */ @Override public List getCollectionNames() { - String query = "SELECT collectionId FROM " + validateSQLidentifier(collectionsTable); + String query = formatQuery("SELECT collectionId FROM %s", + validateSQLidentifier(collectionsTable)); try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(query)) { @@ -339,10 +342,11 @@ public List getRecords(String collectionName, List keys fields = recordDefinition.getNonVectorFields(); } - String query = "SELECT " + getQueryColumnsFromFields(fields) - + " FROM " + getCollectionTableName(collectionName) - + " WHERE " + getKeyColumnName(recordDefinition.getKeyField()) - + " IN (" + getWildcardString(keys.size()) + ")"; + String query = formatQuery("SELECT %s FROM %s WHERE %s IN (%s)", + getQueryColumnsFromFields(fields), + getCollectionTableName(collectionName), + getKeyColumnName(recordDefinition.getKeyField()), + getWildcardString(keys.size())); try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(query)) { @@ -382,9 +386,10 @@ public void upsertRecords(String collectionName, List records, @Override public void deleteRecords(String collectionName, List keys, VectorStoreRecordDefinition recordDefinition, DeleteRecordOptions options) { - String query = "DELETE FROM " + getCollectionTableName(collectionName) - + " WHERE " + getKeyColumnName(recordDefinition.getKeyField()) - + " IN (" + getWildcardString(keys.size()) + ")"; + String query = formatQuery("DELETE FROM %s WHERE %s IN (%s)", + getCollectionTableName(collectionName), + getKeyColumnName(recordDefinition.getKeyField()), + getWildcardString(keys.size())); try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(query)) { @@ -412,6 +417,17 @@ public static String validateSQLidentifier(String identifier) { throw new SKException("Invalid SQL identifier: " + identifier); } + /** + * Formats a query. + * + * @param query the query + * @param args the arguments + * @return the formatted query + */ + public String formatQuery(String query, String... args) { + return String.format(query, (Object[]) args); + } + /** * The builder for {@link JDBCVectorStoreDefaultQueryProvider}. */ diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java index cd6977cd..d01d5c62 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java @@ -86,14 +86,16 @@ public void upsertRecords(String collectionName, List records, List fields = recordDefinition.getAllFields(); String onDuplicateKeyUpdate = fields.stream() - .map(field -> validateSQLidentifier(field.getEffectiveStorageName()) - + " = VALUES(" + validateSQLidentifier(field.getEffectiveStorageName()) + ")") + .map(field -> formatQuery("%s = VALUES(%s)", + validateSQLidentifier(field.getEffectiveStorageName()), + field.getEffectiveStorageName())) .collect(Collectors.joining(", ")); - String query = "INSERT INTO " + getCollectionTableName(collectionName) - + " (" + getQueryColumnsFromFields(fields) + ")" - + " VALUES (" + getWildcardString(fields.size()) + ")" - + " ON DUPLICATE KEY UPDATE " + onDuplicateKeyUpdate; + String query = formatQuery("INSERT INTO %s (%s) VALUES (%s) ON DUPLICATE KEY UPDATE %s", + getCollectionTableName(collectionName), + getQueryColumnsFromFields(fields), + getWildcardString(fields.size()), + onDuplicateKeyUpdate); try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(query)) { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index bcbcc0dc..428707b6 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -168,10 +168,12 @@ private String createIndexForVectorField(String collectionName, "Distance function is required for vector field: " + vectorField.getName()); } - return "CREATE INDEX IF NOT EXISTS " + getCollectionTableName(collectionName) + "_index" - + " ON " + getCollectionTableName(collectionName) - + " USING " + indexKind.getValue() - + " (" + vectorField.getName() + " " + distanceFunction.getValue() + ");"; + return formatQuery("CREATE INDEX IF NOT EXISTS %s ON %s USING %s (%s %s);", + getCollectionTableName(collectionName) + "_index", + getCollectionTableName(collectionName), + indexKind.getValue(), + vectorField.getName(), + distanceFunction.getValue()); } /** @@ -194,14 +196,15 @@ public void createCollection(String collectionName, try (Connection connection = dataSource.getConnection(); Statement createTableAndIndexes = connection.createStatement()) { - String createStorageTable = "CREATE TABLE IF NOT EXISTS " - + getCollectionTableName(collectionName) + " (" - + getKeyColumnName(recordDefinition.getKeyField()) + " VARCHAR(255) PRIMARY KEY, " - + getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), - supportedDataTypes) - + ", " - + getColumnNamesAndTypesForVectorFields(recordDefinition.getVectorFields()) - + ");"; + String createStorageTable = formatQuery("CREATE TABLE IF NOT EXISTS %s (" + + "%s VARCHAR(255) PRIMARY KEY, " + + "%s, " + + "%s);", + getCollectionTableName(collectionName), + getKeyColumnName(recordDefinition.getKeyField()), + getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), + supportedDataTypes), + getColumnNamesAndTypesForVectorFields(recordDefinition.getVectorFields())); createTableAndIndexes.addBatch(createStorageTable); for (VectorStoreRecordVectorField vectorField : vectorFields) { @@ -217,8 +220,8 @@ public void createCollection(String collectionName, throw new SKException("Failed to create collection", e); } - String insertCollectionQuery = "INSERT INTO " + validateSQLidentifier(collectionsTable) - + " (collectionId) VALUES (?)"; + String insertCollectionQuery = formatQuery("INSERT INTO %s (collectionId) VALUES (?)", + validateSQLidentifier(collectionsTable)); try (Connection connection = dataSource.getConnection(); PreparedStatement insert = connection.prepareStatement(insertCollectionQuery)) { @@ -284,16 +287,18 @@ public void upsertRecords(String collectionName, List records, String onDuplicateKeyUpdate = fields.stream() .filter(field -> !(field instanceof VectorStoreRecordKeyField)) // Exclude key fields - .map(field -> validateSQLidentifier(field.getEffectiveStorageName()) - + " = EXCLUDED." + validateSQLidentifier(field.getEffectiveStorageName())) + .map(field -> formatQuery("%s = EXCLUDED.%s", + validateSQLidentifier(field.getEffectiveStorageName()), + field.getEffectiveStorageName())) .collect(Collectors.joining(", ")); - String query = "INSERT INTO " + getCollectionTableName(collectionName) - + " (" + getQueryColumnsFromFields(fields) + ")" - + " VALUES (" + getWildcardStringWithCast(fields) + ")" - + " ON CONFLICT (" + getKeyColumnName(recordDefinition.getKeyField()) - + ") DO UPDATE SET " - + onDuplicateKeyUpdate; + String query = formatQuery( + "INSERT INTO %s (%s) VALUES (%s) ON CONFLICT (%s) DO UPDATE SET %s", + getCollectionTableName(collectionName), + getQueryColumnsFromFields(fields), + getWildcardStringWithCast(fields), + getKeyColumnName(recordDefinition.getKeyField()), + onDuplicateKeyUpdate); try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(query)) { From de36946d05df6e5e4b3367432da6e1ba293bc330 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Wed, 11 Sep 2024 08:25:04 -0700 Subject: [PATCH 029/277] Fix values --- .../data/vectorstorage/options/VectorSearchOptions.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java index 67194bd8..cfdb6908 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java @@ -34,8 +34,8 @@ public VectorSearchOptions(BasicVectorSearchFilter basicVectorSearchFilter, String vectorFieldName, int limit, int offset, boolean includeVectors) { this.basicVectorSearchFilter = basicVectorSearchFilter; this.vectorFieldName = vectorFieldName; - this.limit = limit; - this.offset = offset; + this.limit = Math.max(1, limit); + this.offset = Math.max(0, offset); this.includeVectors = includeVectors; } @@ -134,10 +134,6 @@ public Builder withIncludeVectors(boolean includeVectors) { @Override public VectorSearchOptions build() { - if (limit < 1) { - throw new SKException("The limit must be greater than 0."); - } - return new VectorSearchOptions(basicVectorSearchFilter, vectorFieldName, limit, offset, includeVectors); } From 96f1411fc64a4ed617eeccd4154b11db9ddaf923 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Wed, 11 Sep 2024 16:52:08 +0100 Subject: [PATCH 030/277] Add streaming support to chat completions --- .../chatcompletion/GeminiChatCompletion.java | 16 ++++ .../GeminiStreamingTextContent.java | 9 +- .../GeminiTextGenerationService.java | 20 ++-- .../HuggingFaceTextGenerationService.java | 2 +- .../chatcompletion/OpenAIChatCompletion.java | 88 ++++++++++++++++++ .../OpenAIStreamingChatMessageContent.java | 41 +++++++++ .../OpenAIStreamingTextContent.java | 34 +++++++ .../OpenAITextGenerationService.java | 4 +- .../Example67_KernelStreaming.java | 92 ++++++++++++++++++- .../services/KernelContent.java | 50 +--------- .../services/KernelContentImpl.java | 72 +++++++++++++++ .../services/StreamingKernelContent.java | 42 +-------- .../services/StreamingTextContent.java | 47 ++++++++++ .../chatcompletion/ChatCompletionService.java | 32 ++++++- .../chatcompletion/ChatMessageContent.java | 3 +- .../chatcompletion/StreamingChatContent.java | 9 ++ .../services/textcompletion/TextContent.java | 4 +- .../textcompletion/TextGenerationService.java | 1 + 18 files changed, 452 insertions(+), 114 deletions(-) rename semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textcompletion/StreamingTextContent.java => aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/textcompletion/GeminiStreamingTextContent.java (63%) create mode 100644 aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIStreamingChatMessageContent.java create mode 100644 aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAIStreamingTextContent.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/services/KernelContentImpl.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/services/StreamingTextContent.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/StreamingChatContent.java diff --git a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java index bb2c820b..10a14477 100644 --- a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java +++ b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java @@ -44,6 +44,7 @@ import java.util.UUID; import java.util.stream.Collectors; import javax.annotation.Nullable; +import com.microsoft.semantickernel.services.chatcompletion.StreamingChatContent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import reactor.core.publisher.Flux; @@ -75,6 +76,21 @@ public Mono>> getChatMessageContentsAsync(String prom invocationContext); } + @Override + public Flux> getStreamingChatMessageContentsAsync( + ChatHistory chatHistory, @Nullable Kernel kernel, + @Nullable InvocationContext invocationContext) { + throw new UnsupportedOperationException( + "Streaming chat message contents are not supported"); + } + + @Override + public Flux> getStreamingChatMessageContentsAsync(String prompt, + @Nullable Kernel kernel, @Nullable InvocationContext invocationContext) { + throw new UnsupportedOperationException( + "Streaming chat message contents are not supported"); + } + @Override public Mono>> getChatMessageContentsAsync(ChatHistory chatHistory, @Nullable Kernel kernel, @Nullable InvocationContext invocationContext) { diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textcompletion/StreamingTextContent.java b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/textcompletion/GeminiStreamingTextContent.java similarity index 63% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textcompletion/StreamingTextContent.java rename to aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/textcompletion/GeminiStreamingTextContent.java index 7fdadad7..9bf9a6fd 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textcompletion/StreamingTextContent.java +++ b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/textcompletion/GeminiStreamingTextContent.java @@ -1,13 +1,14 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.services.textcompletion; +package com.microsoft.semantickernel.aiservices.google.textcompletion; -import com.microsoft.semantickernel.services.StreamingKernelContent; +import com.microsoft.semantickernel.services.StreamingTextContent; +import com.microsoft.semantickernel.services.textcompletion.TextContent; import javax.annotation.Nullable; /** * StreamingTextContent is a wrapper for TextContent that allows for streaming. */ -public class StreamingTextContent extends StreamingKernelContent { +public class GeminiStreamingTextContent extends StreamingTextContent { /** * Initializes a new instance of the {@code StreamingTextContent} class with a provided text @@ -15,7 +16,7 @@ public class StreamingTextContent extends StreamingKernelContent { * * @param content The text content. */ - public StreamingTextContent(TextContent content) { + public GeminiStreamingTextContent(TextContent content) { super(content, 0, null, null); } diff --git a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/textcompletion/GeminiTextGenerationService.java b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/textcompletion/GeminiTextGenerationService.java index 4d7c2a25..beec7d2c 100644 --- a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/textcompletion/GeminiTextGenerationService.java +++ b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/textcompletion/GeminiTextGenerationService.java @@ -7,30 +7,29 @@ import com.google.cloud.vertexai.generativeai.GenerativeModel; import com.microsoft.semantickernel.Kernel; import com.microsoft.semantickernel.aiservices.google.GeminiService; +import com.microsoft.semantickernel.aiservices.google.GeminiServiceBuilder; import com.microsoft.semantickernel.aiservices.google.implementation.MonoConverter; import com.microsoft.semantickernel.exceptions.AIException; import com.microsoft.semantickernel.exceptions.SKCheckedException; import com.microsoft.semantickernel.exceptions.SKException; import com.microsoft.semantickernel.orchestration.FunctionResultMetadata; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; -import com.microsoft.semantickernel.aiservices.google.GeminiServiceBuilder; -import com.microsoft.semantickernel.services.textcompletion.StreamingTextContent; +import com.microsoft.semantickernel.services.StreamingTextContent; import com.microsoft.semantickernel.services.textcompletion.TextContent; import com.microsoft.semantickernel.services.textcompletion.TextGenerationService; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Nullable; import java.io.IOException; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.List; import java.util.UUID; +import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; public class GeminiTextGenerationService extends GeminiService implements TextGenerationService { + private static final Logger LOGGER = LoggerFactory.getLogger(GeminiTextGenerationService.class); public GeminiTextGenerationService(VertexAI client, String modelId) { @@ -57,7 +56,7 @@ public Flux getStreamingTextContentsAsync( return this .internalGetTextAsync(prompt, executionSettings) .flatMapMany(it -> Flux.fromStream(it.stream()) - .map(StreamingTextContent::new)); + .map(GeminiStreamingTextContent::new)); } private Mono> internalGetTextAsync(String prompt, @@ -124,6 +123,7 @@ private GenerativeModel getGenerativeModel( public static class Builder extends GeminiServiceBuilder { + @Override public GeminiTextGenerationService build() { if (this.client == null) { diff --git a/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFaceTextGenerationService.java b/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFaceTextGenerationService.java index 96e05ff9..0f5260d3 100644 --- a/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFaceTextGenerationService.java +++ b/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFaceTextGenerationService.java @@ -9,7 +9,7 @@ import com.microsoft.semantickernel.exceptions.SKException; import com.microsoft.semantickernel.orchestration.FunctionResultMetadata; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; -import com.microsoft.semantickernel.services.textcompletion.StreamingTextContent; +import com.microsoft.semantickernel.services.StreamingTextContent; import com.microsoft.semantickernel.services.textcompletion.TextContent; import com.microsoft.semantickernel.services.textcompletion.TextGenerationService; import java.util.List; diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java index 5442e51e..76192d61 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java @@ -56,6 +56,7 @@ import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; import com.microsoft.semantickernel.services.chatcompletion.ChatHistory; import com.microsoft.semantickernel.services.chatcompletion.ChatMessageContent; +import com.microsoft.semantickernel.services.chatcompletion.StreamingChatContent; import com.microsoft.semantickernel.services.chatcompletion.message.ChatMessageContentType; import com.microsoft.semantickernel.services.chatcompletion.message.ChatMessageImageContent; import com.microsoft.semantickernel.services.openai.OpenAiServiceBuilder; @@ -64,6 +65,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; @@ -179,6 +181,92 @@ public Mono>> getChatMessageContentsAsync( }); } + @Override + public Flux> getStreamingChatMessageContentsAsync( + ChatHistory chatHistory, + @Nullable Kernel kernel, + @Nullable InvocationContext invocationContext) { + if (invocationContext != null && invocationContext.getToolCallBehavior() + .isAutoInvokeAllowed()) { + throw new SKException( + "Auto invoke is not supported for streaming chat message contents"); + } + + if (invocationContext != null + && invocationContext.returnMode() != InvocationReturnMode.NEW_MESSAGES_ONLY) { + throw new SKException( + "Streaming chat message contents only supports NEW_MESSAGES_ONLY return mode"); + } + + List chatRequestMessages = getChatRequestMessages(chatHistory); + + ChatMessages messages = new ChatMessages(chatRequestMessages); + + List functions = new ArrayList<>(); + if (kernel != null) { + kernel.getPlugins() + .forEach(plugin -> plugin.getFunctions().forEach((name, function) -> functions + .add(OpenAIFunction.build(function.getMetadata(), plugin.getName())))); + } + + ChatCompletionsOptions options = executeHook( + invocationContext, + kernel, + new PreChatCompletionEvent( + getCompletionsOptions( + this, + messages.allMessages, + functions, + invocationContext))) + .getOptions(); + + return getClient() + .getChatCompletionsStreamWithResponse( + getDeploymentName(), + options, + OpenAIRequestSettings.getRequestOptions()) + .flatMap(completionsResult -> { + if (completionsResult.getStatusCode() >= 400) { + //SemanticKernelTelemetry.endSpanWithError(span); + return Mono.error(new AIException(ErrorCodes.SERVICE_ERROR, + "Request failed: " + completionsResult.getStatusCode())); + } + //SemanticKernelTelemetry.endSpanWithUsage(span, completionsResult.getValue().getUsage()); + + return Mono.just(completionsResult.getValue()); + }) + .flatMap(completions -> { + return Flux.fromIterable(completions.getChoices()) + .map(message -> { + AuthorRole role = message.getDelta().getRole() == null + ? AuthorRole.ASSISTANT + : AuthorRole.valueOf(message.getDelta().getRole().toString() + .toUpperCase(Locale.ROOT)); + + return new OpenAIStreamingChatMessageContent<>( + completions.getId(), + role, + message.getDelta().getContent(), + getModelId(), + null, + null, + null, + Arrays.asList()); + }); + }); + } + + @Override + public Flux> getStreamingChatMessageContentsAsync( + String prompt, + @Nullable Kernel kernel, + @Nullable InvocationContext invocationContext) { + return getStreamingChatMessageContentsAsync( + new ChatHistory().addUserMessage(prompt), + kernel, + invocationContext); + } + // Holds messages temporarily as we build up our result private static class ChatMessages { diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIStreamingChatMessageContent.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIStreamingChatMessageContent.java new file mode 100644 index 00000000..21162c8f --- /dev/null +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIStreamingChatMessageContent.java @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.aiservices.openai.chatcompletion; + +import com.microsoft.semantickernel.orchestration.FunctionResultMetadata; +import com.microsoft.semantickernel.services.chatcompletion.AuthorRole; +import com.microsoft.semantickernel.services.chatcompletion.StreamingChatContent; +import java.nio.charset.Charset; +import java.util.List; +import javax.annotation.Nullable; + +public class OpenAIStreamingChatMessageContent extends OpenAIChatMessageContent implements + StreamingChatContent { + + private final String id; + + public OpenAIStreamingChatMessageContent( + String id, + AuthorRole authorRole, + String content, + @Nullable String modelId, + @Nullable T innerContent, + @Nullable Charset encoding, + @Nullable FunctionResultMetadata metadata, + @Nullable List toolCall) { + super( + authorRole, + content, + modelId, + innerContent, + encoding, + metadata, + toolCall); + + this.id = id; + } + + @Override + public String getId() { + return id; + } +} diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAIStreamingTextContent.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAIStreamingTextContent.java new file mode 100644 index 00000000..13272fb5 --- /dev/null +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAIStreamingTextContent.java @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.aiservices.openai.textcompletion; + +import com.microsoft.semantickernel.services.StreamingTextContent; +import com.microsoft.semantickernel.services.textcompletion.TextContent; + +import javax.annotation.Nullable; + +/** + * StreamingTextContent is a wrapper for TextContent that allows for streaming. + */ +public class OpenAIStreamingTextContent extends StreamingTextContent { + + /** + * Initializes a new instance of the {@code StreamingTextContent} class with a provided text + * content. + * + * @param content The text content. + */ + public OpenAIStreamingTextContent(TextContent content) { + super(content, 0, null, null); + } + + @Override + @Nullable + public String getContent() { + TextContent content = getInnerContent(); + if (content == null) { + return null; + } + return content.getContent(); + } + +} diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAITextGenerationService.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAITextGenerationService.java index 13783229..864c5f71 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAITextGenerationService.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAITextGenerationService.java @@ -11,7 +11,7 @@ import com.microsoft.semantickernel.exceptions.AIException.ErrorCodes; import com.microsoft.semantickernel.orchestration.FunctionResultMetadata; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; -import com.microsoft.semantickernel.services.textcompletion.StreamingTextContent; +import com.microsoft.semantickernel.services.StreamingTextContent; import com.microsoft.semantickernel.services.textcompletion.TextContent; import com.microsoft.semantickernel.services.textcompletion.TextGenerationService; import com.microsoft.semantickernel.implementation.telemetry.SemanticKernelTelemetry; @@ -75,7 +75,7 @@ public Flux getStreamingTextContentsAsync( return this .internalCompleteTextAsync(prompt, executionSettings) .flatMapMany(it -> Flux.fromStream(it.stream()) - .map(StreamingTextContent::new)); + .map(OpenAIStreamingTextContent::new)); } protected Mono> internalCompleteTextAsync( diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example67_KernelStreaming.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example67_KernelStreaming.java index 36a0a424..f6f68681 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example67_KernelStreaming.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example67_KernelStreaming.java @@ -1,10 +1,94 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.samples.syntaxexamples; -/* - * Currently the Java Semantic Kernel does not support streaming. If this is a feature you would like to see, please - * let us know by creating an issue on github or letting us know on Discord. - */ +import com.azure.ai.openai.OpenAIAsyncClient; +import com.azure.ai.openai.OpenAIClientBuilder; +import com.azure.core.credential.AzureKeyCredential; +import com.azure.core.credential.KeyCredential; +import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIChatCompletion; +import com.microsoft.semantickernel.services.chatcompletion.AuthorRole; +import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; +import com.microsoft.semantickernel.services.chatcompletion.ChatHistory; +import com.microsoft.semantickernel.services.chatcompletion.StreamingChatContent; +import java.util.concurrent.CountDownLatch; + public class Example67_KernelStreaming { + private static final String CLIENT_KEY = System.getenv("CLIENT_KEY"); + private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY"); + + // Only required if AZURE_CLIENT_KEY is set + private static final String CLIENT_ENDPOINT = System.getenv("CLIENT_ENDPOINT"); + private static final String MODEL_ID = System.getenv() + .getOrDefault("MODEL_ID", "gpt-35-turbo"); + + public static void main(String[] args) throws InterruptedException { + System.out.println("======== Open AI - ChatGPT Streaming ========"); + + OpenAIAsyncClient client; + + if (AZURE_CLIENT_KEY != null) { + client = new OpenAIClientBuilder() + .credential(new AzureKeyCredential(AZURE_CLIENT_KEY)) + .endpoint(CLIENT_ENDPOINT) + .buildAsyncClient(); + + } else { + client = new OpenAIClientBuilder() + .credential(new KeyCredential(CLIENT_KEY)) + .buildAsyncClient(); + } + + ChatCompletionService chatGPT = OpenAIChatCompletion.builder() + .withModelId(MODEL_ID) + .withOpenAIAsyncClient(client) + .build(); + + System.out.println("Chat content:"); + System.out.println("------------------------"); + + ChatHistory chatHistory = new ChatHistory("You are a librarian, expert about books"); + + // First user message + chatHistory.addUserMessage("Hi, I'm looking for book suggestions"); + messageOutput(chatHistory); + + GPTReply(chatGPT, chatHistory); + + chatHistory.addUserMessage( + "I love history and philosophy, I'd like to learn something new about Greece, any suggestion"); + messageOutput(chatHistory); + + GPTReply(chatGPT, chatHistory); + } + + private static void messageOutput(ChatHistory chatHistory) { + var message = chatHistory.getLastMessage().get(); + System.out.println(message.getAuthorRole() + ": " + message.getContent()); + System.out.println("------------------------"); + } + + private static void GPTReply(ChatCompletionService chatGPT, ChatHistory chatHistory) + throws InterruptedException { + var reply = chatGPT.getStreamingChatMessageContentsAsync(chatHistory, null, null); + System.out.print(AuthorRole.ASSISTANT + ": "); + + CountDownLatch cdl = new CountDownLatch(1); + reply + .groupBy(StreamingChatContent::getId) + .doOnComplete(() -> { + cdl.countDown(); + }) + .subscribe(group -> { + group + .filter(msg -> msg.getContent() != null) + .subscribe( + msg -> { + System.out.print(msg.getContent()); + }); + }); + + cdl.await(); + } + } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/KernelContent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/KernelContent.java index e55f2cdc..2a7ddf00 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/KernelContent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/KernelContent.java @@ -9,7 +9,7 @@ * * @param The type of the content. */ -public abstract class KernelContent { +public interface KernelContent { /* * The inner content representation. Use this to bypass the current @@ -17,55 +17,13 @@ public abstract class KernelContent { * Use it only if strictly necessary. */ @Nullable - private final T innerContent; - - /** - * The model ID used to generate the content. - */ - @Nullable - private final String modelId; + T getInnerContent(); /** * The metadata associated with the content. */ @Nullable - private final FunctionResultMetadata metadata; - - /** - * Initializes a new instance of the {@link KernelContent} class. - * - * @param innerContent The inner content representation. - * @param modelId The model identifier used to generate the content. - * @param metadata The metadata associated with the content. - */ - public KernelContent( - @Nullable T innerContent, - @Nullable String modelId, - @Nullable FunctionResultMetadata metadata) { - this.innerContent = innerContent; - this.modelId = modelId; - this.metadata = metadata; - } - - /** - * Gets the inner content representation. - * - * @return The inner content representation. - */ - @Nullable - public T getInnerContent() { - return innerContent; - } - - /** - * Gets the metadata associated with the content. - * - * @return The metadata associated with the content. - */ - @Nullable - public FunctionResultMetadata getMetadata() { - return metadata; - } + FunctionResultMetadata getMetadata(); /** * Gets the content returned by the AI service. @@ -73,5 +31,5 @@ public FunctionResultMetadata getMetadata() { * @return The content. */ @Nullable - public abstract String getContent(); + String getContent(); } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/KernelContentImpl.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/KernelContentImpl.java new file mode 100644 index 00000000..48bc6d77 --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/KernelContentImpl.java @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.services; + +import com.microsoft.semantickernel.orchestration.FunctionResultMetadata; +import javax.annotation.Nullable; + +/** + * Base class which represents the content returned by an AI service. + * + * @param The type of the content. + */ +public abstract class KernelContentImpl implements KernelContent { + + /* + * The inner content representation. Use this to bypass the current + * abstraction. The usage of this property is considered "unsafe". + * Use it only if strictly necessary. + */ + @Nullable + private final T innerContent; + + /** + * The model ID used to generate the content. + */ + @Nullable + private final String modelId; + + /** + * The metadata associated with the content. + */ + @Nullable + private final FunctionResultMetadata metadata; + + /** + * Initializes a new instance of the {@link KernelContentImpl} class. + * + * @param innerContent The inner content representation. + * @param modelId The model identifier used to generate the content. + * @param metadata The metadata associated with the content. + */ + public KernelContentImpl( + @Nullable T innerContent, + @Nullable String modelId, + @Nullable FunctionResultMetadata metadata) { + this.innerContent = innerContent; + this.modelId = modelId; + this.metadata = metadata; + } + + /** + * Gets the inner content representation. + * + * @return The inner content representation. + */ + @Nullable + @Override + public T getInnerContent() { + return innerContent; + } + + /** + * Gets the metadata associated with the content. + * + * @return The metadata associated with the content. + */ + @Nullable + @Override + public FunctionResultMetadata getMetadata() { + return metadata; + } + +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/StreamingKernelContent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/StreamingKernelContent.java index 6c1ddab1..02d9d737 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/StreamingKernelContent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/StreamingKernelContent.java @@ -1,46 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.services; -import com.microsoft.semantickernel.contextvariables.ContextVariable; -import java.util.Map; -import javax.annotation.Nullable; +public interface StreamingKernelContent extends KernelContent { -/** - * Base class which represents the content returned by a streaming AI service. - * - * @param The type of the content. - */ -public abstract class StreamingKernelContent extends KernelContent { - - /** - * In a scenario of multiple choices per request, this represents the zero-based index of the - * choice in the streaming sequence - */ - private final int choiceIndex; - - /** - * Initializes a new instance of the {@link StreamingKernelContent} class. - * - * @param innerContent The inner content representation. - * @param choiceIndex The zero-based index of the choice in the streaming sequence. - * @param modelId The model identifier used to generate the content. - * @param metadata The metadata associated with the content. - */ - protected StreamingKernelContent( - @Nullable T innerContent, - int choiceIndex, - @Nullable String modelId, - @Nullable Map> metadata) { - super(innerContent, modelId, null); - this.choiceIndex = choiceIndex; - } - - /** - * Gets the zero-based index of the choice in the streaming sequence. - * - * @return The zero-based index of the choice in the streaming sequence. - */ - public int getChoiceIndex() { - return choiceIndex; - } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/StreamingTextContent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/StreamingTextContent.java new file mode 100644 index 00000000..95e78449 --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/StreamingTextContent.java @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.services; + +import com.microsoft.semantickernel.contextvariables.ContextVariable; +import java.util.Map; +import javax.annotation.Nullable; + +/** + * Base class which represents the content returned by a streaming AI service. + * + * @param The type of the content. + */ +public abstract class StreamingTextContent extends KernelContentImpl implements + StreamingKernelContent { + + /** + * In a scenario of multiple choices per request, this represents the zero-based index of the + * choice in the streaming sequence + */ + private final int choiceIndex; + + /** + * Initializes a new instance of the {@link StreamingTextContent} class. + * + * @param innerContent The inner content representation. + * @param choiceIndex The zero-based index of the choice in the streaming sequence. + * @param modelId The model identifier used to generate the content. + * @param metadata The metadata associated with the content. + */ + protected StreamingTextContent( + @Nullable T innerContent, + int choiceIndex, + @Nullable String modelId, + @Nullable Map> metadata) { + super(innerContent, modelId, null); + this.choiceIndex = choiceIndex; + } + + /** + * Gets the zero-based index of the choice in the streaming sequence. + * + * @return The zero-based index of the choice in the streaming sequence. + */ + public int getChoiceIndex() { + return choiceIndex; + } +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatCompletionService.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatCompletionService.java index 0e566f3d..d1655c05 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatCompletionService.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatCompletionService.java @@ -2,13 +2,11 @@ package com.microsoft.semantickernel.services.chatcompletion; import com.microsoft.semantickernel.Kernel; -import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.implementation.ServiceLoadUtil; import com.microsoft.semantickernel.orchestration.InvocationContext; import com.microsoft.semantickernel.services.TextAIService; -import com.microsoft.semantickernel.services.openai.OpenAiServiceBuilder; import java.util.List; import javax.annotation.Nullable; +import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; /** @@ -42,4 +40,32 @@ Mono>> getChatMessageContentsAsync( String prompt, @Nullable Kernel kernel, @Nullable InvocationContext invocationContext); + + /** + * Gets the chat message contents asynchronously using {@code ChatHistory} to support a + * turn-based conversation. Typically, the resulting chat message contents is appended to the + * {@code chatHistory} to continue the conversation. + * + * @param chatHistory the chat history + * @param kernel the kernel + * @param invocationContext the invocation context + * @return the chat message contents + */ + Flux> getStreamingChatMessageContentsAsync( + ChatHistory chatHistory, + @Nullable Kernel kernel, + @Nullable InvocationContext invocationContext); + + /** + * Gets the chat message contents asynchronously using a prompt. + * + * @param prompt the prompt + * @param kernel the kernel + * @param invocationContext the invocation context + * @return the chat message contents + */ + Flux> getStreamingChatMessageContentsAsync( + String prompt, + @Nullable Kernel kernel, + @Nullable InvocationContext invocationContext); } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java index 21535791..9784648b 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java @@ -3,6 +3,7 @@ import com.microsoft.semantickernel.orchestration.FunctionResultMetadata; import com.microsoft.semantickernel.services.KernelContent; +import com.microsoft.semantickernel.services.KernelContentImpl; import com.microsoft.semantickernel.services.chatcompletion.message.ChatMessageContentType; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -21,7 +22,7 @@ * * @param the type of the inner content within the messages */ -public class ChatMessageContent extends KernelContent { +public class ChatMessageContent extends KernelContentImpl { private final AuthorRole authorRole; @Nullable diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/StreamingChatContent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/StreamingChatContent.java new file mode 100644 index 00000000..46182913 --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/StreamingChatContent.java @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.services.chatcompletion; + +import com.microsoft.semantickernel.services.StreamingKernelContent; + +public interface StreamingChatContent extends StreamingKernelContent { + + public String getId(); +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textcompletion/TextContent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textcompletion/TextContent.java index 50295a7c..3cca884d 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textcompletion/TextContent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textcompletion/TextContent.java @@ -2,13 +2,13 @@ package com.microsoft.semantickernel.services.textcompletion; import com.microsoft.semantickernel.orchestration.FunctionResultMetadata; -import com.microsoft.semantickernel.services.KernelContent; +import com.microsoft.semantickernel.services.KernelContentImpl; import javax.annotation.Nullable; /** * Content from a text completion service. */ -public class TextContent extends KernelContent { +public class TextContent extends KernelContentImpl { private final String content; diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textcompletion/TextGenerationService.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textcompletion/TextGenerationService.java index 0ab08f5f..39fdd514 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textcompletion/TextGenerationService.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textcompletion/TextGenerationService.java @@ -5,6 +5,7 @@ import com.microsoft.semantickernel.Kernel; import com.microsoft.semantickernel.implementation.ServiceLoadUtil; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; +import com.microsoft.semantickernel.services.StreamingTextContent; import com.microsoft.semantickernel.services.TextAIService; import com.microsoft.semantickernel.services.openai.OpenAiServiceBuilder; import java.util.List; From 0804814a2c9fdd33c3fedffd9c3e875681654a70 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 12 Sep 2024 01:42:31 -0700 Subject: [PATCH 031/277] Add Postgres Vector Search --- .../tests/connectors/memory/Hotel.java | 18 +- .../JDBCVectorStoreRecordCollectionTest.java | 100 +++- .../memory/jdbc/JDBCVectorStoreTest.java | 4 +- ...ashSetVectorStoreRecordCollectionTest.java | 14 +- ...isJsonVectorStoreRecordCollectionTest.java | 14 +- ...reAISearchVectorStoreRecordCollection.java | 16 +- .../connectors/data/jdbc/JDBCVectorStore.java | 4 +- .../JDBCVectorStoreDefaultQueryProvider.java | 481 ------------------ .../data/jdbc/JDBCVectorStoreOptions.java | 10 +- .../jdbc/JDBCVectorStoreQueryProvider.java | 432 ++++++++++++++-- .../jdbc/JDBCVectorStoreRecordCollection.java | 33 +- ...DBCVectorStoreRecordCollectionOptions.java | 16 +- .../jdbc/JDBCVectorStoreRecordMapper.java | 10 +- .../jdbc/SQLVectorStoreQueryProvider.java | 149 ++++++ .../mysql/MySQLVectorStoreQueryProvider.java | 6 +- .../PostgreSQLVectorStoreQueryProvider.java | 93 +++- .../PostgreSQLVectorStoreRecordMapper.java | 9 +- ...disHashSetVectorStoreRecordCollection.java | 13 +- .../RedisHashSetVectorStoreRecordMapper.java | 38 +- .../RedisJsonVectorStoreRecordCollection.java | 13 +- .../RedisJsonVectorStoreRecordMapper.java | 6 +- .../queries/VectorSearchQuery.java | 6 +- .../VectorStoreRecordMapper.java | 13 +- .../VectorStoreRecordDefinition.java | 49 +- .../options/GetRecordOptions.java | 2 +- 25 files changed, 908 insertions(+), 641 deletions(-) delete mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java index 6cb83041..a733f376 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java @@ -19,15 +19,16 @@ public class Hotel { @VectorStoreRecordDataAttribute() private final String description; @JsonProperty("summaryEmbedding") - @VectorStoreRecordVectorAttribute(dimensions = 3) + @VectorStoreRecordVectorAttribute(dimensions = 8) private final List descriptionEmbedding; - @VectorStoreRecordVectorAttribute(dimensions = 3, indexKind = "hnsw", distanceFunction = "cosine") - private final List additionalEmbedding; + @JsonProperty("indexedSummaryEmbedding") + @VectorStoreRecordVectorAttribute(dimensions = 8, indexKind = "hnsw", distanceFunction = "cosine") + private final List indexedDescriptionEmbedding; @VectorStoreRecordDataAttribute private double rating; public Hotel() { - this(null, null, 0, null, null, 0.0); + this(null, null, 0, null, null, null, 0.0); } @JsonCreator @@ -36,14 +37,15 @@ public Hotel( @JsonProperty("name") String name, @JsonProperty("code") int code, @JsonProperty("summary") String description, - @JsonProperty("summaryVector") List descriptionEmbedding, + @JsonProperty("summaryEmbedding") List descriptionEmbedding, + @JsonProperty("indexedSummaryEmbedding") List indexedDescriptionEmbedding, @JsonProperty("rating") double rating) { this.id = id; this.name = name; this.code = code; this.description = description; this.descriptionEmbedding = descriptionEmbedding; - this.additionalEmbedding = descriptionEmbedding; + this.indexedDescriptionEmbedding = indexedDescriptionEmbedding; this.rating = rating; } @@ -66,8 +68,8 @@ public String getDescription() { public List getDescriptionEmbedding() { return descriptionEmbedding; } - public List getAdditionalEmbedding() { - return additionalEmbedding; + public List getIndexedDescriptionEmbedding() { + return indexedDescriptionEmbedding; } public double getRating() { diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java index cb1bb2de..75f26c17 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -1,13 +1,16 @@ package com.microsoft.semantickernel.tests.connectors.memory.jdbc; -import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import com.microsoft.semantickernel.tests.connectors.memory.Hotel; import com.mysql.cj.jdbc.MysqlDataSource; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.postgresql.ds.PGSimpleDataSource; @@ -44,7 +47,7 @@ public enum QueryProvider { } private JDBCVectorStoreRecordCollection buildRecordCollection(QueryProvider provider, @Nonnull String collectionName) { - JDBCVectorStoreQueryProvider queryProvider; + SQLVectorStoreQueryProvider queryProvider; DataSource dataSource; switch (provider) { @@ -93,12 +96,19 @@ public void buildRecordCollection(QueryProvider provider) { } private List getHotels() { + ArrayList embeddings = new ArrayList<>(); + return List.of( - new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(1.0f, 2.0f, 3.0f), 4.0), - new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(1.0f, 2.0f, 3.0f), 3.0), - new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(1.0f, 2.0f, 3.0f), 5.0), - new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(1.0f, 2.0f, 3.0f), 4.0), - new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(1.0f, 2.0f, 3.0f), 5.0) + new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), + Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f),4.0), + new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), + Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f),3.0), + new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), + Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f),5.0), + new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), + Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f),4.0), + new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), + Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f),5.0) ); } @@ -306,4 +316,80 @@ public void getBatchWithNoVectors(QueryProvider provider) { assertNotNull(hotel.getDescriptionEmbedding()); } } + + + @Test + public void postgresExactSearch() { + String collectionName = "search"; + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(QueryProvider.PostgreSQL, collectionName); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + // Embeddings similar to the third hotel + List embeddings = Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.2f); + List> results = recordCollection.searchAsync(embeddings, null).block(); + assertNotNull(results); + assertEquals(3, results.size()); + // The third hotel should be the most similar + assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + + // Skip the first result + results = recordCollection.searchAsync(embeddings, VectorSearchOptions.builder().withOffset(1).withLimit(-100).build()).block(); + assertNotNull(results); + assertEquals(1, results.size()); + // The first hotel should be the most similar + assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); + } + + @Test + public void postgresApproximateSearch() { + String collectionName = "searchWithIndex"; + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(QueryProvider.PostgreSQL, collectionName); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions options = VectorSearchOptions.builder() + .withVectorFieldName("indexedDescriptionEmbedding") + .withLimit(5) + .build(); + + // Embeddings similar to the third hotel + List embeddings = Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.2f); + List> results = recordCollection.searchAsync(embeddings, options).block(); + assertNotNull(results); + assertEquals(5, results.size()); + // The third hotel should be the most similar + assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + } + + @Test + public void searchIncludeAndNotIncludeVectors() { + String collectionName = "searchIncludeAndNotIncludeVectors"; + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(QueryProvider.PostgreSQL, collectionName); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + // Embeddings similar to the third hotel + List embeddings = Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.2f); + List> results = recordCollection.searchAsync(embeddings, null).block(); + assertNotNull(results); + assertEquals(3, results.size()); + // The third hotel should be the most similar + assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + assertNull(results.get(0).getRecord().getDescriptionEmbedding()); + + VectorSearchOptions options = VectorSearchOptions.builder() + .withIncludeVectors(true) + .build(); + + results = recordCollection.searchAsync(embeddings, options).block(); + assertNotNull(results); + assertEquals(3, results.size()); + // The third hotel should be the most similar + assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + assertNotNull(results.get(0).getRecord().getDescriptionEmbedding()); + } } diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java index 667aa3bd..fe5ec1b9 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java @@ -2,7 +2,7 @@ import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStore; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions; -import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; @@ -36,7 +36,7 @@ public class JDBCVectorStoreTest { private static final PostgreSQLContainer POSTGRESQL_CONTAINER = new PostgreSQLContainer<>(PGVECTOR); private JDBCVectorStore buildVectorStore(QueryProvider provider) { - JDBCVectorStoreQueryProvider queryProvider; + SQLVectorStoreQueryProvider queryProvider; DataSource dataSource; switch (provider) { diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index 24945df6..e812651c 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -97,11 +97,11 @@ private RedisHashSetVectorStoreRecordCollection buildrecordCollection(@No private List getHotels() { return List.of( - new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(1.0f, 2.0f, 3.0f), 4.0), - new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(1.0f, 2.0f, 3.0f), 3.0), - new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(1.0f, 2.0f, 3.0f), 5.0), - new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(1.0f, 2.0f, 3.0f), 4.0), - new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(1.0f, 2.0f, 3.0f), 5.0) + new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(1.0f, 2.0f, 3.0f), Arrays.asList(1.0f, 2.0f, 3.0f),4.0), + new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(1.0f, 2.0f, 3.0f), Arrays.asList(1.0f, 2.0f, 3.0f),3.0), + new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(1.0f, 2.0f, 3.0f), Arrays.asList(1.0f, 2.0f, 3.0f),5.0), + new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(1.0f, 2.0f, 3.0f), Arrays.asList(1.0f, 2.0f, 3.0f),4.0), + new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(1.0f, 2.0f, 3.0f), Arrays.asList(1.0f, 2.0f, 3.0f),5.0) ); } @@ -233,7 +233,7 @@ public void getAsyncWithVectors(RecordCollectionOptions options) { recordCollection.upsertBatchAsync(hotels, null).block(); for (Hotel hotel : hotels) { - Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block(); + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), new GetRecordOptions(true)).block(); assertNotNull(retrievedHotel); assertNotNull(retrievedHotel.getDescriptionEmbedding()); assertEquals(hotel.getId(), retrievedHotel.getId()); @@ -252,7 +252,7 @@ public void getBatchAsyncWithVectors(RecordCollectionOptions options) { List ids = new ArrayList<>(); hotels.forEach(hotel -> ids.add(hotel.getId())); - List retrievedHotels = recordCollection.getBatchAsync(ids, null).block(); + List retrievedHotels = recordCollection.getBatchAsync(ids, new GetRecordOptions(true)).block(); assertNotNull(retrievedHotels); assertEquals(hotels.size(), retrievedHotels.size()); diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index 9d833217..aa5fd43d 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -98,11 +98,11 @@ private RedisJsonVectorStoreRecordCollection buildrecordCollection(@Nonnu private List getHotels() { return List.of( - new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(1.0f, 2.0f, 3.0f), 4.0), - new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(1.0f, 2.0f, 3.0f), 3.0), - new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(1.0f, 2.0f, 3.0f), 5.0), - new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(1.0f, 2.0f, 3.0f), 4.0), - new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(1.0f, 2.0f, 3.0f), 5.0) + new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(1.0f, 2.0f, 3.0f), Arrays.asList(1.0f, 2.0f, 3.0f),4.0), + new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(1.0f, 2.0f, 3.0f), Arrays.asList(1.0f, 2.0f, 3.0f),3.0), + new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(1.0f, 2.0f, 3.0f), Arrays.asList(1.0f, 2.0f, 3.0f),5.0), + new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(1.0f, 2.0f, 3.0f), Arrays.asList(1.0f, 2.0f, 3.0f),4.0), + new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(1.0f, 2.0f, 3.0f), Arrays.asList(1.0f, 2.0f, 3.0f),5.0) ); } @@ -234,7 +234,7 @@ public void getAsyncWithVectors(RecordCollectionOptions options) { recordCollection.upsertBatchAsync(hotels, null).block(); for (Hotel hotel : hotels) { - Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block(); + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), new GetRecordOptions(true)).block(); assertNotNull(retrievedHotel); assertNotNull(retrievedHotel.getDescriptionEmbedding()); assertEquals(hotel.getId(), retrievedHotel.getId()); @@ -253,7 +253,7 @@ public void getBatchAsyncWithVectors(RecordCollectionOptions options) { List ids = new ArrayList<>(); hotels.forEach(hotel -> ids.add(hotel.getId())); - List retrievedHotels = recordCollection.getBatchAsync(ids, null).block(); + List retrievedHotels = recordCollection.getBatchAsync(ids, new GetRecordOptions(true)).block(); assertNotNull(retrievedHotels); assertEquals(hotels.size(), retrievedHotels.size()); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index e4632a36..00846345 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -120,7 +120,7 @@ public AzureAISearchVectorStoreRecordCollection( .map(VectorStoreRecordDataField::getEffectiveStorageName) .collect(Collectors.toList())); - storageNames = recordDefinition.getStorageNames(); + storageNames = recordDefinition.getFieldStorageNames(); firstVectorFieldName = recordDefinition.getVectorFields().isEmpty() ? null : recordDefinition.getVectorFields().get(0).getName(); } @@ -194,7 +194,7 @@ public Mono getAsync( @Nonnull String key, GetRecordOptions options) { // If vectors are not requested, only fetch non-vector fields List selectedFields = null; - if (options != null && !options.includeVectors()) { + if (options == null || !options.includeVectors()) { selectedFields = Collections.unmodifiableList(nonVectorFields); } @@ -204,7 +204,7 @@ public Mono getAsync( // Use custom mapper if available if (mapper != null && mapper.getStorageModelToRecordMapper() != null) { return searchAsyncClient.getDocument(key, SearchDocument.class) - .map(this.options.getVectorStoreRecordMapper()::mapStorageModeltoRecord); + .map(record -> mapper.mapStorageModeltoRecord(record, options)); } return searchAsyncClient @@ -281,7 +281,8 @@ public Mono deleteBatchAsync(List keys, DeleteRecordOptions option }).collect(Collectors.toList())).then(); } - private Mono>> searchAndMapAsync(SearchOptions searchOptions) { + private Mono>> searchAndMapAsync(SearchOptions searchOptions, + GetRecordOptions getRecordOptions) { VectorStoreRecordMapper mapper = this.options .getVectorStoreRecordMapper(); @@ -292,7 +293,8 @@ private Mono>> searchAndMapAsync(SearchOptions s // Use custom mapper if available if (mapper != null && mapper.getStorageModelToRecordMapper() != null) { record = mapper - .mapStorageModeltoRecord(response.getDocument(SearchDocument.class)); + .mapStorageModeltoRecord(response.getDocument(SearchDocument.class), + getRecordOptions); } else { record = response.getDocument(this.options.getRecordClass()); } @@ -313,7 +315,7 @@ public Mono>> searchAsync(VectorSearchQuery quer throw new SKException("No vector fields defined. Cannot perform vector search"); } - VectorSearchOptions options = (VectorSearchOptions) query.getSearchOptions(); + VectorSearchOptions options = query.getSearchOptions(); if (options == null) { options = VectorSearchOptions.createDefault(firstVectorFieldName); } @@ -354,7 +356,7 @@ public Mono>> searchAsync(VectorSearchQuery quer searchOptions.setSelect(nonVectorFields.toArray(new String[0])); } - return searchAndMapAsync(searchOptions); + return searchAndMapAsync(searchOptions, new GetRecordOptions(options.isIncludeVectors())); } /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java index 1591918d..81387d65 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java @@ -19,7 +19,7 @@ public class JDBCVectorStore implements SQLVectorStore { private final DataSource dataSource; private final JDBCVectorStoreOptions options; - private final JDBCVectorStoreQueryProvider queryProvider; + private final SQLVectorStoreQueryProvider queryProvider; /** * Creates a new instance of the {@link JDBCVectorStore}. @@ -37,7 +37,7 @@ public JDBCVectorStore(@Nonnull DataSource dataSource, if (this.options != null && this.options.getQueryProvider() != null) { this.queryProvider = this.options.getQueryProvider(); } else { - this.queryProvider = JDBCVectorStoreDefaultQueryProvider.builder() + this.queryProvider = JDBCVectorStoreQueryProvider.builder() .withDataSource(dataSource) .build(); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java deleted file mode 100644 index 47a6b431..00000000 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java +++ /dev/null @@ -1,481 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.jdbc; - -import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.exceptions.SKException; -import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; -import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; -import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; - -import javax.annotation.Nonnull; -import javax.sql.DataSource; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -public class JDBCVectorStoreDefaultQueryProvider - implements JDBCVectorStoreQueryProvider { - - private final Map, String> supportedKeyTypes; - private final Map, String> supportedDataTypes; - private final Map, String> supportedVectorTypes; - private final DataSource dataSource; - private final String collectionsTable; - private final String prefixForCollectionTables; - - @SuppressFBWarnings("EI_EXPOSE_REP2") // DataSource is not exposed - protected JDBCVectorStoreDefaultQueryProvider( - @Nonnull DataSource dataSource, - @Nonnull String collectionsTable, - @Nonnull String prefixForCollectionTables) { - this.dataSource = dataSource; - this.collectionsTable = collectionsTable; - this.prefixForCollectionTables = prefixForCollectionTables; - - supportedKeyTypes = new HashMap<>(); - supportedKeyTypes.put(String.class, "VARCHAR(255)"); - - supportedDataTypes = new HashMap<>(); - supportedDataTypes.put(String.class, "TEXT"); - supportedDataTypes.put(Integer.class, "INTEGER"); - supportedDataTypes.put(int.class, "INTEGER"); - supportedDataTypes.put(Long.class, "BIGINT"); - supportedDataTypes.put(long.class, "BIGINT"); - supportedDataTypes.put(Float.class, "REAL"); - supportedDataTypes.put(float.class, "REAL"); - supportedDataTypes.put(Double.class, "DOUBLE"); - supportedDataTypes.put(double.class, "DOUBLE"); - supportedDataTypes.put(Boolean.class, "BOOLEAN"); - supportedDataTypes.put(boolean.class, "BOOLEAN"); - supportedDataTypes.put(OffsetDateTime.class, "TIMESTAMPTZ"); - - supportedVectorTypes = new HashMap<>(); - supportedVectorTypes.put(String.class, "TEXT"); - supportedVectorTypes.put(List.class, "TEXT"); - supportedVectorTypes.put(Collection.class, "TEXT"); - } - - /** - * Creates a new builder. - * @return the builder - */ - public static Builder builder() { - return new Builder(); - } - - /** - * Formats a wildcard string for a query. - * @param wildcards the number of wildcards - * @return the formatted wildcard string - */ - protected String getWildcardString(int wildcards) { - return Stream.generate(() -> "?") - .limit(wildcards) - .collect(Collectors.joining(", ")); - } - - /** - * Gets the key column name from a key field. - * @param keyField the key field - * @return the key column name - */ - protected String getKeyColumnName(VectorStoreRecordField keyField) { - return validateSQLidentifier(keyField.getEffectiveStorageName()); - } - - /** - * Formats the query columns from a record definition. - * @param fields the fields to get the columns from - * @return the formatted query columns - */ - protected String getQueryColumnsFromFields(List fields) { - return fields.stream() - .map(VectorStoreRecordField::getEffectiveStorageName) - .map(JDBCVectorStoreDefaultQueryProvider::validateSQLidentifier) - .collect(Collectors.joining(", ")); - } - - /** - * Formats the column names and types for a table. - * @param fields the fields - * @param types the types - * @return the formatted column names and types - */ - protected String getColumnNamesAndTypes(List fields, - Map, String> types) { - List columns = fields.stream() - .map(field -> validateSQLidentifier(field.getEffectiveStorageName()) + " " - + types.get(field.getFieldType())) - .collect(Collectors.toList()); - - return String.join(", ", columns); - } - - protected String getCollectionTableName(String collectionName) { - return validateSQLidentifier(prefixForCollectionTables + collectionName); - } - - /** - * Gets the supported key types and their corresponding SQL types. - * - * @return the supported key types - */ - @Override - public Map, String> getSupportedKeyTypes() { - return new HashMap<>(this.supportedKeyTypes); - } - - /** - * Gets the supported data types and their corresponding SQL types. - * - * @return the supported data types - */ - @Override - public Map, String> getSupportedDataTypes() { - return new HashMap<>(this.supportedDataTypes); - } - - /** - * Gets the supported vector types and their corresponding SQL types. - * - * @return the supported vector types - */ - @Override - public Map, String> getSupportedVectorTypes() { - return new HashMap<>(this.supportedVectorTypes); - } - - /** - * Prepares the vector store. - * Executes any necessary setup steps for the vector store. - * - * @throws SKException if an error occurs while preparing the vector store - */ - @Override - public void prepareVectorStore() { - String createCollectionsTable = formatQuery( - "CREATE TABLE IF NOT EXISTS %s (collectionId VARCHAR(255) PRIMARY KEY);", - validateSQLidentifier(collectionsTable)); - - try (Connection connection = dataSource.getConnection(); - PreparedStatement createTable = connection.prepareStatement(createCollectionsTable)) { - createTable.execute(); - } catch (SQLException e) { - throw new SKException("Failed to prepare vector store", e); - } - } - - /** - * Checks if the types of the record class fields are supported. - * - * @param recordDefinition the record definition - * @throws SKException if the types are not supported - */ - @Override - public void validateSupportedTypes(VectorStoreRecordDefinition recordDefinition) { - - VectorStoreRecordDefinition.validateSupportedTypes( - Collections.singletonList(recordDefinition.getKeyField()), - getSupportedKeyTypes().keySet()); - VectorStoreRecordDefinition.validateSupportedTypes( - new ArrayList<>(recordDefinition.getDataFields()), - getSupportedDataTypes().keySet()); - VectorStoreRecordDefinition.validateSupportedTypes( - new ArrayList<>(recordDefinition.getVectorFields()), - getSupportedVectorTypes().keySet()); - } - - /** - * Checks if a collection exists. - * - * @param collectionName the collection name - * @return true if the collection exists, false otherwise - * @throws SKException if an error occurs while checking if the collection exists - */ - @Override - public boolean collectionExists(String collectionName) { - String query = formatQuery("SELECT 1 FROM %s WHERE collectionId = ?", - validateSQLidentifier(collectionsTable)); - - try (Connection connection = dataSource.getConnection(); - PreparedStatement statement = connection.prepareStatement(query)) { - statement.setObject(1, collectionName); - - return statement.executeQuery().next(); - } catch (SQLException e) { - throw new SKException("Failed to check if collection exists", e); - } - } - - /** - * Creates a collection. - * - * @param collectionName the collection name - * @param recordDefinition the record definition - * @throws SKException if an error occurs while creating the collection - */ - @Override - @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") // SQL query is generated dynamically with valid identifiers - public void createCollection(String collectionName, - VectorStoreRecordDefinition recordDefinition) { - - String createStorageTable = formatQuery("CREATE TABLE IF NOT EXISTS %s (" - + "%s VARCHAR(255) PRIMARY KEY, " - + "%s, " - + "%s);", - getCollectionTableName(collectionName), - getKeyColumnName(recordDefinition.getKeyField()), - getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), - getSupportedDataTypes()), - getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getVectorFields()), - getSupportedVectorTypes())); - - String insertCollectionQuery = formatQuery("INSERT INTO %s (collectionId) VALUES (?)", - validateSQLidentifier(collectionsTable)); - - try (Connection connection = dataSource.getConnection(); - PreparedStatement createTable = connection.prepareStatement(createStorageTable)) { - createTable.execute(); - } catch (SQLException e) { - throw new SKException("Failed to create collection", e); - } - - try (Connection connection = dataSource.getConnection(); - PreparedStatement insert = connection.prepareStatement(insertCollectionQuery)) { - insert.setObject(1, collectionName); - insert.execute(); - } catch (SQLException e) { - throw new SKException("Failed to insert collection", e); - } - } - - /** - * Deletes a collection. - * - * @param collectionName the collection name - * @throws SKException if an error occurs while deleting the collection - */ - @Override - public void deleteCollection(String collectionName) { - String deleteCollectionOperation = formatQuery("DELETE FROM %s WHERE collectionId = ?", - validateSQLidentifier(collectionsTable)); - String dropTableOperation = formatQuery("DROP TABLE %s", - getCollectionTableName(collectionName)); - - try (Connection connection = dataSource.getConnection(); - PreparedStatement deleteCollection = connection - .prepareStatement(deleteCollectionOperation)) { - deleteCollection.setObject(1, collectionName); - deleteCollection.execute(); - } catch (SQLException e) { - throw new SKException("Failed to delete collection", e); - } - - try (Connection connection = dataSource.getConnection(); - PreparedStatement dropTable = connection.prepareStatement(dropTableOperation)) { - dropTable.execute(); - } catch (SQLException e) { - throw new SKException("Failed to drop table", e); - } - } - - /** - * Gets the collection names. - * - * @return the collection names - * @throws SKException if an error occurs while getting the collection names - */ - @Override - public List getCollectionNames() { - String query = formatQuery("SELECT collectionId FROM %s", - validateSQLidentifier(collectionsTable)); - - try (Connection connection = dataSource.getConnection(); - PreparedStatement statement = connection.prepareStatement(query)) { - List collectionNames = new ArrayList<>(); - ResultSet resultSet = statement.executeQuery(); - - while (resultSet.next()) { - collectionNames.add(resultSet.getString(1)); - } - - return Collections.unmodifiableList(collectionNames); - } catch (SQLException e) { - throw new SKException("Failed to get collection names", e); - } - } - - /** - * Gets a list of records from the store. - * - * @param collectionName the collection name - * @param keys the keys - * @param recordDefinition the record definition - * @param mapper the mapper - * @param options the options - * @return the records - * @param the record type - * @throws SKException if an error occurs while getting the records - */ - @Override - public List getRecords(String collectionName, List keys, - VectorStoreRecordDefinition recordDefinition, - VectorStoreRecordMapper mapper, - GetRecordOptions options) { - List fields; - if (options == null || options.includeVectors()) { - fields = recordDefinition.getAllFields(); - } else { - fields = recordDefinition.getNonVectorFields(); - } - - String query = formatQuery("SELECT %s FROM %s WHERE %s IN (%s)", - getQueryColumnsFromFields(fields), - getCollectionTableName(collectionName), - getKeyColumnName(recordDefinition.getKeyField()), - getWildcardString(keys.size())); - - try (Connection connection = dataSource.getConnection(); - PreparedStatement statement = connection.prepareStatement(query)) { - for (int i = 0; i < keys.size(); ++i) { - statement.setObject(i + 1, keys.get(i)); - } - - List records = new ArrayList<>(); - ResultSet resultSet = statement.executeQuery(); - - while (resultSet.next()) { - records.add(mapper.mapStorageModeltoRecord(resultSet)); - } - - return Collections.unmodifiableList(records); - } catch (SQLException e) { - throw new SKException("Failed to set statement values", e); - } - } - - @Override - public void upsertRecords(String collectionName, List records, - VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { - throw new UnsupportedOperationException( - "Upsert is not supported. Try with a specific query provider."); - } - - /** - * Deletes records. - * - * @param collectionName the collection name - * @param keys the keys - * @param recordDefinition the record definition - * @param options the options - * @throws SKException if an error occurs while deleting the records - */ - @Override - public void deleteRecords(String collectionName, List keys, - VectorStoreRecordDefinition recordDefinition, DeleteRecordOptions options) { - String query = formatQuery("DELETE FROM %s WHERE %s IN (%s)", - getCollectionTableName(collectionName), - getKeyColumnName(recordDefinition.getKeyField()), - getWildcardString(keys.size())); - - try (Connection connection = dataSource.getConnection(); - PreparedStatement statement = connection.prepareStatement(query)) { - for (int i = 0; i < keys.size(); ++i) { - statement.setObject(i + 1, keys.get(i)); - } - - statement.execute(); - } catch (SQLException e) { - throw new SKException("Failed to set statement values", e); - } - } - - /** - * Validates an SQL identifier. - * - * @param identifier the identifier - * @return the identifier if it is valid - * @throws SKException if the identifier is invalid - */ - public static String validateSQLidentifier(String identifier) { - if (identifier.matches("[a-zA-Z_][a-zA-Z0-9_]*")) { - return identifier; - } - throw new SKException("Invalid SQL identifier: " + identifier); - } - - /** - * Formats a query. - * - * @param query the query - * @param args the arguments - * @return the formatted query - */ - public String formatQuery(String query, String... args) { - return String.format(query, (Object[]) args); - } - - /** - * The builder for {@link JDBCVectorStoreDefaultQueryProvider}. - */ - public static class Builder - implements JDBCVectorStoreQueryProvider.Builder { - private DataSource dataSource; - private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; - private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; - - /** - * Sets the data source. - * @param dataSource the data source - * @return the builder - */ - @SuppressFBWarnings("EI_EXPOSE_REP2") // DataSource is not exposed - public Builder withDataSource(DataSource dataSource) { - this.dataSource = dataSource; - return this; - } - - /** - * Sets the collections table name. - * @param collectionsTable the collections table name - * @return the builder - */ - public Builder withCollectionsTable(String collectionsTable) { - this.collectionsTable = validateSQLidentifier(collectionsTable); - return this; - } - - /** - * Sets the prefix for collection tables. - * @param prefixForCollectionTables the prefix for collection tables - * @return the builder - */ - public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { - this.prefixForCollectionTables = validateSQLidentifier(prefixForCollectionTables); - return this; - } - - @Override - public JDBCVectorStoreDefaultQueryProvider build() { - if (dataSource == null) { - throw new SKException("DataSource is required"); - } - - return new JDBCVectorStoreDefaultQueryProvider(dataSource, collectionsTable, - prefixForCollectionTables); - } - } -} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java index adb6e13c..58e2c85e 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java @@ -9,7 +9,7 @@ public class JDBCVectorStoreOptions { @Nullable private final JDBCVectorStoreRecordCollectionFactory vectorStoreRecordCollectionFactory; @Nullable - private final JDBCVectorStoreQueryProvider queryProvider; + private final SQLVectorStoreQueryProvider queryProvider; /** * Creates a new instance of the JDBC vector store options. @@ -18,7 +18,7 @@ public class JDBCVectorStoreOptions { */ @SuppressFBWarnings("EI_EXPOSE_REP2") // DataSource in queryProvider is not exposed public JDBCVectorStoreOptions( - @Nullable JDBCVectorStoreQueryProvider queryProvider, + @Nullable SQLVectorStoreQueryProvider queryProvider, @Nullable JDBCVectorStoreRecordCollectionFactory vectorStoreRecordCollectionFactory) { this.queryProvider = queryProvider; this.vectorStoreRecordCollectionFactory = vectorStoreRecordCollectionFactory; @@ -38,7 +38,7 @@ public JDBCVectorStoreOptions() { */ @Nullable @SuppressFBWarnings("EI_EXPOSE_REP") // DataSource in queryProvider is not exposed - public JDBCVectorStoreQueryProvider getQueryProvider() { + public SQLVectorStoreQueryProvider getQueryProvider() { return queryProvider; } @@ -67,7 +67,7 @@ public JDBCVectorStoreRecordCollectionFactory getVectorStoreRecordCollectionFact */ public static class Builder { @Nullable - private JDBCVectorStoreQueryProvider queryProvider; + private SQLVectorStoreQueryProvider queryProvider; @Nullable private JDBCVectorStoreRecordCollectionFactory vectorStoreRecordCollectionFactory; @@ -78,7 +78,7 @@ public static class Builder { * @return The updated builder instance. */ @SuppressFBWarnings("EI_EXPOSE_REP2") // DataSource in queryProvider is not exposed - public Builder withQueryProvider(JDBCVectorStoreQueryProvider queryProvider) { + public Builder withQueryProvider(SQLVectorStoreQueryProvider queryProvider) { this.queryProvider = queryProvider; return this; } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index 757253c8..5b3046be 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -1,98 +1,327 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; -import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.exceptions.SKException; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import javax.annotation.Nonnull; +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class JDBCVectorStoreQueryProvider + implements SQLVectorStoreQueryProvider { + + private final Map, String> supportedKeyTypes; + private final Map, String> supportedDataTypes; + private final Map, String> supportedVectorTypes; + private final DataSource dataSource; + private final String collectionsTable; + private final String prefixForCollectionTables; + + @SuppressFBWarnings("EI_EXPOSE_REP2") // DataSource is not exposed + protected JDBCVectorStoreQueryProvider( + @Nonnull DataSource dataSource, + @Nonnull String collectionsTable, + @Nonnull String prefixForCollectionTables) { + this.dataSource = dataSource; + this.collectionsTable = collectionsTable; + this.prefixForCollectionTables = prefixForCollectionTables; + + supportedKeyTypes = new HashMap<>(); + supportedKeyTypes.put(String.class, "VARCHAR(255)"); + + supportedDataTypes = new HashMap<>(); + supportedDataTypes.put(String.class, "TEXT"); + supportedDataTypes.put(Integer.class, "INTEGER"); + supportedDataTypes.put(int.class, "INTEGER"); + supportedDataTypes.put(Long.class, "BIGINT"); + supportedDataTypes.put(long.class, "BIGINT"); + supportedDataTypes.put(Float.class, "REAL"); + supportedDataTypes.put(float.class, "REAL"); + supportedDataTypes.put(Double.class, "DOUBLE"); + supportedDataTypes.put(double.class, "DOUBLE"); + supportedDataTypes.put(Boolean.class, "BOOLEAN"); + supportedDataTypes.put(boolean.class, "BOOLEAN"); + supportedDataTypes.put(OffsetDateTime.class, "TIMESTAMPTZ"); + + supportedVectorTypes = new HashMap<>(); + supportedVectorTypes.put(String.class, "TEXT"); + supportedVectorTypes.put(List.class, "TEXT"); + supportedVectorTypes.put(Collection.class, "TEXT"); + } + + /** + * Creates a new builder. + * @return the builder + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Formats a wildcard string for a query. + * @param wildcards the number of wildcards + * @return the formatted wildcard string + */ + protected String getWildcardString(int wildcards) { + return Stream.generate(() -> "?") + .limit(wildcards) + .collect(Collectors.joining(", ")); + } -/** - * The JDBC vector store query provider. - * Provides the necessary methods to interact with a JDBC vector store and vector store collections. - */ -public interface JDBCVectorStoreQueryProvider { /** - * The default name for the collections table. + * Gets the key column name from a key field. + * @param keyField the key field + * @return the key column name */ - String DEFAULT_COLLECTIONS_TABLE = "SKCollections"; + protected String getKeyColumnName(VectorStoreRecordField keyField) { + return validateSQLidentifier(keyField.getEffectiveStorageName()); + } + + /** + * Formats the query columns from a record definition. + * @param fields the fields to get the columns from + * @return the formatted query columns + */ + protected String getQueryColumnsFromFields(List fields) { + return fields.stream() + .map(VectorStoreRecordField::getEffectiveStorageName) + .map(JDBCVectorStoreQueryProvider::validateSQLidentifier) + .collect(Collectors.joining(", ")); + } /** - * The prefix for collection tables. + * Formats the column names and types for a table. + * @param fields the fields + * @param types the types + * @return the formatted column names and types */ - String DEFAULT_PREFIX_FOR_COLLECTION_TABLES = "SKCollection_"; + protected String getColumnNamesAndTypes(List fields, + Map, String> types) { + List columns = fields.stream() + .map(field -> validateSQLidentifier(field.getEffectiveStorageName()) + " " + + types.get(field.getFieldType())) + .collect(Collectors.toList()); + + return String.join(", ", columns); + } + + protected String getCollectionTableName(String collectionName) { + return validateSQLidentifier(prefixForCollectionTables + collectionName); + } /** * Gets the supported key types and their corresponding SQL types. * * @return the supported key types */ - Map, String> getSupportedKeyTypes(); + @Override + public Map, String> getSupportedKeyTypes() { + return new HashMap<>(this.supportedKeyTypes); + } /** * Gets the supported data types and their corresponding SQL types. * * @return the supported data types */ - Map, String> getSupportedDataTypes(); + @Override + public Map, String> getSupportedDataTypes() { + return new HashMap<>(this.supportedDataTypes); + } /** * Gets the supported vector types and their corresponding SQL types. * * @return the supported vector types */ - Map, String> getSupportedVectorTypes(); + @Override + public Map, String> getSupportedVectorTypes() { + return new HashMap<>(this.supportedVectorTypes); + } /** * Prepares the vector store. * Executes any necessary setup steps for the vector store. + * + * @throws SKException if an error occurs while preparing the vector store */ - void prepareVectorStore(); + @Override + public void prepareVectorStore() { + String createCollectionsTable = formatQuery( + "CREATE TABLE IF NOT EXISTS %s (collectionId VARCHAR(255) PRIMARY KEY);", + validateSQLidentifier(collectionsTable)); + + try (Connection connection = dataSource.getConnection(); + PreparedStatement createTable = connection.prepareStatement(createCollectionsTable)) { + createTable.execute(); + } catch (SQLException e) { + throw new SKException("Failed to prepare vector store", e); + } + } /** * Checks if the types of the record class fields are supported. * * @param recordDefinition the record definition + * @throws SKException if the types are not supported */ - void validateSupportedTypes(VectorStoreRecordDefinition recordDefinition); + @Override + public void validateSupportedTypes(VectorStoreRecordDefinition recordDefinition) { + + VectorStoreRecordDefinition.validateSupportedTypes( + Collections.singletonList(recordDefinition.getKeyField()), + getSupportedKeyTypes().keySet()); + VectorStoreRecordDefinition.validateSupportedTypes( + new ArrayList<>(recordDefinition.getDataFields()), + getSupportedDataTypes().keySet()); + VectorStoreRecordDefinition.validateSupportedTypes( + new ArrayList<>(recordDefinition.getVectorFields()), + getSupportedVectorTypes().keySet()); + } /** * Checks if a collection exists. * * @param collectionName the collection name * @return true if the collection exists, false otherwise + * @throws SKException if an error occurs while checking if the collection exists */ - boolean collectionExists(String collectionName); + @Override + public boolean collectionExists(String collectionName) { + String query = formatQuery("SELECT 1 FROM %s WHERE collectionId = ?", + validateSQLidentifier(collectionsTable)); + + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(query)) { + statement.setObject(1, collectionName); + + return statement.executeQuery().next(); + } catch (SQLException e) { + throw new SKException("Failed to check if collection exists", e); + } + } /** * Creates a collection. * * @param collectionName the collection name * @param recordDefinition the record definition + * @throws SKException if an error occurs while creating the collection */ - void createCollection(String collectionName, VectorStoreRecordDefinition recordDefinition); + @Override + @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") // SQL query is generated dynamically with valid identifiers + public void createCollection(String collectionName, + VectorStoreRecordDefinition recordDefinition) { + + String createStorageTable = formatQuery("CREATE TABLE IF NOT EXISTS %s (" + + "%s VARCHAR(255) PRIMARY KEY, " + + "%s, " + + "%s);", + getCollectionTableName(collectionName), + getKeyColumnName(recordDefinition.getKeyField()), + getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), + getSupportedDataTypes()), + getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getVectorFields()), + getSupportedVectorTypes())); + + String insertCollectionQuery = formatQuery("INSERT INTO %s (collectionId) VALUES (?)", + validateSQLidentifier(collectionsTable)); + + try (Connection connection = dataSource.getConnection(); + PreparedStatement createTable = connection.prepareStatement(createStorageTable)) { + createTable.execute(); + } catch (SQLException e) { + throw new SKException("Failed to create collection", e); + } + + try (Connection connection = dataSource.getConnection(); + PreparedStatement insert = connection.prepareStatement(insertCollectionQuery)) { + insert.setObject(1, collectionName); + insert.execute(); + } catch (SQLException e) { + throw new SKException("Failed to insert collection", e); + } + } /** * Deletes a collection. * * @param collectionName the collection name + * @throws SKException if an error occurs while deleting the collection */ - void deleteCollection(String collectionName); + @Override + public void deleteCollection(String collectionName) { + String deleteCollectionOperation = formatQuery("DELETE FROM %s WHERE collectionId = ?", + validateSQLidentifier(collectionsTable)); + String dropTableOperation = formatQuery("DROP TABLE %s", + getCollectionTableName(collectionName)); + + try (Connection connection = dataSource.getConnection(); + PreparedStatement deleteCollection = connection + .prepareStatement(deleteCollectionOperation)) { + deleteCollection.setObject(1, collectionName); + deleteCollection.execute(); + } catch (SQLException e) { + throw new SKException("Failed to delete collection", e); + } + + try (Connection connection = dataSource.getConnection(); + PreparedStatement dropTable = connection.prepareStatement(dropTableOperation)) { + dropTable.execute(); + } catch (SQLException e) { + throw new SKException("Failed to drop table", e); + } + } /** * Gets the collection names. * * @return the collection names + * @throws SKException if an error occurs while getting the collection names */ - List getCollectionNames(); + @Override + public List getCollectionNames() { + String query = formatQuery("SELECT collectionId FROM %s", + validateSQLidentifier(collectionsTable)); + + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(query)) { + List collectionNames = new ArrayList<>(); + ResultSet resultSet = statement.executeQuery(); + + while (resultSet.next()) { + collectionNames.add(resultSet.getString(1)); + } + + return Collections.unmodifiableList(collectionNames); + } catch (SQLException e) { + throw new SKException("Failed to get collection names", e); + } + } /** - * Gets records. + * Gets a list of records from the store. * * @param collectionName the collection name * @param keys the keys @@ -100,38 +329,173 @@ public interface JDBCVectorStoreQueryProvider { * @param mapper the mapper * @param options the options * @return the records + * @param the record type + * @throws SKException if an error occurs while getting the records */ - List getRecords(String collectionName, List keys, + @Override + public List getRecords(String collectionName, List keys, VectorStoreRecordDefinition recordDefinition, VectorStoreRecordMapper mapper, - GetRecordOptions options); + GetRecordOptions options) { + List fields; + if (options != null && options.includeVectors()) { + fields = recordDefinition.getAllFields(); + } else { + fields = recordDefinition.getNonVectorFields(); + } + + String query = formatQuery("SELECT %s FROM %s WHERE %s IN (%s)", + getQueryColumnsFromFields(fields), + getCollectionTableName(collectionName), + getKeyColumnName(recordDefinition.getKeyField()), + getWildcardString(keys.size())); + + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(query)) { + for (int i = 0; i < keys.size(); ++i) { + statement.setObject(i + 1, keys.get(i)); + } + + List records = new ArrayList<>(); + ResultSet resultSet = statement.executeQuery(); + + while (resultSet.next()) { + records.add(mapper.mapStorageModeltoRecord(resultSet, options)); + } + + return Collections.unmodifiableList(records); + } catch (SQLException e) { + throw new SKException("Failed to set statement values", e); + } + } + + @Override + public void upsertRecords(String collectionName, List records, + VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { + throw new UnsupportedOperationException( + "Upsert is not supported. Try with a specific query provider."); + } /** - * Upserts records. + * Deletes records. * * @param collectionName the collection name - * @param records the records - * @param vectorStoreRecordDefinition the record definition + * @param keys the keys + * @param recordDefinition the record definition * @param options the options + * @throws SKException if an error occurs while deleting the records */ - void upsertRecords(String collectionName, List records, - VectorStoreRecordDefinition vectorStoreRecordDefinition, UpsertRecordOptions options); + @Override + public void deleteRecords(String collectionName, List keys, + VectorStoreRecordDefinition recordDefinition, DeleteRecordOptions options) { + String query = formatQuery("DELETE FROM %s WHERE %s IN (%s)", + getCollectionTableName(collectionName), + getKeyColumnName(recordDefinition.getKeyField()), + getWildcardString(keys.size())); + + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(query)) { + for (int i = 0; i < keys.size(); ++i) { + statement.setObject(i + 1, keys.get(i)); + } + + statement.execute(); + } catch (SQLException e) { + throw new SKException("Failed to set statement values", e); + } + } /** - * Deletes records. + * Searches for records. * * @param collectionName the collection name - * @param keys the keys + * @param query the query * @param recordDefinition the record definition - * @param options the options + * @param mapper the mapper + * @param the record type + * @return the search results + */ + @Override + public List> search(String collectionName, + VectorSearchQuery query, VectorStoreRecordDefinition recordDefinition, + VectorStoreRecordMapper mapper) { + throw new UnsupportedOperationException( + "Search is not supported. Try with a specific query provider."); + } + + /** + * Validates an SQL identifier. + * + * @param identifier the identifier + * @return the identifier if it is valid + * @throws SKException if the identifier is invalid */ - void deleteRecords(String collectionName, List keys, - VectorStoreRecordDefinition recordDefinition, DeleteRecordOptions options); + public static String validateSQLidentifier(String identifier) { + if (identifier.matches("[a-zA-Z_][a-zA-Z0-9_]*")) { + return identifier; + } + throw new SKException("Invalid SQL identifier: " + identifier); + } + + /** + * Formats a query. + * + * @param query the query + * @param args the arguments + * @return the formatted query + */ + public String formatQuery(String query, String... args) { + return String.format(query, (Object[]) args); + } /** - * The builder for the JDBC vector store query provider. + * The builder for {@link JDBCVectorStoreQueryProvider}. */ - interface Builder extends SemanticKernelBuilder { + public static class Builder + implements SQLVectorStoreQueryProvider.Builder { + private DataSource dataSource; + private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; + private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; + + /** + * Sets the data source. + * @param dataSource the data source + * @return the builder + */ + @SuppressFBWarnings("EI_EXPOSE_REP2") // DataSource is not exposed + public Builder withDataSource(DataSource dataSource) { + this.dataSource = dataSource; + return this; + } + + /** + * Sets the collections table name. + * @param collectionsTable the collections table name + * @return the builder + */ + public Builder withCollectionsTable(String collectionsTable) { + this.collectionsTable = validateSQLidentifier(collectionsTable); + return this; + } + + /** + * Sets the prefix for collection tables. + * @param prefixForCollectionTables the prefix for collection tables + * @return the builder + */ + public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { + this.prefixForCollectionTables = validateSQLidentifier(prefixForCollectionTables); + return this; + } + + @Override + public JDBCVectorStoreQueryProvider build() { + if (dataSource == null) { + throw new SKException("DataSource is required"); + } + return new JDBCVectorStoreQueryProvider(dataSource, collectionsTable, + prefixForCollectionTables); + } } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java index 8db96c20..21b87418 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -5,12 +5,16 @@ import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.VectorizedSearch; +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.lang.reflect.Field; @@ -24,13 +28,14 @@ import reactor.core.scheduler.Schedulers; public class JDBCVectorStoreRecordCollection - implements SQLVectorStoreRecordCollection { + implements SQLVectorStoreRecordCollection, + VectorizedSearch { private final String collectionName; private final VectorStoreRecordDefinition recordDefinition; private final VectorStoreRecordMapper vectorStoreRecordMapper; private final JDBCVectorStoreRecordCollectionOptions options; - private final JDBCVectorStoreQueryProvider queryProvider; + private final SQLVectorStoreQueryProvider queryProvider; /** * Creates a new instance of the {@link JDBCVectorStoreRecordCollection}. @@ -54,7 +59,7 @@ public JDBCVectorStoreRecordCollection( // If the query provider is not provided, set a default one if (options.getQueryProvider() == null) { - this.queryProvider = JDBCVectorStoreDefaultQueryProvider.builder() + this.queryProvider = JDBCVectorStoreQueryProvider.builder() .withDataSource(dataSource) .build(); } else { @@ -285,6 +290,28 @@ public Mono prepareAsync() { .subscribeOn(Schedulers.boundedElastic()).then(); } + @Override + public Mono>> searchAsync(VectorSearchQuery query) { + return Mono.fromCallable( + () -> { + return queryProvider.search(this.collectionName, query, recordDefinition, + vectorStoreRecordMapper); + }).subscribeOn(Schedulers.boundedElastic()); + } + + /** + * Vectorized search. This method searches for records that are similar to the given vector. + * + * @param vector The vector to search with. + * @param vectorSearchOptions The options to use for the search. + * @return A list of search results. + */ + @Override + public Mono>> searchAsync(List vector, + VectorSearchOptions vectorSearchOptions) { + return this.searchAsync(VectorSearchQuery.createQuery(vector, vectorSearchOptions)); + } + public static class Builder implements SemanticKernelBuilder> { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java index c9046f59..8ae826d4 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java @@ -9,16 +9,16 @@ import java.sql.ResultSet; -import static com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreDefaultQueryProvider.validateSQLidentifier; -import static com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider.DEFAULT_COLLECTIONS_TABLE; -import static com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider.DEFAULT_PREFIX_FOR_COLLECTION_TABLES; +import static com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider.validateSQLidentifier; +import static com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider.DEFAULT_COLLECTIONS_TABLE; +import static com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider.DEFAULT_PREFIX_FOR_COLLECTION_TABLES; public class JDBCVectorStoreRecordCollectionOptions implements VectorStoreRecordCollectionOptions { private final Class recordClass; private final VectorStoreRecordMapper vectorStoreRecordMapper; private final VectorStoreRecordDefinition recordDefinition; - private final JDBCVectorStoreQueryProvider queryProvider; + private final SQLVectorStoreQueryProvider queryProvider; private final String collectionsTableName; private final String prefixForCollectionTables; @@ -26,7 +26,7 @@ private JDBCVectorStoreRecordCollectionOptions( Class recordClass, VectorStoreRecordDefinition recordDefinition, VectorStoreRecordMapper vectorStoreRecordMapper, - JDBCVectorStoreQueryProvider queryProvider, + SQLVectorStoreQueryProvider queryProvider, String collectionsTableName, String prefixForCollectionTables) { this.recordClass = recordClass; @@ -101,7 +101,7 @@ public String getPrefixForCollectionTables() { * @return the query provider */ @SuppressFBWarnings("EI_EXPOSE_REP") // DataSource in queryProvider is not exposed - public JDBCVectorStoreQueryProvider getQueryProvider() { + public SQLVectorStoreQueryProvider getQueryProvider() { return queryProvider; } @@ -109,7 +109,7 @@ public static class Builder { private Class recordClass; private VectorStoreRecordDefinition recordDefinition; private VectorStoreRecordMapper vectorStoreRecordMapper; - private JDBCVectorStoreQueryProvider queryProvider; + private SQLVectorStoreQueryProvider queryProvider; private String collectionsTableName = DEFAULT_COLLECTIONS_TABLE; private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; @@ -150,7 +150,7 @@ public Builder withVectorStoreRecordMapper( * @return the builder */ @SuppressFBWarnings("EI_EXPOSE_REP2") // DataSource in queryProvider is not exposed - public Builder withQueryProvider(JDBCVectorStoreQueryProvider queryProvider) { + public Builder withQueryProvider(SQLVectorStoreQueryProvider queryProvider) { this.queryProvider = queryProvider; return this; } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java index 40e654f2..162b55bb 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java @@ -10,6 +10,7 @@ import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -17,6 +18,7 @@ import java.util.List; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.function.BiFunction; import java.util.function.Function; public class JDBCVectorStoreRecordMapper @@ -27,7 +29,8 @@ public class JDBCVectorStoreRecordMapper * * @param storageModelToRecordMapper the function to convert a storage model to a record */ - protected JDBCVectorStoreRecordMapper(Function storageModelToRecordMapper) { + protected JDBCVectorStoreRecordMapper( + BiFunction storageModelToRecordMapper) { super(null, storageModelToRecordMapper); } @@ -104,7 +107,7 @@ public JDBCVectorStoreRecordMapper build() { } return new JDBCVectorStoreRecordMapper<>( - resultSet -> { + (resultSet, options) -> { try { // Create an ObjectNode to hold the values ObjectNode objectNode = objectMapper.createObjectNode(); @@ -112,8 +115,7 @@ public JDBCVectorStoreRecordMapper build() { // Select fields from the record definition. List fields; ResultSetMetaData metaData = resultSet.getMetaData(); - if (metaData.getColumnCount() == vectorStoreRecordDefinition.getAllFields() - .size()) { + if (options != null && options.includeVectors()) { fields = vectorStoreRecordDefinition.getAllFields(); } else { fields = vectorStoreRecordDefinition.getNonVectorFields(); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java new file mode 100644 index 00000000..f5419d78 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.jdbc; + +import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; + +import java.sql.ResultSet; +import java.util.List; +import java.util.Map; + +/** + * The JDBC vector store query provider. + * Provides the necessary methods to interact with a JDBC vector store and vector store collections. + */ +public interface SQLVectorStoreQueryProvider { + /** + * The default name for the collections table. + */ + String DEFAULT_COLLECTIONS_TABLE = "SKCollections"; + + /** + * The prefix for collection tables. + */ + String DEFAULT_PREFIX_FOR_COLLECTION_TABLES = "SKCollection_"; + + /** + * Gets the supported key types and their corresponding SQL types. + * + * @return the supported key types + */ + Map, String> getSupportedKeyTypes(); + + /** + * Gets the supported data types and their corresponding SQL types. + * + * @return the supported data types + */ + Map, String> getSupportedDataTypes(); + + /** + * Gets the supported vector types and their corresponding SQL types. + * + * @return the supported vector types + */ + Map, String> getSupportedVectorTypes(); + + /** + * Prepares the vector store. + * Executes any necessary setup steps for the vector store. + */ + void prepareVectorStore(); + + /** + * Checks if the types of the record class fields are supported. + * + * @param recordDefinition the record definition + */ + void validateSupportedTypes(VectorStoreRecordDefinition recordDefinition); + + /** + * Checks if a collection exists. + * + * @param collectionName the collection name + * @return true if the collection exists, false otherwise + */ + boolean collectionExists(String collectionName); + + /** + * Creates a collection. + * + * @param collectionName the collection name + * @param recordDefinition the record definition + */ + void createCollection(String collectionName, VectorStoreRecordDefinition recordDefinition); + + /** + * Deletes a collection. + * + * @param collectionName the collection name + */ + void deleteCollection(String collectionName); + + /** + * Gets the collection names. + * + * @return the collection names + */ + List getCollectionNames(); + + /** + * Gets records. + * + * @param collectionName the collection name + * @param keys the keys + * @param recordDefinition the record definition + * @param mapper the mapper + * @param options the options + * @return the records + */ + List getRecords(String collectionName, List keys, + VectorStoreRecordDefinition recordDefinition, + VectorStoreRecordMapper mapper, + GetRecordOptions options); + + /** + * Upserts records. + * + * @param collectionName the collection name + * @param records the records + * @param vectorStoreRecordDefinition the record definition + * @param options the options + */ + void upsertRecords(String collectionName, List records, + VectorStoreRecordDefinition vectorStoreRecordDefinition, UpsertRecordOptions options); + + /** + * Deletes records. + * + * @param collectionName the collection name + * @param keys the keys + * @param recordDefinition the record definition + * @param options the options + */ + void deleteRecords(String collectionName, List keys, + VectorStoreRecordDefinition recordDefinition, DeleteRecordOptions options); + + /** + * + * + * @param query + */ + List> search(String collectionName, + VectorSearchQuery query, + VectorStoreRecordDefinition recordDefinition, + VectorStoreRecordMapper mapper); + + /** + * The builder for the JDBC vector store query provider. + */ + interface Builder extends SemanticKernelBuilder { + + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java index d01d5c62..87aa4d69 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java @@ -4,8 +4,8 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreDefaultQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; @@ -22,7 +22,7 @@ import java.util.stream.Collectors; public class MySQLVectorStoreQueryProvider extends - JDBCVectorStoreDefaultQueryProvider implements JDBCVectorStoreQueryProvider { + JDBCVectorStoreQueryProvider implements SQLVectorStoreQueryProvider { private final DataSource dataSource; private final ObjectMapper objectMapper; @@ -111,7 +111,7 @@ public void upsertRecords(String collectionName, List records, } public static class Builder - extends JDBCVectorStoreDefaultQueryProvider.Builder { + extends JDBCVectorStoreQueryProvider.Builder { private DataSource dataSource; private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index 428707b6..e30d9838 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -4,13 +4,19 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreDefaultQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizedSearchQuery; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -18,6 +24,7 @@ import javax.sql.DataSource; import java.sql.Connection; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.time.OffsetDateTime; @@ -29,7 +36,7 @@ import java.util.stream.Collectors; public class PostgreSQLVectorStoreQueryProvider extends - JDBCVectorStoreDefaultQueryProvider implements JDBCVectorStoreQueryProvider { + JDBCVectorStoreQueryProvider implements SQLVectorStoreQueryProvider { private final Map, String> supportedKeyTypes; private final Map, String> supportedDataTypes; @@ -172,7 +179,7 @@ private String createIndexForVectorField(String collectionName, getCollectionTableName(collectionName) + "_index", getCollectionTableName(collectionName), indexKind.getValue(), - vectorField.getName(), + vectorField.getEffectiveStorageName(), distanceFunction.getValue()); } @@ -313,8 +320,86 @@ public void upsertRecords(String collectionName, List records, } } + /** + * @param collectionName + * @param query + * @param recordDefinition + * @param mapper + */ + @Override + public List> search(String collectionName, + VectorSearchQuery query, VectorStoreRecordDefinition recordDefinition, + VectorStoreRecordMapper mapper) { + if (recordDefinition.getVectorFields().isEmpty()) { + throw new SKException("No vector fields defined. Cannot perform vector search"); + } + + if (query instanceof VectorizedSearchQuery) { + VectorizedSearchQuery vectorizedSearchQuery = (VectorizedSearchQuery) query; + VectorSearchOptions options = query.getSearchOptions(); + + VectorStoreRecordVectorField firstVectorField = recordDefinition.getVectorFields() + .get(0); + if (options == null) { + options = VectorSearchOptions.createDefault(firstVectorField.getName()); + } + + VectorStoreRecordVectorField vectorField = options.getVectorFieldName() == null + ? firstVectorField + : (VectorStoreRecordVectorField) recordDefinition + .getField(options.getVectorFieldName()); + + PostgreSQLVectorIndexKind indexKind = PostgreSQLVectorIndexKind + .fromIndexKind(vectorField.getIndexKind()); + PostgreSQLVectorDistanceFunction distanceFunction = PostgreSQLVectorDistanceFunction + .fromDistanceFunction(vectorField.getDistanceFunction()); + + // If indexKind is not specified, there is no index associated to the vector field + // and pgvector performs exact nearest neighbor search. + // If indexKind is specified, a distance function is required. + if (indexKind != null && distanceFunction == null) { + throw new SKException( + "Distance function is required for vector field: " + vectorField.getName()); + } + + String searchQuery = formatQuery( + "SELECT %s, %s %s ?::vector AS score FROM %s ORDER BY score LIMIT ? OFFSET ?", + getQueryColumnsFromFields( + options.isIncludeVectors() ? recordDefinition.getAllFields() + : recordDefinition.getNonVectorFields()), + vectorField.getEffectiveStorageName(), + distanceFunction == null ? PostgreSQLVectorDistanceFunction.L2.getOperator() + : distanceFunction.getOperator(), + getCollectionTableName(collectionName)); + + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(searchQuery)) { + statement.setString(1, + objectMapper.writeValueAsString(vectorizedSearchQuery.getVector())); + statement.setInt(2, options.getLimit()); + statement.setInt(3, options.getOffset()); + + List> records = new ArrayList<>(); + ResultSet resultSet = statement.executeQuery(); + + while (resultSet.next()) { + records.add(new VectorSearchResult<>( + mapper.mapStorageModeltoRecord(resultSet, + new GetRecordOptions(options.isIncludeVectors())), + resultSet.getDouble("score"))); + } + + return records; + } catch (SQLException | JsonProcessingException e) { + throw new SKException("Failed to search records", e); + } + } + + throw new SKException("Unsupported query type"); + } + public static class Builder - extends JDBCVectorStoreDefaultQueryProvider.Builder { + extends JDBCVectorStoreQueryProvider.Builder { private DataSource dataSource; private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java index 5db2c112..65d088f9 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java @@ -10,6 +10,7 @@ import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.postgresql.util.PGobject; @@ -18,6 +19,7 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.util.List; +import java.util.function.BiFunction; import java.util.function.Function; public class PostgreSQLVectorStoreRecordMapper @@ -29,7 +31,7 @@ public class PostgreSQLVectorStoreRecordMapper * @param storageModelToRecordMapper the function to convert a storage model to a record */ protected PostgreSQLVectorStoreRecordMapper( - Function storageModelToRecordMapper) { + BiFunction storageModelToRecordMapper) { super(null, storageModelToRecordMapper); } @@ -98,7 +100,7 @@ public PostgreSQLVectorStoreRecordMapper build() { } return new PostgreSQLVectorStoreRecordMapper<>( - resultSet -> { + (resultSet, options) -> { try { // Create an ObjectNode to hold the values ObjectNode objectNode = objectMapper.createObjectNode(); @@ -106,8 +108,7 @@ public PostgreSQLVectorStoreRecordMapper build() { // Select fields from the record definition. List fields; ResultSetMetaData metaData = resultSet.getMetaData(); - if (metaData.getColumnCount() == vectorStoreRecordDefinition.getAllFields() - .size()) { + if (options != null && options.includeVectors()) { fields = vectorStoreRecordDefinition.getAllFields(); } else { fields = vectorStoreRecordDefinition.getNonVectorFields(); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java index 1c6c5e42..164f58fa 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java @@ -189,6 +189,9 @@ private String getRedisKey(String key, String collectionName) { private Map addDataFieldNames(List result) { Map dataFields = new HashMap<>(); for (int i = 0; i < result.size(); i++) { + if (result.get(i) == null) { + continue; + } dataFields.put(this.dataFields[i], result.get(i)); } return dataFields; @@ -228,7 +231,7 @@ public Mono> getBatchAsync(List keys, keys.forEach(key -> { String redisKey = getRedisKey(key, collectionName); - if (options == null || options.includeVectors()) { + if (options != null && options.includeVectors()) { // Returns Map with the fields and values responses.add(new AbstractMap.SimpleEntry<>(key, pipeline.hgetAll(redisKey))); } else { @@ -244,19 +247,21 @@ public Mono> getBatchAsync(List keys, try { return Mono.just(responses.stream() .map(entry -> { - if (options == null || options.includeVectors()) { + if (options != null && options.includeVectors()) { // Results directly in a Map return this.vectorStoreRecordMapper .mapStorageModeltoRecord( new AbstractMap.SimpleEntry<>(entry.getKey(), - (Map) entry.getValue().get())); + (Map) entry.getValue().get()), + options); } // Results in a List with the values of the fields return this.vectorStoreRecordMapper .mapStorageModeltoRecord( new AbstractMap.SimpleEntry<>(entry.getKey(), - addDataFieldNames((List) entry.getValue().get()))); + addDataFieldNames((List) entry.getValue().get())), + options); }) .collect(Collectors.toList())); } catch (Exception e) { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java index e4f843ad..1ef47adb 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java @@ -10,6 +10,7 @@ import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import javax.annotation.Nullable; @@ -18,6 +19,8 @@ import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; +import java.util.function.BiFunction; import java.util.function.Function; public class RedisHashSetVectorStoreRecordMapper @@ -25,7 +28,7 @@ public class RedisHashSetVectorStoreRecordMapper private RedisHashSetVectorStoreRecordMapper( Function>> toStorageModelMapper, - Function>, Record> toRecordMapper) { + BiFunction>, GetRecordOptions, Record> toRecordMapper) { super(toStorageModelMapper, toRecordMapper); } @@ -116,7 +119,7 @@ public RedisHashSetVectorStoreRecordMapper build() { "Failure to serialize object, by default the Redis connector uses Jackson, ensure your model object can be serialized by Jackson, i.e the class is visible, has getters, constructor, annotations etc.", e); } - }, storageModel -> { + }, (storageModel, options) -> { try { // Empty map means no record found if (storageModel.getValue() == null || storageModel.getValue().isEmpty()) { @@ -132,22 +135,21 @@ public RedisHashSetVectorStoreRecordMapper build() { storageModel.getValue().get(field.getEffectiveStorageName())); } - for (VectorStoreRecordVectorField field : recordDefinition.getVectorFields()) { - String value = storageModel.getValue().get(field.getEffectiveStorageName()); - - // If vector fields were not requested, skip - if (value == null) { - continue; - } - - Class valueType = field.getFieldType(); - - if (valueType.equals(String.class)) { - jsonNode.put(field.getEffectiveStorageName(), value); - } else { - // Convert the String stored in Redis back to the correct type and then put the JSON node - jsonNode.set(field.getEffectiveStorageName(), - mapper.valueToTree(mapper.readValue(value, valueType))); + if (options != null && options.includeVectors()) { + for (VectorStoreRecordVectorField field : recordDefinition + .getVectorFields()) { + String value = storageModel.getValue() + .get(field.getEffectiveStorageName()); + + Class valueType = field.getFieldType(); + + if (valueType.equals(String.class)) { + jsonNode.put(field.getEffectiveStorageName(), value); + } else { + // Convert the String stored in Redis back to the correct type and then put the JSON node + jsonNode.set(field.getEffectiveStorageName(), + mapper.valueToTree(mapper.readValue(value, valueType))); + } } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java index f99e2bba..80dd2e2f 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java @@ -214,7 +214,7 @@ public Mono getAsync(String key, GetRecordOptions options) { return Mono.defer(() -> { try { Object value; - if (options == null || options.includeVectors()) { + if (options != null && options.includeVectors()) { value = client.jsonGet(redisKey); } else { value = client.jsonGet(redisKey, dataFields); @@ -225,7 +225,7 @@ public Mono getAsync(String key, GetRecordOptions options) { } JsonNode jsonNode; - if (options == null || options.includeVectors()) { + if (options != null && options.includeVectors()) { jsonNode = objectMapper.valueToTree(value); } else { // Remove the $. prefix from every key in the JSON object @@ -233,7 +233,7 @@ public Mono getAsync(String key, GetRecordOptions options) { } return Mono.just(this.vectorStoreRecordMapper - .mapStorageModeltoRecord(new SimpleEntry<>(key, jsonNode))); + .mapStorageModeltoRecord(new SimpleEntry<>(key, jsonNode), options)); } catch (Exception e) { return Mono.error(e); } @@ -255,7 +255,7 @@ public Mono> getBatchAsync(List keys, keys.forEach(key -> { String redisKey = getRedisKey(key, collectionName); - if (options == null || options.includeVectors()) { + if (options != null && options.includeVectors()) { responses.add(new SimpleEntry<>(key, pipeline.jsonGet(redisKey))); } else { responses.add(new SimpleEntry<>(key, pipeline.jsonGet(redisKey, dataFields))); @@ -274,13 +274,14 @@ public Mono> getBatchAsync(List keys, } JsonNode jsonNode; - if (options == null || options.includeVectors()) { + if (options != null && options.includeVectors()) { jsonNode = objectMapper.valueToTree(value); } else { jsonNode = removeRedisPathPrefix((JSONObject) value); } return this.vectorStoreRecordMapper - .mapStorageModeltoRecord(new SimpleEntry<>(entry.getKey(), jsonNode)); + .mapStorageModeltoRecord(new SimpleEntry<>(entry.getKey(), jsonNode), + options); }) .collect(Collectors.toList())); } catch (Exception e) { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java index 76db6862..201d8246 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java @@ -5,11 +5,13 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.AbstractMap; import java.util.Map.Entry; +import java.util.function.BiFunction; import java.util.function.Function; import javax.annotation.Nullable; @@ -18,7 +20,7 @@ public class RedisJsonVectorStoreRecordMapper private RedisJsonVectorStoreRecordMapper( Function> toStorageModelMapper, - Function, Record> toRecordMapper) { + BiFunction, GetRecordOptions, Record> toRecordMapper) { super(toStorageModelMapper, toRecordMapper); } @@ -105,7 +107,7 @@ public RedisJsonVectorStoreRecordMapper build() { "Failure to serialize object, by default the Redis connector uses Jackson, ensure your model object can be serialized by Jackson, i.e the class is visible, has getters, constructor, annotations etc.", e); } - }, storageModel -> { + }, (storageModel, options) -> { try { ObjectNode jsonNode = objectMapper.valueToTree(storageModel.getValue()); // Add the key back to the record diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java index f62a9362..694950d1 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java @@ -7,9 +7,9 @@ public class VectorSearchQuery { private final VectorSearchQueryType queryType; - private final Object searchOptions; + private final VectorSearchOptions searchOptions; - public VectorSearchQuery(VectorSearchQueryType queryType, Object searchOptions) { + public VectorSearchQuery(VectorSearchQueryType queryType, VectorSearchOptions searchOptions) { this.queryType = queryType; this.searchOptions = searchOptions; } @@ -18,7 +18,7 @@ public VectorSearchQueryType getQueryType() { return queryType; } - public Object getSearchOptions() { + public VectorSearchOptions getSearchOptions() { return searchOptions; } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java index d77e9123..44ff95f3 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java @@ -1,9 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.vectorstorage; -import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import javax.annotation.Nullable; +import java.util.function.BiFunction; import java.util.function.Function; /** @@ -15,7 +16,7 @@ public class VectorStoreRecordMapper { @Nullable private final Function recordToStorageModelMapper; - private final Function storageModelToRecordMapper; + private final BiFunction storageModelToRecordMapper; /** * Constructs a new instance of the VectorStoreRecordMapper. @@ -25,7 +26,7 @@ public class VectorStoreRecordMapper { */ protected VectorStoreRecordMapper( @Nullable Function recordToStorageModelMapper, - Function storageModelToRecordMapper) { + BiFunction storageModelToRecordMapper) { this.recordToStorageModelMapper = recordToStorageModelMapper; this.storageModelToRecordMapper = storageModelToRecordMapper; } @@ -45,7 +46,7 @@ public Function getRecordToStorageModelMapper() { * * @return the function to convert a storage model to a record */ - public Function getStorageModelToRecordMapper() { + public BiFunction getStorageModelToRecordMapper() { return storageModelToRecordMapper; } @@ -65,7 +66,7 @@ public StorageModel mapRecordToStorageModel(Record record) { * @param storageModel the storage model to convert * @return the record */ - public Record mapStorageModeltoRecord(StorageModel storageModel) { - return getStorageModelToRecordMapper().apply(storageModel); + public Record mapStorageModeltoRecord(StorageModel storageModel, GetRecordOptions options) { + return getStorageModelToRecordMapper().apply(storageModel, options); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java index 1acf0a13..225210f8 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java @@ -14,6 +14,7 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Represents a definition of a vector store record. @@ -23,12 +24,17 @@ public class VectorStoreRecordDefinition { private final List dataFields; private final List vectorFields; + // Cached information + private final List nonVectorFields; + private final List allFields; + private final Map allFieldsMap; + public VectorStoreRecordKeyField getKeyField() { return keyField; } public List getDataFields() { - return Collections.unmodifiableList(dataFields); + return dataFields; } /** @@ -36,7 +42,7 @@ public List getDataFields() { * @return List of VectorStoreRecordVectorField */ public List getVectorFields() { - return Collections.unmodifiableList(vectorFields); + return vectorFields; } /** @@ -44,22 +50,27 @@ public List getVectorFields() { * @return List of VectorStoreRecordField */ public List getAllFields() { - List fields = new ArrayList<>(); - fields.add(keyField); - fields.addAll(dataFields); - fields.addAll(vectorFields); - return fields; + return allFields; } + /** + * Gets the non-vector fields in the record definition. + * @return List of VectorStoreRecordField + */ public List getNonVectorFields() { - List fields = new ArrayList<>(); - fields.add(keyField); - fields.addAll(dataFields); - return fields; + return nonVectorFields; } - public Map getStorageNames() { - return getAllFields().stream() + public VectorStoreRecordField getField(String fieldName) { + return allFieldsMap.get(fieldName); + } + + /** + * Gets the storage names of the fields in the record definition. + * @return Map of field names to storage names + */ + public Map getFieldStorageNames() { + return allFields.stream() .collect(Collectors.toMap(VectorStoreRecordField::getName, VectorStoreRecordField::getEffectiveStorageName)); } @@ -69,8 +80,16 @@ private VectorStoreRecordDefinition( List dataFields, List vectorFields) { this.keyField = keyField; - this.dataFields = dataFields; - this.vectorFields = vectorFields; + this.dataFields = Collections.unmodifiableList(dataFields); + this.vectorFields = Collections.unmodifiableList(vectorFields); + this.nonVectorFields = Collections + .unmodifiableList(Stream.concat(Stream.of(keyField), dataFields.stream()) + .collect(Collectors.toList())); + this.allFields = Collections + .unmodifiableList(Stream.concat(nonVectorFields.stream(), vectorFields.stream()) + .collect(Collectors.toList())); + this.allFieldsMap = Collections.unmodifiableMap(allFields.stream() + .collect(Collectors.toMap(VectorStoreRecordField::getName, p -> p))); } private static VectorStoreRecordDefinition checkFields( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java index bff9d6a1..735c3f53 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java @@ -9,7 +9,7 @@ public class GetRecordOptions { private final boolean includeVectors; - private GetRecordOptions(boolean includeVectors) { + public GetRecordOptions(boolean includeVectors) { this.includeVectors = includeVectors; } From 5b54ec96b867d0aeea705f53d44addf402edce73 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 12 Sep 2024 11:38:56 -0700 Subject: [PATCH 032/277] Update --- .../data/postgres/PostgreSQLVectorStoreQueryProvider.java | 2 +- .../data/postgres/PostgreSQLVectorStoreRecordMapper.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index e30d9838..d4146e6d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -367,7 +367,7 @@ public List> search(String collectionName, getQueryColumnsFromFields( options.isIncludeVectors() ? recordDefinition.getAllFields() : recordDefinition.getNonVectorFields()), - vectorField.getEffectiveStorageName(), + validateSQLidentifier(vectorField.getEffectiveStorageName()), distanceFunction == null ? PostgreSQLVectorDistanceFunction.L2.getOperator() : distanceFunction.getOperator(), getCollectionTableName(collectionName)); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java index 65d088f9..2e445075 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java @@ -107,7 +107,6 @@ public PostgreSQLVectorStoreRecordMapper build() { // Select fields from the record definition. List fields; - ResultSetMetaData metaData = resultSet.getMetaData(); if (options != null && options.includeVectors()) { fields = vectorStoreRecordDefinition.getAllFields(); } else { From 7ef3600bd95e766a7d2eb4498bf590c7dabebe1d Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 12 Sep 2024 11:48:48 -0700 Subject: [PATCH 033/277] Update docs --- .../data/jdbc/SQLVectorStoreQueryProvider.java | 12 ++++++++++-- .../PostgreSQLVectorStoreQueryProvider.java | 15 +++++++++++---- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java index f5419d78..8210d81c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java @@ -131,9 +131,17 @@ void deleteRecords(String collectionName, List keys, VectorStoreRecordDefinition recordDefinition, DeleteRecordOptions options); /** + * Vector search. + * Executes a vector search query and returns the results. + * The results are mapped to the specified record type using the provided mapper. + * The query is executed against the specified collection. * - * - * @param query + * @param the record type + * @param collectionName the collection name + * @param query the vectorized search query, containing the vector and search options + * @param recordDefinition the record definition + * @param mapper the mapper, responsible for mapping the result set to the record type. + * @return the search results */ List> search(String collectionName, VectorSearchQuery query, diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index d4146e6d..b7942cb9 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -321,10 +321,17 @@ public void upsertRecords(String collectionName, List records, } /** - * @param collectionName - * @param query - * @param recordDefinition - * @param mapper + * Vector search. + * Executes a vector search query and returns the results. + * The results are mapped to the specified record type using the provided mapper. + * The query is executed against the specified collection. + * + * @param the record type + * @param collectionName the collection name + * @param query the vectorized search query, containing the vector and search options + * @param recordDefinition the record definition + * @param mapper the mapper, responsible for mapping the result set to the record type. + * @return the search results */ @Override public List> search(String collectionName, From 8d66e48bc6d9f40057edd1e9b57f1b92288b4bf8 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 12 Sep 2024 14:38:04 -0700 Subject: [PATCH 034/277] Rename GetRecordOptions.isIncludeVectors --- .../AzureAISearchVectorStoreRecordCollection.java | 3 +-- .../data/jdbc/JDBCVectorStoreQueryProvider.java | 2 +- .../connectors/data/jdbc/JDBCVectorStoreRecordMapper.java | 3 +-- .../data/postgres/PostgreSQLVectorStoreRecordMapper.java | 4 +--- .../redis/RedisHashSetVectorStoreRecordCollection.java | 4 ++-- .../data/redis/RedisHashSetVectorStoreRecordMapper.java | 3 +-- .../data/redis/RedisJsonVectorStoreRecordCollection.java | 8 ++++---- .../data/vectorstorage/options/GetRecordOptions.java | 2 +- 8 files changed, 12 insertions(+), 17 deletions(-) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index 00846345..1d51fe16 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -15,7 +15,6 @@ import com.azure.search.documents.models.VectorizableTextQuery; import com.azure.search.documents.models.VectorizedQuery; import com.microsoft.semantickernel.data.vectorsearch.VectorizableSearch; -import com.microsoft.semantickernel.data.vectorsearch.VectorSearch; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorizedSearch; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; @@ -194,7 +193,7 @@ public Mono getAsync( @Nonnull String key, GetRecordOptions options) { // If vectors are not requested, only fetch non-vector fields List selectedFields = null; - if (options == null || !options.includeVectors()) { + if (options == null || !options.isIncludeVectors()) { selectedFields = Collections.unmodifiableList(nonVectorFields); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index 5b3046be..5d9c0581 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -338,7 +338,7 @@ public List getRecords(String collectionName, List keys VectorStoreRecordMapper mapper, GetRecordOptions options) { List fields; - if (options != null && options.includeVectors()) { + if (options != null && options.isIncludeVectors()) { fields = recordDefinition.getAllFields(); } else { fields = recordDefinition.getNonVectorFields(); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java index 162b55bb..8393c9f2 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java @@ -19,7 +19,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.function.BiFunction; -import java.util.function.Function; public class JDBCVectorStoreRecordMapper extends VectorStoreRecordMapper { @@ -115,7 +114,7 @@ public JDBCVectorStoreRecordMapper build() { // Select fields from the record definition. List fields; ResultSetMetaData metaData = resultSet.getMetaData(); - if (options != null && options.includeVectors()) { + if (options != null && options.isIncludeVectors()) { fields = vectorStoreRecordDefinition.getAllFields(); } else { fields = vectorStoreRecordDefinition.getNonVectorFields(); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java index 2e445075..9550ade8 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java @@ -16,11 +16,9 @@ import org.postgresql.util.PGobject; import java.sql.ResultSet; -import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.util.List; import java.util.function.BiFunction; -import java.util.function.Function; public class PostgreSQLVectorStoreRecordMapper extends VectorStoreRecordMapper { @@ -107,7 +105,7 @@ public PostgreSQLVectorStoreRecordMapper build() { // Select fields from the record definition. List fields; - if (options != null && options.includeVectors()) { + if (options != null && options.isIncludeVectors()) { fields = vectorStoreRecordDefinition.getAllFields(); } else { fields = vectorStoreRecordDefinition.getNonVectorFields(); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java index 164f58fa..f1968384 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java @@ -231,7 +231,7 @@ public Mono> getBatchAsync(List keys, keys.forEach(key -> { String redisKey = getRedisKey(key, collectionName); - if (options != null && options.includeVectors()) { + if (options != null && options.isIncludeVectors()) { // Returns Map with the fields and values responses.add(new AbstractMap.SimpleEntry<>(key, pipeline.hgetAll(redisKey))); } else { @@ -247,7 +247,7 @@ public Mono> getBatchAsync(List keys, try { return Mono.just(responses.stream() .map(entry -> { - if (options != null && options.includeVectors()) { + if (options != null && options.isIncludeVectors()) { // Results directly in a Map return this.vectorStoreRecordMapper .mapStorageModeltoRecord( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java index 1ef47adb..adf5e97e 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java @@ -19,7 +19,6 @@ import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; -import java.util.Objects; import java.util.function.BiFunction; import java.util.function.Function; @@ -135,7 +134,7 @@ public RedisHashSetVectorStoreRecordMapper build() { storageModel.getValue().get(field.getEffectiveStorageName())); } - if (options != null && options.includeVectors()) { + if (options != null && options.isIncludeVectors()) { for (VectorStoreRecordVectorField field : recordDefinition .getVectorFields()) { String value = storageModel.getValue() diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java index 80dd2e2f..4f742d97 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java @@ -214,7 +214,7 @@ public Mono getAsync(String key, GetRecordOptions options) { return Mono.defer(() -> { try { Object value; - if (options != null && options.includeVectors()) { + if (options != null && options.isIncludeVectors()) { value = client.jsonGet(redisKey); } else { value = client.jsonGet(redisKey, dataFields); @@ -225,7 +225,7 @@ public Mono getAsync(String key, GetRecordOptions options) { } JsonNode jsonNode; - if (options != null && options.includeVectors()) { + if (options != null && options.isIncludeVectors()) { jsonNode = objectMapper.valueToTree(value); } else { // Remove the $. prefix from every key in the JSON object @@ -255,7 +255,7 @@ public Mono> getBatchAsync(List keys, keys.forEach(key -> { String redisKey = getRedisKey(key, collectionName); - if (options != null && options.includeVectors()) { + if (options != null && options.isIncludeVectors()) { responses.add(new SimpleEntry<>(key, pipeline.jsonGet(redisKey))); } else { responses.add(new SimpleEntry<>(key, pipeline.jsonGet(redisKey, dataFields))); @@ -274,7 +274,7 @@ public Mono> getBatchAsync(List keys, } JsonNode jsonNode; - if (options != null && options.includeVectors()) { + if (options != null && options.isIncludeVectors()) { jsonNode = objectMapper.valueToTree(value); } else { jsonNode = removeRedisPathPrefix((JSONObject) value); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java index 735c3f53..0025394a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java @@ -52,7 +52,7 @@ public GetRecordOptions build() { * * @return whether to include vectors */ - public boolean includeVectors() { + public boolean isIncludeVectors() { return includeVectors; } } From 1f92dc4d4e61d8b3069f772305008a2255f7843c Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 12 Sep 2024 19:05:30 -0700 Subject: [PATCH 035/277] Add vector search for JDBC vector store --- .../tests/connectors/memory/Hotel.java | 46 ++++-- .../JDBCVectorStoreRecordCollectionTest.java | 59 ++++--- ...ashSetVectorStoreRecordCollectionTest.java | 18 +-- ...isJsonVectorStoreRecordCollectionTest.java | 18 +-- .../services/textembedding/Embedding.java | 61 -------- ...rchVectorStoreCollectionCreateMapping.java | 4 +- .../jdbc/JDBCVectorStoreQueryProvider.java | 144 +++++++++++++++++- .../jdbc/JDBCVectorStoreRecordCollection.java | 25 +-- .../PostgreSQLVectorDistanceFunction.java | 4 +- ...disVectorStoreCollectionCreateMapping.java | 4 +- .../data/vectorsearch}/VectorOperations.java | 60 +++++++- .../definition/DistanceFunction.java | 10 +- .../VectorStoreRecordDefinition.java | 4 + 13 files changed, 308 insertions(+), 149 deletions(-) rename {semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding => semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch}/VectorOperations.java (61%) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java index a733f376..13ca8d17 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java @@ -11,24 +11,37 @@ public class Hotel { @VectorStoreRecordKeyAttribute private final String id; + @VectorStoreRecordDataAttribute private final String name; + @VectorStoreRecordDataAttribute private final int code; + @JsonProperty("summary") @VectorStoreRecordDataAttribute() private final String description; - @JsonProperty("summaryEmbedding") - @VectorStoreRecordVectorAttribute(dimensions = 8) - private final List descriptionEmbedding; + + @JsonProperty("summaryEmbedding1") + @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "euclidean") + private final List euclidean; + + @JsonProperty("summaryEmbedding2") + @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "cosineDistance") + private final List cosineDistance; + + @JsonProperty("summaryEmbedding3") + @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "dotProduct") + private final List dotProduct; + @JsonProperty("indexedSummaryEmbedding") - @VectorStoreRecordVectorAttribute(dimensions = 8, indexKind = "hnsw", distanceFunction = "cosine") - private final List indexedDescriptionEmbedding; + @VectorStoreRecordVectorAttribute(dimensions = 8, indexKind = "hnsw", distanceFunction = "euclidean") + private final List indexedEuclidean; @VectorStoreRecordDataAttribute private double rating; public Hotel() { - this(null, null, 0, null, null, null, 0.0); + this(null, null, 0, null, null, null, null, null, 0.0); } @JsonCreator @@ -37,15 +50,19 @@ public Hotel( @JsonProperty("name") String name, @JsonProperty("code") int code, @JsonProperty("summary") String description, - @JsonProperty("summaryEmbedding") List descriptionEmbedding, - @JsonProperty("indexedSummaryEmbedding") List indexedDescriptionEmbedding, + @JsonProperty("summaryEmbedding1") List euclidean, + @JsonProperty("summaryEmbedding2") List cosineDistance, + @JsonProperty("summaryEmbedding3") List dotProduct, + @JsonProperty("indexedSummaryEmbedding") List indexedEuclidean, @JsonProperty("rating") double rating) { this.id = id; this.name = name; this.code = code; this.description = description; - this.descriptionEmbedding = descriptionEmbedding; - this.indexedDescriptionEmbedding = indexedDescriptionEmbedding; + this.euclidean = euclidean; + this.cosineDistance = euclidean; + this.dotProduct = euclidean; + this.indexedEuclidean = euclidean; this.rating = rating; } @@ -65,11 +82,12 @@ public String getDescription() { return description; } - public List getDescriptionEmbedding() { - return descriptionEmbedding; + public List getEuclidean() { + return euclidean; } - public List getIndexedDescriptionEmbedding() { - return indexedDescriptionEmbedding; + + public List getIndexedEuclidean() { + return indexedEuclidean; } public double getRating() { diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java index 75f26c17..56a256ca 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -12,7 +12,9 @@ import com.mysql.cj.jdbc.MysqlDataSource; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; import org.postgresql.ds.PGSimpleDataSource; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.containers.PostgreSQLContainer; @@ -25,6 +27,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -99,16 +102,11 @@ private List getHotels() { ArrayList embeddings = new ArrayList<>(); return List.of( - new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), - Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f),4.0), - new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), - Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f),3.0), - new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), - Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f),5.0), - new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), - Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f),4.0), - new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), - Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f),5.0) + new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f),null, null, null, 4.0), + new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f),null, null, null, 3.0), + new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f),null, null, null, 5.0), + new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f),null, null, null, 2.0), + new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f),null, null, null, 1.0) ); } @@ -263,7 +261,7 @@ public void getWithNoVectors(QueryProvider provider) { Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), options).block(); assertNotNull(retrievedHotel); assertEquals(hotel.getId(), retrievedHotel.getId()); - assertNull(retrievedHotel.getDescriptionEmbedding()); + assertNull(retrievedHotel.getEuclidean()); } options = GetRecordOptions.builder() @@ -274,7 +272,7 @@ public void getWithNoVectors(QueryProvider provider) { Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), options).block(); assertNotNull(retrievedHotel); assertEquals(hotel.getId(), retrievedHotel.getId()); - assertNotNull(retrievedHotel.getDescriptionEmbedding()); + assertNotNull(retrievedHotel.getEuclidean()); } } @@ -301,7 +299,7 @@ public void getBatchWithNoVectors(QueryProvider provider) { assertEquals(hotels.size(), retrievedHotels.size()); for (Hotel hotel : retrievedHotels) { - assertNull(hotel.getDescriptionEmbedding()); + assertNull(hotel.getEuclidean()); } options = GetRecordOptions.builder() @@ -313,15 +311,26 @@ public void getBatchWithNoVectors(QueryProvider provider) { assertEquals(hotels.size(), retrievedHotels.size()); for (Hotel hotel : retrievedHotels) { - assertNotNull(hotel.getDescriptionEmbedding()); + assertNotNull(hotel.getEuclidean()); } } + private static Stream provideParameters() { + return Stream.of( + Arguments.of(QueryProvider.MySQL, "euclidean"), + Arguments.of(QueryProvider.MySQL, "cosineDistance"), + Arguments.of(QueryProvider.MySQL, "dotProduct"), + Arguments.of(QueryProvider.PostgreSQL, "euclidean"), + Arguments.of(QueryProvider.PostgreSQL, "cosineDistance"), + Arguments.of(QueryProvider.PostgreSQL, "dotProduct") + ); + } - @Test - public void postgresExactSearch() { + @ParameterizedTest + @MethodSource("provideParameters") + public void exactSearch(QueryProvider provider, String embeddingName) { String collectionName = "search"; - JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(QueryProvider.PostgreSQL, collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -342,16 +351,17 @@ public void postgresExactSearch() { assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); } - @Test - public void postgresApproximateSearch() { + @ParameterizedTest + @EnumSource(QueryProvider.class) + public void approximateSearch(QueryProvider provider) { String collectionName = "searchWithIndex"; - JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(QueryProvider.PostgreSQL, collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); VectorSearchOptions options = VectorSearchOptions.builder() - .withVectorFieldName("indexedDescriptionEmbedding") + .withVectorFieldName("indexedEuclidean") .withLimit(5) .build(); @@ -364,8 +374,9 @@ public void postgresApproximateSearch() { assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); } + // MySQL will always return the vectors as they're needed to compute the distances @Test - public void searchIncludeAndNotIncludeVectors() { + public void postgresSearchIncludeAndNotIncludeVectors() { String collectionName = "searchIncludeAndNotIncludeVectors"; JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(QueryProvider.PostgreSQL, collectionName); @@ -379,7 +390,7 @@ public void searchIncludeAndNotIncludeVectors() { assertEquals(3, results.size()); // The third hotel should be the most similar assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); - assertNull(results.get(0).getRecord().getDescriptionEmbedding()); + assertNull(results.get(0).getRecord().getEuclidean()); VectorSearchOptions options = VectorSearchOptions.builder() .withIncludeVectors(true) @@ -390,6 +401,6 @@ public void searchIncludeAndNotIncludeVectors() { assertEquals(3, results.size()); // The third hotel should be the most similar assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); - assertNotNull(results.get(0).getRecord().getDescriptionEmbedding()); + assertNotNull(results.get(0).getRecord().getEuclidean()); } } diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index e812651c..4b11aa9d 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -97,11 +97,11 @@ private RedisHashSetVectorStoreRecordCollection buildrecordCollection(@No private List getHotels() { return List.of( - new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(1.0f, 2.0f, 3.0f), Arrays.asList(1.0f, 2.0f, 3.0f),4.0), - new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(1.0f, 2.0f, 3.0f), Arrays.asList(1.0f, 2.0f, 3.0f),3.0), - new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(1.0f, 2.0f, 3.0f), Arrays.asList(1.0f, 2.0f, 3.0f),5.0), - new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(1.0f, 2.0f, 3.0f), Arrays.asList(1.0f, 2.0f, 3.0f),4.0), - new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(1.0f, 2.0f, 3.0f), Arrays.asList(1.0f, 2.0f, 3.0f),5.0) + new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(1.0f, 2.0f, 3.0f), null, null, null, 4.0), + new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(1.0f, 2.0f, 3.0f), null, null, null, 3.0), + new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(1.0f, 2.0f, 3.0f), null, null, null, 2.0), + new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(1.0f, 2.0f, 3.0f), null, null, null, 4.0), + new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(1.0f, 2.0f, 3.0f), null, null, null, 5.0) ); } @@ -235,7 +235,7 @@ public void getAsyncWithVectors(RecordCollectionOptions options) { for (Hotel hotel : hotels) { Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), new GetRecordOptions(true)).block(); assertNotNull(retrievedHotel); - assertNotNull(retrievedHotel.getDescriptionEmbedding()); + assertNotNull(retrievedHotel.getEuclidean()); assertEquals(hotel.getId(), retrievedHotel.getId()); assertEquals(hotel.getDescription(), retrievedHotel.getDescription()); } @@ -259,7 +259,7 @@ public void getBatchAsyncWithVectors(RecordCollectionOptions options) { for (int i = 0; i < hotels.size(); i++) { assertEquals(hotels.get(i).getId(), retrievedHotels.get(i).getId()); assertEquals(hotels.get(i).getDescription(), retrievedHotels.get(i).getDescription()); - assertNotNull(retrievedHotels.get(i).getDescriptionEmbedding()); + assertNotNull(retrievedHotels.get(i).getEuclidean()); } } @@ -275,7 +275,7 @@ public void getAsyncWithNoVectors(RecordCollectionOptions options) { for (Hotel hotel : hotels) { Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), getRecordOptions).block(); assertNotNull(retrievedHotel); - assertNull(retrievedHotel.getDescriptionEmbedding()); + assertNull(retrievedHotel.getEuclidean()); assertEquals(hotel.getId(), retrievedHotel.getId()); assertEquals(hotel.getDescription(), retrievedHotel.getDescription()); } @@ -300,7 +300,7 @@ public void getBatchAsyncWithNoVectors(RecordCollectionOptions options) { for (int i = 0; i < hotels.size(); i++) { assertEquals(hotels.get(i).getId(), retrievedHotels.get(i).getId()); assertEquals(hotels.get(i).getDescription(), retrievedHotels.get(i).getDescription()); - assertNull(retrievedHotels.get(i).getDescriptionEmbedding()); + assertNull(retrievedHotels.get(i).getEuclidean()); } } } diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index aa5fd43d..bdd99229 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -98,11 +98,11 @@ private RedisJsonVectorStoreRecordCollection buildrecordCollection(@Nonnu private List getHotels() { return List.of( - new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(1.0f, 2.0f, 3.0f), Arrays.asList(1.0f, 2.0f, 3.0f),4.0), - new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(1.0f, 2.0f, 3.0f), Arrays.asList(1.0f, 2.0f, 3.0f),3.0), - new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(1.0f, 2.0f, 3.0f), Arrays.asList(1.0f, 2.0f, 3.0f),5.0), - new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(1.0f, 2.0f, 3.0f), Arrays.asList(1.0f, 2.0f, 3.0f),4.0), - new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(1.0f, 2.0f, 3.0f), Arrays.asList(1.0f, 2.0f, 3.0f),5.0) + new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(1.0f, 2.0f, 3.0f), null, null, null, 4.0), + new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(1.0f, 2.0f, 3.0f), null, null, null, 3.0), + new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(1.0f, 2.0f, 3.0f), null, null, null, 2.0), + new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(1.0f, 2.0f, 3.0f), null, null, null, 4.0), + new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(1.0f, 2.0f, 3.0f), null, null, null, 5.0) ); } @@ -236,7 +236,7 @@ public void getAsyncWithVectors(RecordCollectionOptions options) { for (Hotel hotel : hotels) { Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), new GetRecordOptions(true)).block(); assertNotNull(retrievedHotel); - assertNotNull(retrievedHotel.getDescriptionEmbedding()); + assertNotNull(retrievedHotel.getEuclidean()); assertEquals(hotel.getId(), retrievedHotel.getId()); assertEquals(hotel.getDescription(), retrievedHotel.getDescription()); } @@ -260,7 +260,7 @@ public void getBatchAsyncWithVectors(RecordCollectionOptions options) { for (int i = 0; i < hotels.size(); i++) { assertEquals(hotels.get(i).getId(), retrievedHotels.get(i).getId()); assertEquals(hotels.get(i).getDescription(), retrievedHotels.get(i).getDescription()); - assertNotNull(retrievedHotels.get(i).getDescriptionEmbedding()); + assertNotNull(retrievedHotels.get(i).getEuclidean()); } } @@ -276,7 +276,7 @@ public void getAsyncWithNoVectors(RecordCollectionOptions options) { for (Hotel hotel : hotels) { Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), getRecordOptions).block(); assertNotNull(retrievedHotel); - assertNull(retrievedHotel.getDescriptionEmbedding()); + assertNull(retrievedHotel.getEuclidean()); assertEquals(hotel.getId(), retrievedHotel.getId()); assertEquals(hotel.getDescription(), retrievedHotel.getDescription()); } @@ -301,7 +301,7 @@ public void getBatchAsyncWithNoVectors(RecordCollectionOptions options) { for (int i = 0; i < hotels.size(); i++) { assertEquals(hotels.get(i).getId(), retrievedHotels.get(i).getId()); assertEquals(hotels.get(i).getDescription(), retrievedHotels.get(i).getDescription()); - assertNull(retrievedHotels.get(i).getDescriptionEmbedding()); + assertNull(retrievedHotels.get(i).getEuclidean()); } } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java index 0a11e869..66ea2791 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java @@ -3,7 +3,6 @@ import javax.annotation.Nonnull; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -59,64 +58,4 @@ public Embedding(@Nonnull float[] vector) { public List getVector() { return Collections.unmodifiableList(this.vector); } - - /** - * Calculates the dot product of this {@code Embedding} with another. - * - * @param other The other {@code Embedding} to compute the dot product with - * @return The dot product between the {@code Embedding} vectors - */ - public float dot(@Nonnull Embedding other) { - Objects.requireNonNull(other); - return VectorOperations.dot(this.vector, other.getVector()); - } - - /** - * Calculates the Euclidean length of this vector. - * - * @return Euclidean length - */ - public float euclideanLength() { - return VectorOperations.euclideanLength(this.vector); - } - - /** - * Calculates the cosine similarity of this vector with another. - * - * @param other The other vector to compute cosine similarity with. - * @return Cosine similarity between vectors - */ - public float cosineSimilarity(@Nonnull Embedding other) { - Objects.requireNonNull(other); - return VectorOperations.cosineSimilarity(this.vector, other.getVector()); - } - - /** - * Multiply the {@code Embedding} vector by a multiplier. - * - * @param multiplier The multiplier to multiply the {@code Embedding} vector by - * @return A new {@code Embedding} with the vector multiplied by the multiplier - */ - public Embedding multiply(float multiplier) { - return new Embedding(VectorOperations.multiply(this.vector, multiplier)); - } - - /** - * Divide the {@code Embedding} vector by a divisor. - * - * @param divisor The divisor to divide the {@code Embedding} vector by - * @return A new {@code Embedding} with the vector divided by the divisor - */ - public Embedding divide(float divisor) { - return new Embedding(VectorOperations.divide(this.vector, divisor)); - } - - /** - * Normalizes the underlying vector, such that the Euclidean length is 1. - * - * @return A new {@code Embedding} with the normalized vector - */ - public Embedding normalize() { - return new Embedding(VectorOperations.normalize(this.vector)); - } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java index 724fe185..2c733b33 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java @@ -36,11 +36,11 @@ private static VectorSearchAlgorithmMetric getAlgorithmMetric( } switch (vectorField.getDistanceFunction()) { - case COSINE: + case COSINE_SIMILARITY: return VectorSearchAlgorithmMetric.COSINE; case DOT_PRODUCT: return VectorSearchAlgorithmMetric.DOT_PRODUCT; - case EUCLIDEAN: + case EUCLIDEAN_DISTANCE: return VectorSearchAlgorithmMetric.EUCLIDEAN; default: throw new SKException( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index 5d9c0581..2cc6309f 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -1,16 +1,27 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.microsoft.semantickernel.data.vectorsearch.VectorOperations; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizedSearchQuery; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import com.microsoft.semantickernel.exceptions.SKException; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; +import com.microsoft.semantickernel.services.OrderedAIServiceSelector; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; import javax.sql.DataSource; @@ -22,6 +33,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -30,6 +42,8 @@ public class JDBCVectorStoreQueryProvider implements SQLVectorStoreQueryProvider { + private static final Logger LOGGER = LoggerFactory + .getLogger(JDBCVectorStoreQueryProvider.class); private final Map, String> supportedKeyTypes; private final Map, String> supportedDataTypes; @@ -234,6 +248,14 @@ public boolean collectionExists(String collectionName) { public void createCollection(String collectionName, VectorStoreRecordDefinition recordDefinition) { + // No approximate search is supported in JDBCVectorStoreQueryProvider + if (recordDefinition.getVectorFields().stream() + .anyMatch(field -> field.getIndexKind() != null)) { + LOGGER + .warn(String.format("Indexes are not supported in %s. Ignoring indexKind property.", + this.getClass().getName())); + } + String createStorageTable = formatQuery("CREATE TABLE IF NOT EXISTS %s (" + "%s VARCHAR(255) PRIMARY KEY, " + "%s, " @@ -405,22 +427,132 @@ public void deleteRecords(String collectionName, List keys, } } + protected List getAllRecords(String collectionName, + VectorStoreRecordDefinition recordDefinition, + VectorStoreRecordMapper mapper, GetRecordOptions options) { + List fields; + if (options.isIncludeVectors()) { + fields = recordDefinition.getAllFields(); + } else { + fields = recordDefinition.getNonVectorFields(); + } + + String selectQuery = formatQuery("SELECT %s FROM %s", + getQueryColumnsFromFields(fields), + getCollectionTableName(collectionName)); + + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(selectQuery)) { + List records = new ArrayList<>(); + ResultSet resultSet = statement.executeQuery(); + while (resultSet.next()) { + records.add(mapper.mapStorageModeltoRecord(resultSet, options)); + } + + return Collections.unmodifiableList(records); + } catch (SQLException e) { + throw new SKException("Failed to set statement values", e); + } + } + /** - * Searches for records. + * Vector search. + * Executes a vector search query and returns the results. + * The results are mapped to the specified record type using the provided mapper. + * The query is executed against the specified collection. * + * @param the record type * @param collectionName the collection name - * @param query the query + * @param query the vectorized search query, containing the vector and search options * @param recordDefinition the record definition - * @param mapper the mapper - * @param the record type + * @param mapper the mapper, responsible for mapping the result set to the record type. * @return the search results */ @Override public List> search(String collectionName, VectorSearchQuery query, VectorStoreRecordDefinition recordDefinition, VectorStoreRecordMapper mapper) { - throw new UnsupportedOperationException( - "Search is not supported. Try with a specific query provider."); + if (recordDefinition.getVectorFields().isEmpty()) { + throw new SKException("No vector fields defined. Cannot perform vector search"); + } + + if (query instanceof VectorizedSearchQuery) { + VectorizedSearchQuery vectorizedSearchQuery = (VectorizedSearchQuery) query; + VectorSearchOptions options = query.getSearchOptions(); + + VectorStoreRecordVectorField firstVectorField = recordDefinition.getVectorFields() + .get(0); + if (options == null) { + options = VectorSearchOptions.createDefault(firstVectorField.getName()); + } + + VectorStoreRecordVectorField vectorField = options.getVectorFieldName() == null + ? firstVectorField + : (VectorStoreRecordVectorField) recordDefinition + .getField(options.getVectorFieldName()); + + List records = getAllRecords(collectionName, recordDefinition, mapper, + new GetRecordOptions(true)); + List> results = new ArrayList<>(); + + DistanceFunction distanceFunction = vectorField.getDistanceFunction() == null + ? DistanceFunction.EUCLIDEAN_DISTANCE + : vectorField.getDistanceFunction(); + + for (Record record : records) { + List vector; + try { + String json = new ObjectMapper().writeValueAsString(record); + ArrayNode arrayNode = (ArrayNode) new ObjectMapper().readTree(json) + .get(vectorField.getEffectiveStorageName()); + + vector = Stream.iterate(0, i -> i + 1) + .limit(arrayNode.size()) + .map(i -> arrayNode.get(i).floatValue()) + .collect(Collectors.toList()); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + double score; + switch (distanceFunction) { + case COSINE_SIMILARITY: + score = VectorOperations.cosineSimilarity(vectorizedSearchQuery.getVector(), + vector); + break; + case COSINE_DISTANCE: + score = VectorOperations.cosineDistance(vectorizedSearchQuery.getVector(), + vector); + break; + case EUCLIDEAN_DISTANCE: + score = VectorOperations + .euclideanDistance(vectorizedSearchQuery.getVector(), vector); + break; + case DOT_PRODUCT: + score = VectorOperations.dot(vectorizedSearchQuery.getVector(), vector); + break; + default: + throw new SKException("Unsupported distance function"); + } + + results.add(new VectorSearchResult<>(record, score)); + } + + Comparator> comparator = Comparator + .comparingDouble(VectorSearchResult::getScore); + // Higher scores are better + if (distanceFunction == DistanceFunction.COSINE_SIMILARITY + || distanceFunction == DistanceFunction.DOT_PRODUCT) { + comparator = comparator.reversed(); + } + return results.stream() + .sorted(comparator) + .skip(options.getOffset()) + .limit(options.getLimit()) + .collect(Collectors.toList()); + } + + throw new SKException("Unsupported query type"); } /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java index 21b87418..2247264c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -21,6 +21,7 @@ import java.sql.ResultSet; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.sql.DataSource; @@ -175,6 +176,8 @@ public Mono deleteCollectionAsync() { */ @Override public Mono getAsync(String key, GetRecordOptions options) { + Objects.requireNonNull(key, "key is required"); + return this.getBatchAsync(Collections.singletonList(key), options) .mapNotNull(records -> { if (records.isEmpty()) { @@ -193,12 +196,13 @@ public Mono getAsync(String key, GetRecordOptions options) { * @throws SKException if the operation fails */ @Override - public Mono> getBatchAsync(List keys, GetRecordOptions options) { + public Mono> getBatchAsync(@Nonnull List keys, GetRecordOptions options) { + Objects.requireNonNull(keys, "keys is required"); + return Mono.fromCallable( - () -> { - return queryProvider.getRecords(this.collectionName, keys, recordDefinition, - vectorStoreRecordMapper, options); - }).subscribeOn(Schedulers.boundedElastic()); + () -> queryProvider.getRecords(this.collectionName, keys, recordDefinition, + vectorStoreRecordMapper, options)) + .subscribeOn(Schedulers.boundedElastic()); } protected String getKeyFromRecord(Record data) { @@ -222,6 +226,8 @@ protected String getKeyFromRecord(Record data) { */ @Override public Mono upsertAsync(Record data, UpsertRecordOptions options) { + Objects.requireNonNull(data, "data is required"); + return this.upsertBatchAsync(Collections.singletonList(data), options) .mapNotNull(keys -> { if (keys.isEmpty()) { @@ -241,6 +247,8 @@ public Mono upsertAsync(Record data, UpsertRecordOptions options) { */ @Override public Mono> upsertBatchAsync(List data, UpsertRecordOptions options) { + Objects.requireNonNull(data, "data is required"); + return Mono.fromCallable( () -> { queryProvider.upsertRecords(this.collectionName, data, recordDefinition, options); @@ -293,10 +301,9 @@ public Mono prepareAsync() { @Override public Mono>> searchAsync(VectorSearchQuery query) { return Mono.fromCallable( - () -> { - return queryProvider.search(this.collectionName, query, recordDefinition, - vectorStoreRecordMapper); - }).subscribeOn(Schedulers.boundedElastic()); + () -> queryProvider.search(this.collectionName, query, recordDefinition, + vectorStoreRecordMapper)) + .subscribeOn(Schedulers.boundedElastic()); } /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java index e36104c6..ae2a6bff 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java @@ -29,9 +29,9 @@ public static PostgreSQLVectorDistanceFunction fromDistanceFunction(DistanceFunc } switch (function) { - case EUCLIDEAN: + case EUCLIDEAN_DISTANCE: return L2; - case COSINE: + case COSINE_DISTANCE: return COSINE; case DOT_PRODUCT: return INNER_PRODUCT; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java index 7e139f14..c8d9f39e 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java @@ -37,11 +37,11 @@ private static String getAlgorithmMetric( } switch (vectorField.getDistanceFunction()) { - case COSINE: + case COSINE_SIMILARITY: return RedisVectorDistanceMetric.COSINE; case DOT_PRODUCT: return RedisVectorDistanceMetric.DOT_PRODUCT; - case EUCLIDEAN: + case EUCLIDEAN_DISTANCE: return RedisVectorDistanceMetric.EUCLIDEAN; default: throw new SKException( diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/VectorOperations.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java similarity index 61% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/VectorOperations.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java index ea0b77e0..d1e36fab 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/VectorOperations.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java @@ -1,12 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.services.textembedding; +package com.microsoft.semantickernel.data.vectorsearch; import javax.annotation.Nonnull; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; -final class VectorOperations { +public final class VectorOperations { /** * Calculates the cosine similarity of two vectors. The vectors must be equal in length and have @@ -16,7 +16,7 @@ final class VectorOperations { * @param y Second vector, which is not modified * @return The cosine similarity of the two vectors */ - static float cosineSimilarity(@Nonnull List x, @Nonnull List y) { + public static float cosineSimilarity(@Nonnull List x, @Nonnull List y) { Objects.requireNonNull(x); Objects.requireNonNull(y); @@ -35,6 +35,43 @@ static float cosineSimilarity(@Nonnull List x, @Nonnull List y) { return dotProduct / (float) (Math.sqrt(normX) * Math.sqrt(normY)); } + /** + * Calculates the cosine distance of two vectors. The vectors must be equal in length and have + * non-zero norm. + * + * @param x First vector, which is not modified + * @param y Second vector, which is not modified + * @return The cosine distance of the two vectors + */ + public static double cosineDistance(List x, List y) { + return 1.0 - cosineSimilarity(x, y); + } + + /** + * Calculates the Euclidean distance between two vectors. + * + * @param x First vector, which is not modified + * @param y Second vector, which is not modified + * @return The Euclidean distance between the two vectors + */ + public static float euclideanDistance(@Nonnull List x, @Nonnull List y) { + Objects.requireNonNull(x); + Objects.requireNonNull(y); + + if (x.size() != y.size()) { + throw new IllegalArgumentException("Vectors lengths must be equal"); + } + + float sumOfSquaredDifferences = 0.0f; + + for (int i = 0; i < x.size(); ++i) { + float difference = x.get(i) - y.get(i); + sumOfSquaredDifferences += difference * difference; + } + + return (float) Math.sqrt(sumOfSquaredDifferences); + } + /** * Divides the elements of the vector by the divisor. * @@ -42,7 +79,7 @@ static float cosineSimilarity(@Nonnull List x, @Nonnull List y) { * @param divisor Divisor to apply to each element of the vector * @return A new vector with the elements divided by the divisor */ - static List divide(@Nonnull List vector, float divisor) { + public static List divide(@Nonnull List vector, float divisor) { Objects.requireNonNull(vector); if (Float.isNaN(divisor)) { throw new IllegalArgumentException("Divisor cannot be NaN"); @@ -54,7 +91,14 @@ static List divide(@Nonnull List vector, float divisor) { return vector.stream().map(x -> x / divisor).collect(Collectors.toList()); } - static float dot(@Nonnull List x, @Nonnull List y) { + /** + * Calculates the dot product of two vectors. + * + * @param x First vector, which is not modified + * @param y Second vector, which is not modified + * @return The dot product of the two vectors + */ + public static float dot(@Nonnull List x, @Nonnull List y) { Objects.requireNonNull(x); Objects.requireNonNull(y); @@ -76,7 +120,7 @@ static float dot(@Nonnull List x, @Nonnull List y) { * @param vector Vector to calculate the length of, which is not modified * @return The Euclidean length of the vector */ - static float euclideanLength(@Nonnull List vector) { + public static float euclideanLength(@Nonnull List vector) { Objects.requireNonNull(vector); return (float) Math.sqrt(dot(vector, vector)); } @@ -88,7 +132,7 @@ static float euclideanLength(@Nonnull List vector) { * @param multiplier Multiplier to apply to each element of the vector * @return A new vector with the elements multiplied by the multiplier */ - static List multiply(@Nonnull List vector, float multiplier) { + public static List multiply(@Nonnull List vector, float multiplier) { Objects.requireNonNull(vector); if (Float.isNaN(multiplier)) { throw new IllegalArgumentException("Multiplier cannot be NaN"); @@ -106,7 +150,7 @@ static List multiply(@Nonnull List vector, float multiplier) { * @param vector Vector to normalize, which is not modified * @return A new, normalized vector */ - static List normalize(@Nonnull List vector) { + public static List normalize(@Nonnull List vector) { Objects.requireNonNull(vector); return divide(vector, euclideanLength(vector)); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java index 62e6a3f6..aacdbba6 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java @@ -5,15 +5,19 @@ public enum DistanceFunction { /** * Cosine (angular) similarity function. */ - COSINE("cosine"), + COSINE_SIMILARITY("cosineSimilarity"), + /** + * Cosine distance function. 1 - cosine similarity. + */ + COSINE_DISTANCE("cosineDistance"), /** * Dot product between two vectors. */ DOT_PRODUCT("dotProduct"), /** - * Euclidean distance function. Also known as L2 norm. + * Euclidean distance function. Also known as L2 distance. */ - EUCLIDEAN("euclidean"); + EUCLIDEAN_DISTANCE("euclidean"); private final String value; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java index 225210f8..4c9bb94a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java @@ -5,6 +5,7 @@ import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.exceptions.SKException; import java.lang.reflect.Field; import java.util.ArrayList; @@ -62,6 +63,9 @@ public List getNonVectorFields() { } public VectorStoreRecordField getField(String fieldName) { + if (!allFieldsMap.containsKey(fieldName)) { + throw new SKException("Field not found: " + fieldName); + } return allFieldsMap.get(fieldName); } From 25fc4f82fc2ad68e071659e47149dcab77e212b3 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 12 Sep 2024 19:28:13 -0700 Subject: [PATCH 036/277] Fix redis tests --- ...ashSetVectorStoreRecordCollectionTest.java | 24 ++++++++++++++++--- ...isJsonVectorStoreRecordCollectionTest.java | 24 ++++++++++++++++--- ...disVectorStoreCollectionCreateMapping.java | 6 ++--- 3 files changed, 45 insertions(+), 9 deletions(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index 4b11aa9d..f6849d00 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -69,10 +69,28 @@ static void setup() { .withFieldType(String.class) .build()); fields.add(VectorStoreRecordVectorField.builder() - .withName("descriptionEmbedding") - .withStorageName("summaryEmbedding") + .withName("euclidean") + .withStorageName("summaryEmbedding1") .withFieldType(List.class) - .withDimensions(768) + .withDimensions(8) + .build()); + fields.add(VectorStoreRecordVectorField.builder() + .withName("cosineDistance") + .withStorageName("summaryEmbedding2") + .withFieldType(List.class) + .withDimensions(8) + .build()); + fields.add(VectorStoreRecordVectorField.builder() + .withName("dotProduct") + .withStorageName("summaryEmbedding3") + .withFieldType(List.class) + .withDimensions(8) + .build()); + fields.add(VectorStoreRecordVectorField.builder() + .withName("indexedEuclidean") + .withStorageName("indexedSummaryEmbedding") + .withFieldType(List.class) + .withDimensions(8) .build()); fields.add(VectorStoreRecordDataField.builder() .withName("rating") diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index bdd99229..04effca7 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -70,10 +70,28 @@ static void setup() { .withFieldType(String.class) .build()); fields.add(VectorStoreRecordVectorField.builder() - .withName("descriptionEmbedding") - .withStorageName("summaryEmbedding") + .withName("euclidean") + .withStorageName("summaryEmbedding1") .withFieldType(List.class) - .withDimensions(768) + .withDimensions(8) + .build()); + fields.add(VectorStoreRecordVectorField.builder() + .withName("cosineDistance") + .withStorageName("summaryEmbedding2") + .withFieldType(List.class) + .withDimensions(8) + .build()); + fields.add(VectorStoreRecordVectorField.builder() + .withName("dotProduct") + .withStorageName("summaryEmbedding3") + .withFieldType(List.class) + .withDimensions(8) + .build()); + fields.add(VectorStoreRecordVectorField.builder() + .withName("indexedEuclidean") + .withStorageName("indexedSummaryEmbedding") + .withFieldType(List.class) + .withDimensions(8) .build()); fields.add(VectorStoreRecordDataField.builder() .withName("rating") diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java index c8d9f39e..f84a8e9a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java @@ -33,11 +33,11 @@ public class RedisVectorStoreCollectionCreateMapping { private static String getAlgorithmMetric( VectorStoreRecordVectorField vectorField) { if (vectorField.getDistanceFunction() == null) { - return RedisVectorDistanceMetric.COSINE; + return RedisVectorDistanceMetric.EUCLIDEAN; } switch (vectorField.getDistanceFunction()) { - case COSINE_SIMILARITY: + case COSINE_DISTANCE: return RedisVectorDistanceMetric.COSINE; case DOT_PRODUCT: return RedisVectorDistanceMetric.DOT_PRODUCT; @@ -134,7 +134,7 @@ static class RedisIndexSchemaParams { static class RedisVectorDistanceMetric { public static final String EUCLIDEAN = "L2"; public static final String DOT_PRODUCT = "IP"; - public static final String COSINE = "COSINE"; + public static final String COSINE = "COSINE"; // Cosine distance } } From 93889d8f4a4f7dbad373a311ba5b9700e7423b7f Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Fri, 13 Sep 2024 13:59:29 +0100 Subject: [PATCH 037/277] Fixes from review --- .../chatcompletion/GeminiChatCompletion.java | 46 ++++++++++++++++--- .../GeminiStreamingChatMessageContent.java | 45 ++++++++++++++++++ 2 files changed, 84 insertions(+), 7 deletions(-) create mode 100644 aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiStreamingChatMessageContent.java diff --git a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java index 10a14477..8ba16bc4 100644 --- a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java +++ b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java @@ -16,6 +16,7 @@ import com.google.protobuf.Value; import com.microsoft.semantickernel.Kernel; import com.microsoft.semantickernel.aiservices.google.GeminiService; +import com.microsoft.semantickernel.aiservices.google.GeminiServiceBuilder; import com.microsoft.semantickernel.aiservices.google.implementation.MonoConverter; import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; import com.microsoft.semantickernel.exceptions.AIException; @@ -36,7 +37,7 @@ import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; import com.microsoft.semantickernel.services.chatcompletion.ChatHistory; import com.microsoft.semantickernel.services.chatcompletion.ChatMessageContent; -import com.microsoft.semantickernel.aiservices.google.GeminiServiceBuilder; +import com.microsoft.semantickernel.services.chatcompletion.StreamingChatContent; import java.io.IOException; import java.time.OffsetDateTime; import java.util.ArrayList; @@ -44,7 +45,6 @@ import java.util.UUID; import java.util.stream.Collectors; import javax.annotation.Nullable; -import com.microsoft.semantickernel.services.chatcompletion.StreamingChatContent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import reactor.core.publisher.Flux; @@ -78,17 +78,49 @@ public Mono>> getChatMessageContentsAsync(String prom @Override public Flux> getStreamingChatMessageContentsAsync( - ChatHistory chatHistory, @Nullable Kernel kernel, + ChatHistory chatHistory, + @Nullable Kernel kernel, @Nullable InvocationContext invocationContext) { - throw new UnsupportedOperationException( - "Streaming chat message contents are not supported"); + + LOGGER.warn("Streaming has been called on GeminiChatCompletion service. " + + "This is currently not supported in Gemini. " + + "The results will be returned in a non streaming fashion."); + + return getChatMessageContentsAsync(chatHistory, kernel, invocationContext) + .flatMapIterable(chatMessageContents -> chatMessageContents) + .map(content -> { + return new GeminiStreamingChatMessageContent( + content.getAuthorRole(), + content.getContent(), + getModelId(), + content.getInnerContent(), + content.getEncoding(), + content.getMetadata(), + null, + UUID.randomUUID().toString()); + }); } @Override public Flux> getStreamingChatMessageContentsAsync(String prompt, @Nullable Kernel kernel, @Nullable InvocationContext invocationContext) { - throw new UnsupportedOperationException( - "Streaming chat message contents are not supported"); + LOGGER.warn("Streaming has been called on GeminiChatCompletion service. " + + "This is currently not supported in Gemini. " + + "The results will be returned in a non streaming fashion."); + + return getChatMessageContentsAsync(prompt, kernel, invocationContext) + .flatMapIterable(chatMessageContents -> chatMessageContents) + .map(content -> { + return new GeminiStreamingChatMessageContent( + content.getAuthorRole(), + content.getContent(), + getModelId(), + content.getInnerContent(), + content.getEncoding(), + content.getMetadata(), + null, + UUID.randomUUID().toString()); + }); } @Override diff --git a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiStreamingChatMessageContent.java b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiStreamingChatMessageContent.java new file mode 100644 index 00000000..45aed993 --- /dev/null +++ b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiStreamingChatMessageContent.java @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.aiservices.google.chatcompletion; + +import com.microsoft.semantickernel.orchestration.FunctionResultMetadata; +import com.microsoft.semantickernel.services.chatcompletion.AuthorRole; +import com.microsoft.semantickernel.services.chatcompletion.StreamingChatContent; +import java.nio.charset.Charset; +import java.util.List; +import javax.annotation.Nullable; + +/** + * Represents the content of a chat message. + * + * @param The type of the inner content. + */ +public class GeminiStreamingChatMessageContent extends GeminiChatMessageContent implements + StreamingChatContent { + + private final String id; + + /** + * Creates a new instance of the {@link GeminiChatMessageContent} class. + * + * @param authorRole The author role that generated the content. + * @param content The content. + * @param modelId The model id. + * @param innerContent The inner content. + * @param encoding The encoding. + * @param metadata The metadata. + * @param geminiFunctionCalls The function calls. + */ + public GeminiStreamingChatMessageContent(AuthorRole authorRole, String content, + @Nullable String modelId, @Nullable T innerContent, @Nullable Charset encoding, + @Nullable FunctionResultMetadata metadata, + @Nullable List geminiFunctionCalls, + String id) { + super(authorRole, content, modelId, innerContent, encoding, metadata, geminiFunctionCalls); + this.id = id; + } + + @Override + public String getId() { + return id; + } +} From cc81f8175d9922969f4f6961db966a53f7651157 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Fri, 13 Sep 2024 09:28:31 -0700 Subject: [PATCH 038/277] Add suggestions --- .../AzureAISearchVectorStoreRecordCollection.java | 4 ++-- .../connectors/data/jdbc/JDBCVectorStoreQueryProvider.java | 2 +- .../data/postgres/PostgreSQLVectorStoreQueryProvider.java | 2 +- .../data/redis/RedisHashSetVectorStoreRecordCollection.java | 4 ++-- .../data/redis/RedisJsonVectorStoreRecordCollection.java | 4 ++-- .../data/vectorstorage/VectorStoreRecordMapper.java | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index 1d51fe16..798e7eae 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -203,7 +203,7 @@ public Mono getAsync( // Use custom mapper if available if (mapper != null && mapper.getStorageModelToRecordMapper() != null) { return searchAsyncClient.getDocument(key, SearchDocument.class) - .map(record -> mapper.mapStorageModeltoRecord(record, options)); + .map(record -> mapper.mapStorageModelToRecord(record, options)); } return searchAsyncClient @@ -292,7 +292,7 @@ private Mono>> searchAndMapAsync(SearchOptions s // Use custom mapper if available if (mapper != null && mapper.getStorageModelToRecordMapper() != null) { record = mapper - .mapStorageModeltoRecord(response.getDocument(SearchDocument.class), + .mapStorageModelToRecord(response.getDocument(SearchDocument.class), getRecordOptions); } else { record = response.getDocument(this.options.getRecordClass()); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index 5d9c0581..db456384 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -360,7 +360,7 @@ public List getRecords(String collectionName, List keys ResultSet resultSet = statement.executeQuery(); while (resultSet.next()) { - records.add(mapper.mapStorageModeltoRecord(resultSet, options)); + records.add(mapper.mapStorageModelToRecord(resultSet, options)); } return Collections.unmodifiableList(records); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index b7942cb9..701c8be4 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -391,7 +391,7 @@ public List> search(String collectionName, while (resultSet.next()) { records.add(new VectorSearchResult<>( - mapper.mapStorageModeltoRecord(resultSet, + mapper.mapStorageModelToRecord(resultSet, new GetRecordOptions(options.isIncludeVectors())), resultSet.getDouble("score"))); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java index f1968384..bb9ef070 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java @@ -250,7 +250,7 @@ public Mono> getBatchAsync(List keys, if (options != null && options.isIncludeVectors()) { // Results directly in a Map return this.vectorStoreRecordMapper - .mapStorageModeltoRecord( + .mapStorageModelToRecord( new AbstractMap.SimpleEntry<>(entry.getKey(), (Map) entry.getValue().get()), options); @@ -258,7 +258,7 @@ public Mono> getBatchAsync(List keys, // Results in a List with the values of the fields return this.vectorStoreRecordMapper - .mapStorageModeltoRecord( + .mapStorageModelToRecord( new AbstractMap.SimpleEntry<>(entry.getKey(), addDataFieldNames((List) entry.getValue().get())), options); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java index 4f742d97..dc468b1c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java @@ -233,7 +233,7 @@ public Mono getAsync(String key, GetRecordOptions options) { } return Mono.just(this.vectorStoreRecordMapper - .mapStorageModeltoRecord(new SimpleEntry<>(key, jsonNode), options)); + .mapStorageModelToRecord(new SimpleEntry<>(key, jsonNode), options)); } catch (Exception e) { return Mono.error(e); } @@ -280,7 +280,7 @@ public Mono> getBatchAsync(List keys, jsonNode = removeRedisPathPrefix((JSONObject) value); } return this.vectorStoreRecordMapper - .mapStorageModeltoRecord(new SimpleEntry<>(entry.getKey(), jsonNode), + .mapStorageModelToRecord(new SimpleEntry<>(entry.getKey(), jsonNode), options); }) .collect(Collectors.toList())); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java index 44ff95f3..8e3e7ae2 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java @@ -66,7 +66,7 @@ public StorageModel mapRecordToStorageModel(Record record) { * @param storageModel the storage model to convert * @return the record */ - public Record mapStorageModeltoRecord(StorageModel storageModel, GetRecordOptions options) { + public Record mapStorageModelToRecord(StorageModel storageModel, GetRecordOptions options) { return getStorageModelToRecordMapper().apply(storageModel, options); } } From 12a0925084b74ad8a66ac35e865ee5bdc8189fce Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Fri, 13 Sep 2024 09:53:21 -0700 Subject: [PATCH 039/277] Update --- .../connectors/data/jdbc/JDBCVectorStoreQueryProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index 6613be5b..85ec3ecf 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -446,7 +446,7 @@ protected List getAllRecords(String collectionName, List records = new ArrayList<>(); ResultSet resultSet = statement.executeQuery(); while (resultSet.next()) { - records.add(mapper.mapStorageModeltoRecord(resultSet, options)); + records.add(mapper.mapStorageModelToRecord(resultSet, options)); } return Collections.unmodifiableList(records); From b0dbb3734c30ed32a6da7dd40b46a7a1a8e7ecc0 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Fri, 13 Sep 2024 09:58:23 -0700 Subject: [PATCH 040/277] Fix test --- .../jdbc/JDBCVectorStoreRecordCollectionTest.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java index 56a256ca..a6bc78ca 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -335,16 +335,27 @@ public void exactSearch(QueryProvider provider, String embeddingName) { List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); + VectorSearchOptions options = VectorSearchOptions.builder() + .withVectorFieldName(embeddingName) + .withLimit(3) + .build(); + // Embeddings similar to the third hotel List embeddings = Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.2f); - List> results = recordCollection.searchAsync(embeddings, null).block(); + List> results = recordCollection.searchAsync(embeddings, options).block(); assertNotNull(results); assertEquals(3, results.size()); // The third hotel should be the most similar assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + options = VectorSearchOptions.builder() + .withVectorFieldName(embeddingName) + .withOffset(1) + .withLimit(-100) + .build(); + // Skip the first result - results = recordCollection.searchAsync(embeddings, VectorSearchOptions.builder().withOffset(1).withLimit(-100).build()).block(); + results = recordCollection.searchAsync(embeddings, options).block(); assertNotNull(results); assertEquals(1, results.size()); // The first hotel should be the most similar From c4cab96f6ddc98933b733e7e13977cbc96fe9a9a Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Fri, 13 Sep 2024 22:04:10 -0700 Subject: [PATCH 041/277] Add JDBC and Postgres filtering --- .../JDBCVectorStoreRecordCollectionTest.java | 61 ++++++++++++++----- ...rchVectorStoreCollectionSearchMapping.java | 40 ++++++------ ...reAISearchVectorStoreRecordCollection.java | 18 +++--- ...ureAISearchAnyTagEqualToFilterClause.java} | 8 +-- .../AzureAISearchEqualToFilterClause.java} | 8 +-- .../jdbc/JDBCVectorStoreQueryProvider.java | 27 ++++++-- ...torStoreRecordCollectionSearchMapping.java | 53 ++++++++++++++++ .../jdbc/filter/SQLEqualToFilterClause.java | 51 ++++++++++++++++ .../PostgreSQLVectorStoreQueryProvider.java | 24 ++++++-- .../AnyTagEqualToFilterClause.java} | 6 +- .../EqualToFilterClause.java} | 6 +- .../{filtering => filter}/FilterClause.java | 2 +- .../VectorSearchFilter.java} | 37 ++++++----- .../VectorStoreRecordDefinition.java | 24 +++++--- .../options/VectorSearchOptions.java | 21 +++---- 15 files changed, 280 insertions(+), 106 deletions(-) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/{AzureAISearchTagListContainsFilterClause.java => filter/AzureAISearchAnyTagEqualToFilterClause.java} (68%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/{AzureAISearchEqualityFilterClause.java => filter/AzureAISearchEqualToFilterClause.java} (88%) create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollectionSearchMapping.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/filter/SQLEqualToFilterClause.java rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{filtering/TagListContainsFilterClause.java => filter/AnyTagEqualToFilterClause.java} (71%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{filtering/EqualityFilterClause.java => filter/EqualToFilterClause.java} (72%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{filtering => filter}/FilterClause.java (78%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/{filtering/BasicVectorSearchFilter.java => vectorsearch/VectorSearchFilter.java} (50%) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java index a6bc78ca..99414ea4 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -5,6 +5,8 @@ import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.connectors.data.jdbc.filter.SQLEqualToFilterClause; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; @@ -103,13 +105,22 @@ private List getHotels() { return List.of( new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f),null, null, null, 4.0), - new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f),null, null, null, 3.0), + new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f),null, null, null, 4.0), new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f),null, null, null, 5.0), - new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f),null, null, null, 2.0), - new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f),null, null, null, 1.0) + new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f),null, null, null, 4.0), + new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f),null, null, null, 4.0) ); } + /** + * Search embeddings similar to the third hotel embeddings. + * In order of similarity: + * 1. Hotel 3 + * 2. Hotel 1 + * 3. Hotel 4 + */ + private static final List SEARCH_EMBEDDINGS = Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.2f); + @ParameterizedTest @EnumSource(QueryProvider.class) public void upsertAndGetRecordAsync(QueryProvider provider) { @@ -315,7 +326,7 @@ public void getBatchWithNoVectors(QueryProvider provider) { } } - private static Stream provideParameters() { + private static Stream provideSearchParameters() { return Stream.of( Arguments.of(QueryProvider.MySQL, "euclidean"), Arguments.of(QueryProvider.MySQL, "cosineDistance"), @@ -327,9 +338,9 @@ private static Stream provideParameters() { } @ParameterizedTest - @MethodSource("provideParameters") + @MethodSource("provideSearchParameters") public void exactSearch(QueryProvider provider, String embeddingName) { - String collectionName = "search"; + String collectionName = "search" + embeddingName; JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); List hotels = getHotels(); @@ -341,8 +352,7 @@ public void exactSearch(QueryProvider provider, String embeddingName) { .build(); // Embeddings similar to the third hotel - List embeddings = Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.2f); - List> results = recordCollection.searchAsync(embeddings, options).block(); + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, options).block(); assertNotNull(results); assertEquals(3, results.size()); // The third hotel should be the most similar @@ -355,7 +365,7 @@ public void exactSearch(QueryProvider provider, String embeddingName) { .build(); // Skip the first result - results = recordCollection.searchAsync(embeddings, options).block(); + results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, options).block(); assertNotNull(results); assertEquals(1, results.size()); // The first hotel should be the most similar @@ -377,14 +387,37 @@ public void approximateSearch(QueryProvider provider) { .build(); // Embeddings similar to the third hotel - List embeddings = Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.2f); - List> results = recordCollection.searchAsync(embeddings, options).block(); + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, options).block(); assertNotNull(results); assertEquals(5, results.size()); // The third hotel should be the most similar assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); } + @ParameterizedTest + @MethodSource("provideSearchParameters") + public void searchWithFilter(QueryProvider provider, String embeddingName) { + String collectionName = "searchWithFilter"; + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions options = VectorSearchOptions.builder() + .withVectorFieldName(embeddingName) + .withLimit(3) + .withBasicVectorSearchFilter( + VectorSearchFilter.builder().withEqualToFilterClause(new SQLEqualToFilterClause("rating", 4.0)).build()) + .build(); + + // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, options).block(); + assertNotNull(results); + assertEquals(3, results.size()); + // The first hotel should be the most similar + assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); + } + // MySQL will always return the vectors as they're needed to compute the distances @Test public void postgresSearchIncludeAndNotIncludeVectors() { @@ -394,9 +427,7 @@ public void postgresSearchIncludeAndNotIncludeVectors() { List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); - // Embeddings similar to the third hotel - List embeddings = Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.2f); - List> results = recordCollection.searchAsync(embeddings, null).block(); + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, null).block(); assertNotNull(results); assertEquals(3, results.size()); // The third hotel should be the most similar @@ -407,7 +438,7 @@ public void postgresSearchIncludeAndNotIncludeVectors() { .withIncludeVectors(true) .build(); - results = recordCollection.searchAsync(embeddings, options).block(); + results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, options).block(); assertNotNull(results); assertEquals(3, results.size()); // The third hotel should be the most similar diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java index 016568f6..eb3a5ff2 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java @@ -1,34 +1,40 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.azureaisearch; -import com.microsoft.semantickernel.data.filtering.BasicVectorSearchFilter; +import com.microsoft.semantickernel.connectors.data.azureaisearch.filter.AzureAISearchEqualToFilterClause; +import com.microsoft.semantickernel.connectors.data.azureaisearch.filter.AzureAISearchAnyTagEqualToFilterClause; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; -import java.util.Map; import java.util.stream.Collectors; class AzureAISearchVectorStoreCollectionSearchMapping { - public static String buildFilterString(BasicVectorSearchFilter basicVectorSearchFilter, - Map storageNames) { - if (basicVectorSearchFilter == null - || basicVectorSearchFilter.getFilterClauses().isEmpty()) { + public static String buildFilterString(VectorSearchFilter vectorSearchFilter, + VectorStoreRecordDefinition recordDefinition) { + if (vectorSearchFilter == null + || vectorSearchFilter.getFilterClauses().isEmpty()) { return ""; } return String.join(" and ", - basicVectorSearchFilter.getFilterClauses().stream().map(filterClause -> { - if (filterClause instanceof AzureAISearchEqualityFilterClause) { - AzureAISearchEqualityFilterClause azureFilterClause = (AzureAISearchEqualityFilterClause) filterClause; + vectorSearchFilter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof AzureAISearchEqualToFilterClause) { + AzureAISearchEqualToFilterClause azureFilterClause = (AzureAISearchEqualToFilterClause) filterClause; // Create new instance with the storage name of the field - return new AzureAISearchEqualityFilterClause( - storageNames.get(azureFilterClause.getFieldName()), - azureFilterClause.getValue()).getFilter(); - } else if (filterClause instanceof AzureAISearchTagListContainsFilterClause) { - AzureAISearchTagListContainsFilterClause azureFilterClause = (AzureAISearchTagListContainsFilterClause) filterClause; + return new AzureAISearchEqualToFilterClause( + recordDefinition.getField(azureFilterClause.getFieldName()) + .getEffectiveStorageName(), + azureFilterClause.getValue()) + .getFilter(); + } else if (filterClause instanceof AzureAISearchAnyTagEqualToFilterClause) { + AzureAISearchAnyTagEqualToFilterClause azureFilterClause = (AzureAISearchAnyTagEqualToFilterClause) filterClause; // Create new instance with the storage name of the field - return new AzureAISearchTagListContainsFilterClause( - storageNames.get(azureFilterClause.getFieldName()), - azureFilterClause.getValue()).getFilter(); + return new AzureAISearchAnyTagEqualToFilterClause( + recordDefinition.getField(azureFilterClause.getFieldName()) + .getEffectiveStorageName(), + azureFilterClause.getValue()) + .getFilter(); } else { throw new SKException("Unsupported filter clause type '" + filterClause.getClass().getSimpleName() + "'."); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index 798e7eae..9999f6ee 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -84,7 +84,6 @@ public class AzureAISearchVectorStoreRecordCollection implements // List of non-vector fields. Used to fetch only non-vector fields when vectors are not requested private final List nonVectorFields = new ArrayList<>(); - private final Map storageNames; private final String firstVectorFieldName; @SuppressFBWarnings("EI_EXPOSE_REP2") @@ -119,7 +118,6 @@ public AzureAISearchVectorStoreRecordCollection( .map(VectorStoreRecordDataField::getEffectiveStorageName) .collect(Collectors.toList())); - storageNames = recordDefinition.getFieldStorageNames(); firstVectorFieldName = recordDefinition.getVectorFields().isEmpty() ? null : recordDefinition.getVectorFields().get(0).getName(); } @@ -323,25 +321,23 @@ public Mono>> searchAsync(VectorSearchQuery quer if (query instanceof VectorizedSearchQuery) { vectorQueries.add(new VectorizedQuery(((VectorizedSearchQuery) query).getVector()) - .setFields( - storageNames - .get(options.getVectorFieldName() != null ? options.getVectorFieldName() - : firstVectorFieldName)) + .setFields(recordDefinition.getField(options.getVectorFieldName() != null + ? options.getVectorFieldName() + : firstVectorFieldName).getEffectiveStorageName()) .setKNearestNeighborsCount(options.getLimit())); } else if (query instanceof VectorizableTextSearchQuery) { vectorQueries .add(new VectorizableTextQuery(((VectorizableTextSearchQuery) query).getQueryText()) - .setFields( - storageNames - .get(options.getVectorFieldName() != null ? options.getVectorFieldName() - : firstVectorFieldName)) + .setFields(recordDefinition.getField(options.getVectorFieldName() != null + ? options.getVectorFieldName() + : firstVectorFieldName).getEffectiveStorageName()) .setKNearestNeighborsCount(options.getLimit())); } else { throw new SKException("Unsupported query type: " + query.getQueryType()); } String filter = AzureAISearchVectorStoreCollectionSearchMapping - .buildFilterString(options.getBasicVectorSearchFilter(), storageNames); + .buildFilterString(options.getBasicVectorSearchFilter(), recordDefinition); SearchOptions searchOptions = new SearchOptions() .setFilter(filter) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchTagListContainsFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchAnyTagEqualToFilterClause.java similarity index 68% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchTagListContainsFilterClause.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchAnyTagEqualToFilterClause.java index 8e2ba0bf..7c84174e 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchTagListContainsFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchAnyTagEqualToFilterClause.java @@ -1,9 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.azureaisearch; +package com.microsoft.semantickernel.connectors.data.azureaisearch.filter; -import com.microsoft.semantickernel.data.filtering.TagListContainsFilterClause; +import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; -public class AzureAISearchTagListContainsFilterClause extends TagListContainsFilterClause { +public class AzureAISearchAnyTagEqualToFilterClause extends AnyTagEqualToFilterClause { /** * Initializes a new instance of the AzureAISearchTagListContainsFilterClause class. @@ -11,7 +11,7 @@ public class AzureAISearchTagListContainsFilterClause extends TagListContainsFil * @param fieldName The field name to filter on. * @param value The value. */ - public AzureAISearchTagListContainsFilterClause(String fieldName, Object value) { + public AzureAISearchAnyTagEqualToFilterClause(String fieldName, Object value) { super(fieldName, value); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchEqualityFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchEqualToFilterClause.java similarity index 88% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchEqualityFilterClause.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchEqualToFilterClause.java index 98da48d1..0bacb35c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchEqualityFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchEqualToFilterClause.java @@ -1,13 +1,13 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.azureaisearch; +package com.microsoft.semantickernel.connectors.data.azureaisearch.filter; -import com.microsoft.semantickernel.data.filtering.EqualityFilterClause; +import com.microsoft.semantickernel.data.filter.EqualToFilterClause; import com.microsoft.semantickernel.exceptions.SKException; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; -public class AzureAISearchEqualityFilterClause extends EqualityFilterClause { +public class AzureAISearchEqualToFilterClause extends EqualToFilterClause { /** * Initializes a new instance of the AzureAISearchEqualityFilterClause class. @@ -15,7 +15,7 @@ public class AzureAISearchEqualityFilterClause extends EqualityFilterClause { * @param fieldName The field name to filter on. * @param value The value. */ - public AzureAISearchEqualityFilterClause(String fieldName, Object value) { + public AzureAISearchEqualToFilterClause(String fieldName, Object value) { super(fieldName, value); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index 85ec3ecf..993125e0 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; +import com.microsoft.semantickernel.data.filter.FilterClause; import com.microsoft.semantickernel.data.vectorsearch.VectorOperations; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; @@ -427,9 +428,10 @@ public void deleteRecords(String collectionName, List keys, } } - protected List getAllRecords(String collectionName, + protected List getRecordsWithFilter(String collectionName, VectorStoreRecordDefinition recordDefinition, - VectorStoreRecordMapper mapper, GetRecordOptions options) { + VectorStoreRecordMapper mapper, GetRecordOptions options, String filter, + List parameters) { List fields; if (options.isIncludeVectors()) { fields = recordDefinition.getAllFields(); @@ -437,12 +439,20 @@ protected List getAllRecords(String collectionName, fields = recordDefinition.getNonVectorFields(); } - String selectQuery = formatQuery("SELECT %s FROM %s", + String filterClause = filter == null || filter.isEmpty() ? "" : "WHERE " + filter; + String selectQuery = formatQuery("SELECT %s FROM %s %s", getQueryColumnsFromFields(fields), - getCollectionTableName(collectionName)); + getCollectionTableName(collectionName), + filterClause); try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(selectQuery)) { + if (parameters != null) { + for (int i = 0; i < parameters.size(); ++i) { + statement.setObject(i + 1, parameters.get(i)); + } + } + List records = new ArrayList<>(); ResultSet resultSet = statement.executeQuery(); while (resultSet.next()) { @@ -491,8 +501,13 @@ public List> search(String collectionName, : (VectorStoreRecordVectorField) recordDefinition .getField(options.getVectorFieldName()); - List records = getAllRecords(collectionName, recordDefinition, mapper, - new GetRecordOptions(true)); + String filter = SQLVectorStoreRecordCollectionSearchMapping + .buildFilter(options.getBasicVectorSearchFilter(), recordDefinition); + List parameters = SQLVectorStoreRecordCollectionSearchMapping + .getFilterParameters(options.getBasicVectorSearchFilter()); + + List records = getRecordsWithFilter(collectionName, recordDefinition, mapper, + new GetRecordOptions(true), filter, parameters); List> results = new ArrayList<>(); DistanceFunction distanceFunction = vectorField.getDistanceFunction() == null diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollectionSearchMapping.java new file mode 100644 index 00000000..17164acc --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollectionSearchMapping.java @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.jdbc; + +import com.microsoft.semantickernel.connectors.data.jdbc.filter.SQLEqualToFilterClause; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.exceptions.SKException; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +public class SQLVectorStoreRecordCollectionSearchMapping { + + public static String buildFilter(VectorSearchFilter vectorSearchFilter, + VectorStoreRecordDefinition recordDefinition) { + if (vectorSearchFilter == null + || vectorSearchFilter.getFilterClauses().isEmpty()) { + return ""; + } + + return vectorSearchFilter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof SQLEqualToFilterClause) { + SQLEqualToFilterClause equalityFilterClause = (SQLEqualToFilterClause) filterClause; + // Create new instance with the storage name of the field + return new SQLEqualToFilterClause( + recordDefinition.getField(equalityFilterClause.getFieldName()) + .getEffectiveStorageName(), + equalityFilterClause.getValue()).getFilter(); + } else { + throw new SKException("Unsupported filter clause type '" + + filterClause.getClass().getSimpleName() + "'."); + } + }).collect(Collectors.joining(" AND ")); + } + + public static List getFilterParameters(VectorSearchFilter vectorSearchFilter) { + if (vectorSearchFilter == null + || vectorSearchFilter.getFilterClauses().isEmpty()) { + return Collections.emptyList(); + } + + return vectorSearchFilter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof SQLEqualToFilterClause) { + SQLEqualToFilterClause equalityFilterClause = (SQLEqualToFilterClause) filterClause; + return equalityFilterClause.getValue(); + } else { + throw new SKException("Unsupported filter clause type '" + + filterClause.getClass().getSimpleName() + "'."); + } + }).collect(Collectors.toList()); + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/filter/SQLEqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/filter/SQLEqualToFilterClause.java new file mode 100644 index 00000000..d90b27b7 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/filter/SQLEqualToFilterClause.java @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.jdbc.filter; + +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.filter.EqualToFilterClause; +import com.microsoft.semantickernel.exceptions.SKException; + +import java.time.OffsetDateTime; + +public class SQLEqualToFilterClause extends EqualToFilterClause { + + /** + * Initializes a new instance of the SQLEqualityFilterClause class. + * + * @param fieldName The field name to filter on. + * @param value The value. + */ + public SQLEqualToFilterClause(String fieldName, Object value) { + super(fieldName, value); + } + + /** + * Gets the filter string. + * + * @return The filter string. + */ + @Override + public String getFilter() { + String fieldName = JDBCVectorStoreQueryProvider.validateSQLidentifier(getFieldName()); + Object value = getValue(); + + if (value instanceof String) { + return String.format("%s = ?", fieldName); + } else if (value instanceof Boolean) { + return String.format("%s = ?", fieldName); + } else if (value instanceof Integer) { + return String.format("%s = ?", fieldName); + } else if (value instanceof Long) { + return String.format("%s = ?", fieldName); + } else if (value instanceof Float) { + return String.format("%s = ?", fieldName); + } else if (value instanceof Double) { + return String.format("%s = ?", fieldName); + } else if (value instanceof OffsetDateTime) { + return String.format("%s = ?", fieldName); + } else { + throw new SKException("Unsupported filter value type '" + + value.getClass().getSimpleName() + "'."); + } + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index 701c8be4..8f7fdfd7 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreRecordCollectionSearchMapping; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizedSearchQuery; @@ -369,22 +370,35 @@ public List> search(String collectionName, "Distance function is required for vector field: " + vectorField.getName()); } + String filter = SQLVectorStoreRecordCollectionSearchMapping.buildFilter( + options.getBasicVectorSearchFilter(), + recordDefinition); + List parameters = SQLVectorStoreRecordCollectionSearchMapping + .getFilterParameters(options.getBasicVectorSearchFilter()); + + String filterClause = filter.isEmpty() ? "" : "WHERE " + filter; String searchQuery = formatQuery( - "SELECT %s, %s %s ?::vector AS score FROM %s ORDER BY score LIMIT ? OFFSET ?", + "SELECT %s, %s %s ?::vector AS score FROM %s %s ORDER BY score LIMIT ? OFFSET ?", getQueryColumnsFromFields( options.isIncludeVectors() ? recordDefinition.getAllFields() : recordDefinition.getNonVectorFields()), validateSQLidentifier(vectorField.getEffectiveStorageName()), distanceFunction == null ? PostgreSQLVectorDistanceFunction.L2.getOperator() : distanceFunction.getOperator(), - getCollectionTableName(collectionName)); + getCollectionTableName(collectionName), + filterClause); try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(searchQuery)) { - statement.setString(1, + int parameterIndex = 1; + + statement.setString(parameterIndex++, objectMapper.writeValueAsString(vectorizedSearchQuery.getVector())); - statement.setInt(2, options.getLimit()); - statement.setInt(3, options.getOffset()); + for (Object parameter : parameters) { + statement.setObject(parameterIndex++, parameter); + } + statement.setInt(parameterIndex++, options.getLimit()); + statement.setInt(parameterIndex, options.getOffset()); List> records = new ArrayList<>(); ResultSet resultSet = statement.executeQuery(); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/TagListContainsFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java similarity index 71% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/TagListContainsFilterClause.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java index bcf2e64f..1b940717 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/TagListContainsFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java @@ -1,12 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.filtering; +package com.microsoft.semantickernel.data.filter; -public abstract class TagListContainsFilterClause implements FilterClause { +public abstract class AnyTagEqualToFilterClause implements FilterClause { private final String fieldName; private final Object value; - public TagListContainsFilterClause(String fieldName, Object value) { + public AnyTagEqualToFilterClause(String fieldName, Object value) { this.fieldName = fieldName; this.value = value; } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/EqualityFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java similarity index 72% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/EqualityFilterClause.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java index ec6072ef..bbf8fb77 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/EqualityFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java @@ -1,12 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.filtering; +package com.microsoft.semantickernel.data.filter; -public abstract class EqualityFilterClause implements FilterClause { +public abstract class EqualToFilterClause implements FilterClause { private final String fieldName; private final Object value; - public EqualityFilterClause(String fieldName, Object value) { + public EqualToFilterClause(String fieldName, Object value) { this.fieldName = fieldName; this.value = value; } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/FilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java similarity index 78% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/FilterClause.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java index a63e85dd..e3425be6 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/FilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.filtering; +package com.microsoft.semantickernel.data.filter; public interface FilterClause { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/BasicVectorSearchFilter.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java similarity index 50% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/BasicVectorSearchFilter.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java index 642f4589..b794d7d3 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filtering/BasicVectorSearchFilter.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java @@ -1,24 +1,28 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.filtering; +package com.microsoft.semantickernel.data.vectorsearch; + +import com.microsoft.semantickernel.data.filter.EqualToFilterClause; +import com.microsoft.semantickernel.data.filter.FilterClause; +import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; import java.util.ArrayList; import java.util.Collections; import java.util.List; -public class BasicVectorSearchFilter { +public class VectorSearchFilter { /** * Creates a new instance of the BasicVectorSearchFilter class. * * @return A new instance of the BasicVectorSearchFilter class. */ - public static BasicVectorSearchFilter createDefault() { - return new BasicVectorSearchFilter(); + public static VectorSearchFilter createDefault() { + return new VectorSearchFilter(); } private final List filterClauses; - public BasicVectorSearchFilter() { + public VectorSearchFilter() { this(new ArrayList<>()); } @@ -27,7 +31,7 @@ public BasicVectorSearchFilter() { * * @param filterClauses The filter clauses. */ - public BasicVectorSearchFilter(List filterClauses) { + public VectorSearchFilter(List filterClauses) { this.filterClauses = Collections.unmodifiableList(filterClauses); } @@ -48,29 +52,30 @@ public static class Builder { private final List filterClauses = new ArrayList<>(); /** - * Adds an equality filter clause to the filter. + * Adds an EqualToFilterClause to the filter. * - * @param equalityFilterClause The equality filter clause to add. + * @param equalToFilterClause The EqualToFilterClause to add. * @return The builder. */ - public Builder equality(EqualityFilterClause equalityFilterClause) { - filterClauses.add(equalityFilterClause); + public Builder withEqualToFilterClause(EqualToFilterClause equalToFilterClause) { + filterClauses.add(equalToFilterClause); return this; } /** - * Adds a tag list contains filter clause to the filter. + * Adds an AnyTagEqualToFilterClause to the filter. * - * @param tagListContainsFilterClause The tag list contains filter clause to add. + * @param anyTagEqualToFilterClause The AnyTagEqualToFilterClause clause to add. * @return The builder. */ - public Builder tagListContains(TagListContainsFilterClause tagListContainsFilterClause) { - filterClauses.add(tagListContainsFilterClause); + public Builder withAnyTagEqualToFilterClause( + AnyTagEqualToFilterClause anyTagEqualToFilterClause) { + filterClauses.add(anyTagEqualToFilterClause); return this; } - public BasicVectorSearchFilter build() { - return new BasicVectorSearchFilter(filterClauses); + public VectorSearchFilter build() { + return new VectorSearchFilter(filterClauses); } } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java index 4c9bb94a..98378254 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java @@ -62,6 +62,20 @@ public List getNonVectorFields() { return nonVectorFields; } + /** + * Checks if the record definition contains a field with the specified name. + * @param fieldName The name of the field to check. + * @return boolean + */ + public boolean containsField(String fieldName) { + return allFieldsMap.containsKey(fieldName); + } + + /** + * Gets the field with the specified name. + * @param fieldName The name of the field to get. + * @return VectorStoreRecordField + */ public VectorStoreRecordField getField(String fieldName) { if (!allFieldsMap.containsKey(fieldName)) { throw new SKException("Field not found: " + fieldName); @@ -69,16 +83,6 @@ public VectorStoreRecordField getField(String fieldName) { return allFieldsMap.get(fieldName); } - /** - * Gets the storage names of the fields in the record definition. - * @return Map of field names to storage names - */ - public Map getFieldStorageNames() { - return allFields.stream() - .collect(Collectors.toMap(VectorStoreRecordField::getName, - VectorStoreRecordField::getEffectiveStorageName)); - } - private VectorStoreRecordDefinition( VectorStoreRecordKeyField keyField, List dataFields, diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java index cfdb6908..66955c2e 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java @@ -2,8 +2,7 @@ package com.microsoft.semantickernel.data.vectorstorage.options; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.data.filtering.BasicVectorSearchFilter; -import com.microsoft.semantickernel.exceptions.SKException; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import javax.annotation.Nullable; @@ -23,16 +22,16 @@ public static VectorSearchOptions createDefault(String vectorFieldName) { } @Nullable - private final BasicVectorSearchFilter basicVectorSearchFilter; + private final VectorSearchFilter vectorSearchFilter; @Nullable private final String vectorFieldName; private final int limit; private final int offset; private final boolean includeVectors; - public VectorSearchOptions(BasicVectorSearchFilter basicVectorSearchFilter, + public VectorSearchOptions(VectorSearchFilter vectorSearchFilter, String vectorFieldName, int limit, int offset, boolean includeVectors) { - this.basicVectorSearchFilter = basicVectorSearchFilter; + this.vectorSearchFilter = vectorSearchFilter; this.vectorFieldName = vectorFieldName; this.limit = Math.max(1, limit); this.offset = Math.max(0, offset); @@ -45,8 +44,8 @@ public VectorSearchOptions(BasicVectorSearchFilter basicVectorSearchFilter, * @return The basic vector search filter. */ @Nullable - public BasicVectorSearchFilter getBasicVectorSearchFilter() { - return basicVectorSearchFilter; + public VectorSearchFilter getBasicVectorSearchFilter() { + return vectorSearchFilter; } /** @@ -96,7 +95,7 @@ public static Builder builder() { } public static class Builder implements SemanticKernelBuilder { - private BasicVectorSearchFilter basicVectorSearchFilter; + private VectorSearchFilter vectorSearchFilter; private String vectorFieldName; private int limit; private int offset; @@ -107,8 +106,8 @@ public Builder() { } public Builder withBasicVectorSearchFilter( - BasicVectorSearchFilter basicVectorSearchFilter) { - this.basicVectorSearchFilter = basicVectorSearchFilter; + VectorSearchFilter vectorSearchFilter) { + this.vectorSearchFilter = vectorSearchFilter; return this; } @@ -134,7 +133,7 @@ public Builder withIncludeVectors(boolean includeVectors) { @Override public VectorSearchOptions build() { - return new VectorSearchOptions(basicVectorSearchFilter, vectorFieldName, limit, offset, + return new VectorSearchOptions(vectorSearchFilter, vectorFieldName, limit, offset, includeVectors); } } From 470f6c1a2f0344be28151c29135bb26cf0e9430d Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Fri, 13 Sep 2024 22:15:49 -0700 Subject: [PATCH 042/277] Update VectorSearchFilter builder --- .../memory/jdbc/JDBCVectorStoreRecordCollectionTest.java | 2 +- .../AzureAISearchVectorStoreRecordCollection.java | 3 +-- .../data/jdbc/JDBCVectorStoreQueryProvider.java | 6 ++---- .../data/postgres/PostgreSQLVectorStoreQueryProvider.java | 4 ++-- .../data/vectorsearch/VectorSearchFilter.java | 8 ++++---- .../data/vectorstorage/options/VectorSearchOptions.java | 8 ++++---- 6 files changed, 14 insertions(+), 17 deletions(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java index 99414ea4..9c472868 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -406,7 +406,7 @@ public void searchWithFilter(QueryProvider provider, String embeddingName) { VectorSearchOptions options = VectorSearchOptions.builder() .withVectorFieldName(embeddingName) .withLimit(3) - .withBasicVectorSearchFilter( + .withVectorSearchFilter( VectorSearchFilter.builder().withEqualToFilterClause(new SQLEqualToFilterClause("rating", 4.0)).build()) .build(); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index 9999f6ee..e20e24d8 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -41,7 +41,6 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import javax.annotation.Nonnull; import reactor.core.publisher.Flux; @@ -337,7 +336,7 @@ public Mono>> searchAsync(VectorSearchQuery quer } String filter = AzureAISearchVectorStoreCollectionSearchMapping - .buildFilterString(options.getBasicVectorSearchFilter(), recordDefinition); + .buildFilterString(options.getVectorSearchFilter(), recordDefinition); SearchOptions searchOptions = new SearchOptions() .setFilter(filter) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index 993125e0..c051638d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -4,7 +4,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; -import com.microsoft.semantickernel.data.filter.FilterClause; import com.microsoft.semantickernel.data.vectorsearch.VectorOperations; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; @@ -19,7 +18,6 @@ import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; -import com.microsoft.semantickernel.services.OrderedAIServiceSelector; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -502,9 +500,9 @@ public List> search(String collectionName, .getField(options.getVectorFieldName()); String filter = SQLVectorStoreRecordCollectionSearchMapping - .buildFilter(options.getBasicVectorSearchFilter(), recordDefinition); + .buildFilter(options.getVectorSearchFilter(), recordDefinition); List parameters = SQLVectorStoreRecordCollectionSearchMapping - .getFilterParameters(options.getBasicVectorSearchFilter()); + .getFilterParameters(options.getVectorSearchFilter()); List records = getRecordsWithFilter(collectionName, recordDefinition, mapper, new GetRecordOptions(true), filter, parameters); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index 8f7fdfd7..46f27dd5 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -371,10 +371,10 @@ public List> search(String collectionName, } String filter = SQLVectorStoreRecordCollectionSearchMapping.buildFilter( - options.getBasicVectorSearchFilter(), + options.getVectorSearchFilter(), recordDefinition); List parameters = SQLVectorStoreRecordCollectionSearchMapping - .getFilterParameters(options.getBasicVectorSearchFilter()); + .getFilterParameters(options.getVectorSearchFilter()); String filterClause = filter.isEmpty() ? "" : "WHERE " + filter; String searchQuery = formatQuery( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java index b794d7d3..cde9594f 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java @@ -12,9 +12,9 @@ public class VectorSearchFilter { /** - * Creates a new instance of the BasicVectorSearchFilter class. + * Creates a new instance of the VectorSearchFilter class. * - * @return A new instance of the BasicVectorSearchFilter class. + * @return A new instance of the VectorSearchFilter class. */ public static VectorSearchFilter createDefault() { return new VectorSearchFilter(); @@ -23,11 +23,11 @@ public static VectorSearchFilter createDefault() { private final List filterClauses; public VectorSearchFilter() { - this(new ArrayList<>()); + this(Collections.emptyList()); } /** - * Creates a new instance of the BasicVectorSearchFilter class. + * Creates a new instance of the VectorSearchFilter class. * * @param filterClauses The filter clauses. */ diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java index 66955c2e..acaddf9b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java @@ -39,12 +39,12 @@ public VectorSearchOptions(VectorSearchFilter vectorSearchFilter, } /** - * Gets the basic vector search filter. + * Gets the vector search filter. * - * @return The basic vector search filter. + * @return The vector search filter. */ @Nullable - public VectorSearchFilter getBasicVectorSearchFilter() { + public VectorSearchFilter getVectorSearchFilter() { return vectorSearchFilter; } @@ -105,7 +105,7 @@ public Builder() { this.limit = DEFAULT_RESULT_LIMIT; } - public Builder withBasicVectorSearchFilter( + public Builder withVectorSearchFilter( VectorSearchFilter vectorSearchFilter) { this.vectorSearchFilter = vectorSearchFilter; return this; From 5888b79c7af4eeb7b45c45cdd0fd19df799efcb9 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Fri, 13 Sep 2024 13:29:57 +0100 Subject: [PATCH 043/277] Add json_schema support --- CHANGELOG.md | 15 + aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- .../HuggingFacePromptExecutionSettings.java | 7 +- aiservices/openai/pom.xml | 16 +- .../chatcompletion/OpenAIChatCompletion.java | 35 +- ...atCompletionsJsonSchemaResponseFormat.java | 35 ++ .../JacksonResponseFormatGenerator.java | 68 ++++ .../aiservices/openai/chatcompletion/Bar.java | 19 ++ .../aiservices/openai/chatcompletion/Baz.java | 22 ++ .../aiservices/openai/chatcompletion/Foo.java | 22 ++ .../openai/chatcompletion/JsonSchemaTest.java | 27 ++ .../OpenAiChatCompletionTest.java | 9 +- api-test/integration-tests/pom.xml | 14 +- .../tests/ResponseSchemaTest.java | 307 ++++++++++++++++++ .../tests/ToolCallBehaviourTest.java | 4 +- .../test/resources/responseSchema.prompt.yaml | 27 ++ ...-12b6744e-443f-4fe7-82e2-55cc41195ff1.json | 2 +- ...-71b07779-49a8-44e5-a60b-ee5b0a3ad697.json | 2 +- ...-0c8046c5-74ad-4836-8aa9-09da60f367a2.json | 2 +- ...-0215b128-4822-4368-ac3d-2f580a221f00.json | 2 +- api-test/pom.xml | 2 +- pom.xml | 4 +- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 10 +- .../Example_ChatWithResponseFormat.java | 71 ++++ ...xample_ChatWithResponseFormatToolCall.java | 90 +++++ .../chatcompletion/responseschema/Pet.java | 71 ++++ .../memory/AzureAISearchVectorStore.java | 26 +- .../src/main/resources/log4j2.xml | 3 + .../booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- .../sk-presidio-sample/pom.xml | 2 +- .../semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- .../contextvariables/ContextVariable.java | 2 +- .../ContextVariableTypeConverter.java | 25 +- .../ContextVariableTypes.java | 6 +- .../localization/SemanticKernelResources.java | 23 +- .../orchestration/FunctionInvocation.java | 32 +- .../PromptExecutionSettings.java | 64 +++- .../orchestration/ResponseFormat.java | 10 - .../JsonObjectResponseFormat.java | 12 + .../responseformat/JsonResponseSchema.java | 35 ++ .../JsonSchemaResponseFormat.java | 88 +++++ .../responseformat/ResponseFormat.java | 40 +++ .../responseformat/ResponseSchema.java | 6 + .../ResponseSchemaGenerator.java | 48 +++ .../responseformat/TextResponseFormat.java | 12 + .../PromptTemplateConfig.java | 5 +- .../PromptExecutionSettingsTest.java | 11 +- .../KernelFunctionYamlTest.java | 15 +- .../PromptTemplateConfigTest.java | 5 +- semantickernel-bom/pom.xml | 19 +- semantickernel-experimental/pom.xml | 2 +- 60 files changed, 1303 insertions(+), 95 deletions(-) create mode 100644 aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/responseformat/ChatCompletionsJsonSchemaResponseFormat.java create mode 100644 aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/responseformat/JacksonResponseFormatGenerator.java create mode 100644 aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/Bar.java create mode 100644 aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/Baz.java create mode 100644 aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/Foo.java create mode 100644 aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/JsonSchemaTest.java create mode 100644 api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ResponseSchemaTest.java create mode 100644 api-test/integration-tests/src/test/resources/responseSchema.prompt.yaml create mode 100644 samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/responseschema/Example_ChatWithResponseFormat.java create mode 100644 samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/responseschema/Example_ChatWithResponseFormatToolCall.java create mode 100644 samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/responseschema/Pet.java delete mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/ResponseFormat.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonObjectResponseFormat.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonResponseSchema.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonSchemaResponseFormat.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseFormat.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseSchema.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseSchemaGenerator.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/TextResponseFormat.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 8942e497..c1209d7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,18 @@ +# 1.3.0 + +- Added support for Json Schema to Open AI Chat Completions +- Upgraded to openai sdk 1.0.0-beta.11 +- Added convinience method `FunctionInvocation.withResultTypeAutoConversion` which sets the return type and registers a + type converter based on Jackson for the return type. + +### Bug Fixes + +- Fixed type converters not being passed on to be used in tool invocations + +### Breaking Changes + +- To support the new Json Schema feature, ResponseFormat has changed from an enum to a class. + # 1.2.2 - Fix bug in `FunctionInvocation` not using per-invocation type conversion when calling `withResultType`. diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index 7899fe11..7dd0a007 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.3-SNAPSHOT + 1.3.0-SNAPSHOT ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index bce95e64..fd1f418b 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.3-SNAPSHOT + 1.3.0-SNAPSHOT ../../pom.xml diff --git a/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFacePromptExecutionSettings.java b/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFacePromptExecutionSettings.java index ed3a4df5..be6cd2ca 100644 --- a/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFacePromptExecutionSettings.java +++ b/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFacePromptExecutionSettings.java @@ -2,6 +2,7 @@ package com.microsoft.semantickernel.aiservices.huggingface.services; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; +import com.microsoft.semantickernel.orchestration.responseformat.ResponseFormat; import java.util.List; import java.util.Map; import java.util.Objects; @@ -38,7 +39,7 @@ public HuggingFacePromptExecutionSettings(PromptExecutionSettings copy) { copy.getUser(), copy.getStopSequences(), copy.getTokenSelectionBiases(), - copy.getResponseFormat() == null ? null : copy.getResponseFormat().toString()); + copy.getResponseFormat() == null ? null : copy.getResponseFormat()); this.topK = null; this.repetitionPenalty = null; this.maxTime = null; @@ -78,7 +79,7 @@ public HuggingFacePromptExecutionSettings( String user, @Nullable List stopSequences, @Nullable Map tokenSelectionBiases, - @Nullable String responseFormat, + @Nullable ResponseFormat responseFormat, @Nullable Integer topK, @Nullable Double repetitionPenalty, @Nullable Double maxTime, @@ -119,7 +120,7 @@ public static HuggingFacePromptExecutionSettings fromExecutionSettings( promptExecutionSettings.getStopSequences(), promptExecutionSettings.getTokenSelectionBiases(), promptExecutionSettings.getResponseFormat() != null - ? promptExecutionSettings.getResponseFormat().toString() + ? promptExecutionSettings.getResponseFormat() : null, null, null, diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index bcc7f135..302a354e 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -1,12 +1,13 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.2.3-SNAPSHOT + 1.3.0-SNAPSHOT ../../pom.xml @@ -55,6 +56,17 @@ mockito-core test + + + com.github.victools + jsonschema-generator + true + + + com.github.victools + jsonschema-module-jackson + true + diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java index 5442e51e..a73438b7 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java @@ -7,10 +7,13 @@ import com.azure.ai.openai.models.ChatCompletionsFunctionToolCall; import com.azure.ai.openai.models.ChatCompletionsFunctionToolDefinition; import com.azure.ai.openai.models.ChatCompletionsJsonResponseFormat; +import com.azure.ai.openai.models.ChatCompletionsNamedToolSelection; import com.azure.ai.openai.models.ChatCompletionsOptions; import com.azure.ai.openai.models.ChatCompletionsTextResponseFormat; import com.azure.ai.openai.models.ChatCompletionsToolCall; import com.azure.ai.openai.models.ChatCompletionsToolDefinition; +import com.azure.ai.openai.models.ChatCompletionsToolSelection; +import com.azure.ai.openai.models.ChatCompletionsToolSelectionPreset; import com.azure.ai.openai.models.ChatMessageImageContentItem; import com.azure.ai.openai.models.ChatMessageImageDetailLevel; import com.azure.ai.openai.models.ChatMessageImageUrl; @@ -23,13 +26,15 @@ import com.azure.ai.openai.models.ChatResponseMessage; import com.azure.ai.openai.models.CompletionsUsage; import com.azure.ai.openai.models.FunctionCall; -import com.azure.core.util.BinaryData; +import com.azure.json.JsonOptions; +import com.azure.json.implementation.DefaultJsonReader; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ContainerNode; import com.microsoft.semantickernel.Kernel; import com.microsoft.semantickernel.aiservices.openai.OpenAiService; +import com.microsoft.semantickernel.aiservices.openai.chatcompletion.responseformat.ChatCompletionsJsonSchemaResponseFormat; import com.microsoft.semantickernel.aiservices.openai.implementation.OpenAIRequestSettings; import com.microsoft.semantickernel.contextvariables.ContextVariable; import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; @@ -50,6 +55,8 @@ import com.microsoft.semantickernel.orchestration.InvocationReturnMode; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; import com.microsoft.semantickernel.orchestration.ToolCallBehavior; +import com.microsoft.semantickernel.orchestration.responseformat.JsonResponseSchema; +import com.microsoft.semantickernel.orchestration.responseformat.JsonSchemaResponseFormat; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; import com.microsoft.semantickernel.services.chatcompletion.AuthorRole; @@ -60,6 +67,7 @@ import com.microsoft.semantickernel.services.chatcompletion.message.ChatMessageImageContent; import com.microsoft.semantickernel.services.openai.OpenAiServiceBuilder; import io.opentelemetry.api.trace.Span; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -530,6 +538,8 @@ private Mono> invokeFunctionTool( return function .invokeAsync(kernel) .withArguments(arguments) + .withTypes(invocationContext.getContextVariableTypes()) + .withTypes(contextVariableTypes) .withResultType(contextVariableTypes.getVariableTypeForClass(String.class)); } catch (JsonProcessingException e) { return Mono.error(new SKException("Failed to parse tool arguments", e)); @@ -812,13 +822,21 @@ private static ChatCompletionsOptions getCompletionsOptions( .setLogitBias(logit); if (promptExecutionSettings.getResponseFormat() != null) { - switch (promptExecutionSettings.getResponseFormat()) { + switch (promptExecutionSettings.getResponseFormat().getType()) { + case JSON_SCHEMA: + JsonResponseSchema schema = ((JsonSchemaResponseFormat) promptExecutionSettings + .getResponseFormat()) + .getJsonSchema(); + + options.setResponseFormat(new ChatCompletionsJsonSchemaResponseFormat(schema)); + break; case JSON_OBJECT: options.setResponseFormat(new ChatCompletionsJsonResponseFormat()); break; case TEXT: options.setResponseFormat(new ChatCompletionsTextResponseFormat()); break; + default: throw new SKException( "Unknown response format: " + promptExecutionSettings.getResponseFormat()); @@ -869,9 +887,17 @@ private static void configureToolCallBehaviorOptions( try { String json = String.format( "{\"type\":\"function\",\"function\":{\"name\":\"%s\"}}", toolChoiceName); - options.setToolChoice(BinaryData.fromObject(new ObjectMapper().readTree(json))); + + options.setToolChoice( + new ChatCompletionsToolSelection( + ChatCompletionsNamedToolSelection.fromJson( + DefaultJsonReader.fromString( + json, + new JsonOptions())))); } catch (JsonProcessingException e) { throw SKException.build("Failed to parse tool choice", e); + } catch (IOException e) { + throw new SKException(e); } return; } @@ -897,7 +923,8 @@ private static void configureToolCallBehaviorOptions( } options.setTools(toolDefinitions); - options.setToolChoice(BinaryData.fromString("auto")); + options.setToolChoice( + new ChatCompletionsToolSelection(ChatCompletionsToolSelectionPreset.AUTO)); } private static boolean hasToolCallBeenExecuted(List chatRequestMessages, diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/responseformat/ChatCompletionsJsonSchemaResponseFormat.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/responseformat/ChatCompletionsJsonSchemaResponseFormat.java new file mode 100644 index 00000000..96fa2526 --- /dev/null +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/responseformat/ChatCompletionsJsonSchemaResponseFormat.java @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.aiservices.openai.chatcompletion.responseformat; + +import com.azure.ai.openai.models.ChatCompletionsResponseFormat; +import com.azure.json.JsonWriter; +import com.microsoft.semantickernel.orchestration.responseformat.JsonResponseSchema; +import java.io.IOException; + +public class ChatCompletionsJsonSchemaResponseFormat extends ChatCompletionsResponseFormat { + + private final JsonResponseSchema schema; + private String type = "json_schema"; + + public ChatCompletionsJsonSchemaResponseFormat(JsonResponseSchema schema) { + this.schema = schema; + } + + public String getType() { + return this.type; + } + + public JsonWriter toJson(JsonWriter jsonWriter) throws IOException { + jsonWriter.writeStartObject(); + jsonWriter.writeStringField("type", this.type); + jsonWriter.writeStartObject("json_schema"); + + jsonWriter.writeBooleanField("strict", this.schema.isStrict()); + jsonWriter.writeStringField("name", this.schema.getName()); + + jsonWriter.writeRawField("schema", this.schema.getSchema()); + jsonWriter.writeEndObject(); + return jsonWriter.writeEndObject(); + } + +} diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/responseformat/JacksonResponseFormatGenerator.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/responseformat/JacksonResponseFormatGenerator.java new file mode 100644 index 00000000..93ea5b5f --- /dev/null +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/responseformat/JacksonResponseFormatGenerator.java @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.aiservices.openai.chatcompletion.responseformat; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ContainerNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.github.victools.jsonschema.generator.OptionPreset; +import com.github.victools.jsonschema.generator.SchemaGenerator; +import com.github.victools.jsonschema.generator.SchemaGeneratorConfigBuilder; +import com.github.victools.jsonschema.generator.SchemaVersion; +import com.github.victools.jsonschema.module.jackson.JacksonModule; +import com.microsoft.semantickernel.orchestration.responseformat.ResponseSchemaGenerator; + +public class JacksonResponseFormatGenerator implements ResponseSchemaGenerator { + + private final SchemaGenerator generator; + + public JacksonResponseFormatGenerator() { + JacksonModule module = new JacksonModule(); + SchemaGeneratorConfigBuilder builder = new SchemaGeneratorConfigBuilder( + SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON) + .with(module); + + builder + .forFields() + .withRequiredCheck(fieldScope -> { + return true; + }); + + generator = new SchemaGenerator(builder.build()); + } + + public JacksonResponseFormatGenerator(SchemaGenerator generator) { + this.generator = generator; + } + + @Override + public String generateSchema(Class clazz) { + ObjectNode schema = generator.generateSchema(clazz); + + sanitize(schema); + + return schema.toPrettyString(); + } + + private static void sanitize(ContainerNode schema) { + if (schema instanceof ObjectNode) { + ((ObjectNode) schema).remove("$schema"); + + if (schema.has("type") && schema.get("type").asText().equals("object")) { + ((ObjectNode) schema).put("additionalProperties", false); + } + + for (JsonNode node : (ObjectNode) schema) { + if (node instanceof ContainerNode) { + sanitize((ContainerNode) node); + } + } + } else if (schema instanceof ArrayNode) { + for (JsonNode node : (ArrayNode) schema) { + if (node instanceof ContainerNode) { + sanitize((ContainerNode) node); + } + } + } + } +} diff --git a/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/Bar.java b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/Bar.java new file mode 100644 index 00000000..45d1caa7 --- /dev/null +++ b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/Bar.java @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.aiservices.openai.chatcompletion; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class Bar { + + private final String bar; + + public Bar( + @JsonProperty("bar") String bar) { + this.bar = bar; + } + + public String getBar() { + return bar; + } + +} diff --git a/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/Baz.java b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/Baz.java new file mode 100644 index 00000000..90ad60ac --- /dev/null +++ b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/Baz.java @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.aiservices.openai.chatcompletion; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class Baz { + + @JsonProperty("bar") + private final Bar bar; + + @JsonCreator + public Baz( + @JsonProperty("bar") Bar bar) { + this.bar = bar; + } + + @JsonProperty("bar") + public Bar getBar() { + return bar; + } +} \ No newline at end of file diff --git a/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/Foo.java b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/Foo.java new file mode 100644 index 00000000..34479b22 --- /dev/null +++ b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/Foo.java @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.aiservices.openai.chatcompletion; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class Foo { + + @JsonProperty("bar") + private final T bar; + + @JsonCreator + public Foo( + @JsonProperty("bar") T bar) { + this.bar = bar; + } + + @JsonProperty("bar") + public T getBar() { + return bar; + } +} \ No newline at end of file diff --git a/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/JsonSchemaTest.java b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/JsonSchemaTest.java new file mode 100644 index 00000000..fad9ab80 --- /dev/null +++ b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/JsonSchemaTest.java @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.aiservices.openai.chatcompletion; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.microsoft.semantickernel.orchestration.responseformat.JsonSchemaResponseFormat; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class JsonSchemaTest { + + @Test + public void jacksonGenerationTest() throws JsonProcessingException { + JsonSchemaResponseFormat format = JsonSchemaResponseFormat.builder() + .setResponseFormat(Foo.class) + .setName("foo") + .build(); + + Assertions.assertEquals("foo", format.getJsonSchema().getName()); + + Assertions.assertTrue(format.getJsonSchema().getSchema() + .replaceAll("\n", "") + .replaceAll(" +", "") + .contains( + "\"type\":\"object\",\"properties\":{\"bar\":{}}")); + } + +} diff --git a/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAiChatCompletionTest.java b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAiChatCompletionTest.java index 5a25e1ae..05aebc3d 100644 --- a/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAiChatCompletionTest.java +++ b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAiChatCompletionTest.java @@ -9,7 +9,8 @@ import com.azure.core.http.HttpHeaders; import com.azure.core.http.HttpRequest; import com.azure.core.http.rest.Response; -import com.fasterxml.jackson.databind.ObjectMapper; +import com.azure.json.JsonOptions; +import com.azure.json.implementation.DefaultJsonReader; import com.microsoft.semantickernel.implementation.EmbeddedResourceLoader; import com.microsoft.semantickernel.orchestration.FunctionResultMetadata; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; @@ -104,9 +105,9 @@ public ChatCompletions getValue() { String message = EmbeddedResourceLoader.readFile("chatCompletion.txt", OpenAiChatCompletionTest.class); - return new ObjectMapper() - .readValue(String.format(message, "Snuggles"), - ChatCompletions.class); + return ChatCompletions.fromJson( + DefaultJsonReader.fromString( + String.format(message, "Snuggles"), new JsonOptions())); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 6def803a..69d83d0b 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.2.3-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml @@ -105,6 +105,18 @@ wiremock test + + + + com.github.victools + jsonschema-generator + test + + + com.github.victools + jsonschema-module-jackson + test + diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ResponseSchemaTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ResponseSchemaTest.java new file mode 100644 index 00000000..099aaa36 --- /dev/null +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ResponseSchemaTest.java @@ -0,0 +1,307 @@ +package com.microsoft.semantickernel.tests; + +import com.azure.ai.openai.OpenAIAsyncClient; +import com.azure.ai.openai.models.ChatCompletions; +import com.azure.ai.openai.models.ChatCompletionsOptions; +import com.azure.core.http.HttpHeaders; +import com.azure.core.http.HttpRequest; +import com.azure.core.http.rest.RequestOptions; +import com.azure.core.http.rest.Response; +import com.azure.json.JsonOptions; +import com.azure.json.JsonWriter; +import com.azure.json.implementation.DefaultJsonReader; +import com.azure.json.implementation.DefaultJsonWriter; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.microsoft.semantickernel.Kernel; +import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIChatCompletion; +import com.microsoft.semantickernel.contextvariables.converters.ContextVariableJacksonConverter; +import com.microsoft.semantickernel.implementation.EmbeddedResourceLoader; +import com.microsoft.semantickernel.implementation.EmbeddedResourceLoader.ResourceLocation; +import com.microsoft.semantickernel.orchestration.FunctionResult; +import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; +import com.microsoft.semantickernel.orchestration.responseformat.JsonSchemaResponseFormat; +import com.microsoft.semantickernel.semanticfunctions.HandlebarsPromptTemplateFactory; +import com.microsoft.semantickernel.semanticfunctions.KernelFunctionYaml; +import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; +import java.io.IOException; +import java.io.StringWriter; +import org.apache.commons.text.StringEscapeUtils; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import reactor.core.publisher.Mono; + +public class ResponseSchemaTest { + + public static class TestClass { + + private final String name; + + @JsonCreator + public TestClass( + @JsonProperty("name") String name) { + this.name = name; + } + + public String getName() { + return name; + } + } + + @Test + public void sendsResponseSchemaFromTemplate() throws IOException { + OpenAIAsyncClient client = getOpenAIAsyncClient( + """ + { + "name": "Test name" + } + """ + .stripIndent()); + Kernel kernel = buildKernel(client); + + var getIntent = KernelFunctionYaml.fromPromptYaml( + EmbeddedResourceLoader + .readFile("responseSchema.prompt.yaml", ResponseSchemaTest.class, + ResourceLocation.CLASSPATH_ROOT), + new HandlebarsPromptTemplateFactory()); + + FunctionResult response = getIntent.invokeAsync(kernel) + .withResultTypeAutoConversion(TestClass.class) + .block(); + + verifyCalled(client, + """ + { + "type":"json_schema", + "json_schema":{ + "strict":true, + "name":"Test", + "schema":{ + "type" : "object", + "properties" : { + "name" : { + "type" : "string" + } + }, + "required" : [ + "name" + ], + "additionalProperties" : false + } + } + } + """ + ); + } + + @Test + public void sendsResponseSchema() { + OpenAIAsyncClient client = getOpenAIAsyncClient( + """ + { + "name": "Test name" + } + """ + .stripIndent()); + + Kernel kernel = buildKernel(client); + + PromptExecutionSettings promptExecutionSettings = PromptExecutionSettings.builder() + .withResponseFormat( + JsonSchemaResponseFormat.builder() + .setResponseFormat(TestClass.class) + .setName("Test") + .build() + ) + .build(); + + FunctionResult response = kernel.invokePromptAsync( + "Generate TestClass") + .withTypeConverter(ContextVariableJacksonConverter.create(TestClass.class)) + .withResultType(TestClass.class) + .withPromptExecutionSettings(promptExecutionSettings) + .block(); + + verifyCalled(client, + """ + { + "type":"json_schema", + "json_schema":{ + "strict":true, + "name":"Test", + "schema":{ + "type" : "object", + "properties" : { + "name" : { + "type" : "string" + } + }, + "required" : [ "name" ], + "additionalProperties" : false + } + } + } + """ + ); + } + + private static void verifyCalled(OpenAIAsyncClient client, String expected) { + Mockito.verify(client, Mockito.atLeastOnce()) + .getChatCompletionsWithResponse( + Mockito.any(), + Mockito.argThat( + (ChatCompletionsOptions chatCompletionsOptions) -> { + StringWriter writer = new StringWriter(); + try { + JsonWriter jsonWriter = DefaultJsonWriter.toWriter( + writer, + new JsonOptions() + ); + JsonWriter format = chatCompletionsOptions.getResponseFormat() + .toJson(jsonWriter); + jsonWriter.flush(); + writer.flush(); + + String json = String.valueOf(writer.getBuffer()) + .replaceAll("\n", "") + .replaceAll(" +", ""); + String expectedClean = expected + .stripIndent() + .replaceAll("\n", "") + .replaceAll(" +", ""); + + return json.equals(expectedClean); + + } catch (IOException e) { + throw new RuntimeException(e); + } + }), + Mockito.any()); + } + + private @NotNull OpenAIAsyncClient getOpenAIAsyncClient(String response) { + OpenAIAsyncClient client = Mockito.mock(OpenAIAsyncClient.class); + + Mockito.when( + client.getChatCompletionsWithResponse( + Mockito.any(), + Mockito.any(), + Mockito.any())) + .thenReturn( + Mono.just( + new Response() { + @Override + public int getStatusCode() { + return 200; + } + + @Override + public HttpHeaders getHeaders() { + return new HttpHeaders(); + } + + @Override + public HttpRequest getRequest() { + return null; + } + + @Override + public ChatCompletions getValue() { + return buildResponse(response); + } + })); + return client; + } + + + private ChatCompletions buildResponse(String response) { + String str = String.format(""" + { + "choices" : [ + { + "content_filter_results" : { + "hate" : { + "filtered" : false, + "severity" : "safe" + }, + "self_harm" : { + "filtered" : false, + "severity" : "safe" + }, + "sexual" : { + "filtered" : false, + "severity" : "safe" + }, + "violence" : { + "filtered" : false, + "severity" : "safe" + } + }, + "finish_reason" : "stop", + "index" : 0, + "message" : { + "content" : "%s", + "role" : "assistant" + } + } + ], + "created" : 1707253039, + "id" : "chatcmpl-xxx", + "prompt_filter_results" : [ + { + "content_filter_results" : { + "hate" : { + "filtered" : false, + "severity" : "safe" + }, + "self_harm" : { + "filtered" : false, + "severity" : "safe" + }, + "sexual" : { + "filtered" : false, + "severity" : "safe" + }, + "violence" : { + "filtered" : false, + "severity" : "safe" + } + }, + "prompt_index" : 0 + } + ], + "usage" : { + "completion_tokens" : 131, + "prompt_tokens" : 26, + "total_tokens" : 157 + } + } + """, StringEscapeUtils.escapeJson(response)); + + try { + return ChatCompletions.fromJson( + DefaultJsonReader.fromString( + str, new JsonOptions()) + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + + private Kernel buildKernel(OpenAIAsyncClient client) { + + ChatCompletionService openAIChatCompletion = OpenAIChatCompletion.builder() + .withOpenAIAsyncClient(client) + .withModelId("a-model") + .build(); + + Kernel kernel = Kernel.builder() + .withAIService(ChatCompletionService.class, openAIChatCompletion) + .build(); + + return kernel; + + } +} diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ToolCallBehaviourTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ToolCallBehaviourTest.java index 852f5578..4f96f782 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ToolCallBehaviourTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ToolCallBehaviourTest.java @@ -282,7 +282,7 @@ private ChatCompletionService getChatCompletionService() { public static MappingBuilder buildTextResponse(String bodyMatcher, String responseBody) { return post(urlEqualTo( - "//openai/deployments/gpt-35-turbo-2/completions?api-version=2024-03-01-preview")) + "//openai/deployments/gpt-35-turbo-2/completions?api-version=2024-07-01-preview")) .withRequestBody(new ContainsPattern(bodyMatcher)) .willReturn( aResponse() @@ -293,7 +293,7 @@ public static MappingBuilder buildTextResponse(String bodyMatcher, String respon public static MappingBuilder buildResponse(String bodyMatcher, String responseBody) { return post(urlEqualTo( - "//openai/deployments/gpt-35-turbo-2/chat/completions?api-version=2024-03-01-preview")) + "//openai/deployments/gpt-35-turbo-2/chat/completions?api-version=2024-07-01-preview")) .withRequestBody(new ContainsPattern(bodyMatcher)) .willReturn( aResponse() diff --git a/api-test/integration-tests/src/test/resources/responseSchema.prompt.yaml b/api-test/integration-tests/src/test/resources/responseSchema.prompt.yaml new file mode 100644 index 00000000..08fa066a --- /dev/null +++ b/api-test/integration-tests/src/test/resources/responseSchema.prompt.yaml @@ -0,0 +1,27 @@ +name: testResponseSchema +description: Test Response Schema +template: | + Do something +template_format: handlebars +execution_settings: + default: + max_tokens: 10 + temperature: 0.2 + response_format: + type: json_schema + json_schema: + name: Test + strict: true + schema: | + { + "type" : "object", + "properties" : { + "name" : { + "type" : "string" + } + }, + "required" : [ + "name" + ], + "additionalProperties" : false + } diff --git a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-12b6744e-443f-4fe7-82e2-55cc41195ff1.json b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-12b6744e-443f-4fe7-82e2-55cc41195ff1.json index 24f15f34..9697bca8 100644 --- a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-12b6744e-443f-4fe7-82e2-55cc41195ff1.json +++ b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-12b6744e-443f-4fe7-82e2-55cc41195ff1.json @@ -2,7 +2,7 @@ "priority": 1, "request": { "method": "POST", - "url": "//openai/deployments/gpt-35-turbo-2/chat/completions?api-version=2024-03-01-preview", + "url": "//openai/deployments/gpt-35-turbo-2/chat/completions?api-version=2024-07-01-preview", "bodyPatterns": [ { "contains": "That is all" diff --git a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-71b07779-49a8-44e5-a60b-ee5b0a3ad697.json b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-71b07779-49a8-44e5-a60b-ee5b0a3ad697.json index 2b2e290b..c99d0145 100644 --- a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-71b07779-49a8-44e5-a60b-ee5b0a3ad697.json +++ b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-71b07779-49a8-44e5-a60b-ee5b0a3ad697.json @@ -1,7 +1,7 @@ { "request": { "method": "POST", - "url": "//openai/deployments/gpt-35-turbo-2/chat/completions?api-version=2024-03-01-preview" + "url": "//openai/deployments/gpt-35-turbo-2/chat/completions?api-version=2024-07-01-preview" }, "response": { "body": "{\"id\":\"chatcmpl-xxx\",\"object\":\"chat.completion\",\"created\":1707253061,\"model\":\"gpt-35-turbo\",\"prompt_filter_results\":[{\"prompt_index\":0,\"content_filter_results\":{\"hate\":{\"filtered\":false,\"severity\":\"safe\"},\"self_harm\":{\"filtered\":false,\"severity\":\"safe\"},\"sexual\":{\"filtered\":false,\"severity\":\"safe\"},\"violence\":{\"filtered\":false,\"severity\":\"safe\"}}}],\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"1979b4ce-5463-4cfb-8ec8-1d05c4b44ccf\"},\"content_filter_results\":{\"hate\":{\"filtered\":false,\"severity\":\"safe\"},\"self_harm\":{\"filtered\":false,\"severity\":\"safe\"},\"sexual\":{\"filtered\":false,\"severity\":\"safe\"},\"violence\":{\"filtered\":false,\"severity\":\"safe\"}}}],\"usage\":{\"prompt_tokens\":17,\"completion_tokens\":67,\"total_tokens\":84}}", diff --git a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo_chat_completions-0c8046c5-74ad-4836-8aa9-09da60f367a2.json b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo_chat_completions-0c8046c5-74ad-4836-8aa9-09da60f367a2.json index c909297f..d55d6bae 100644 --- a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo_chat_completions-0c8046c5-74ad-4836-8aa9-09da60f367a2.json +++ b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo_chat_completions-0c8046c5-74ad-4836-8aa9-09da60f367a2.json @@ -1,7 +1,7 @@ { "request": { "method": "POST", - "url": "//openai/deployments/gpt-35-turbo/chat/completions?api-version=2024-03-01-preview" + "url": "//openai/deployments/gpt-35-turbo/chat/completions?api-version=2024-07-01-preview" }, "response": { "body": "{\"id\":\"chatcmpl-xxx\",\"object\":\"chat.completion\",\"created\":1707253039,\"model\":\"gpt-35-turbo\",\"prompt_filter_results\":[{\"prompt_index\":0,\"content_filter_results\":{\"hate\":{\"filtered\":false,\"severity\":\"safe\"},\"self_harm\":{\"filtered\":false,\"severity\":\"safe\"},\"sexual\":{\"filtered\":false,\"severity\":\"safe\"},\"violence\":{\"filtered\":false,\"severity\":\"safe\"}}}],\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"ac9817bc-7e1a-48e4-b06c-0ff7618b88c6\"},\"content_filter_results\":{\"hate\":{\"filtered\":false,\"severity\":\"safe\"},\"self_harm\":{\"filtered\":false,\"severity\":\"safe\"},\"sexual\":{\"filtered\":false,\"severity\":\"safe\"},\"violence\":{\"filtered\":false,\"severity\":\"safe\"}}}],\"usage\":{\"prompt_tokens\":26,\"completion_tokens\":131,\"total_tokens\":157}}", diff --git a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_text-davinci-003_completions-0215b128-4822-4368-ac3d-2f580a221f00.json b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_text-davinci-003_completions-0215b128-4822-4368-ac3d-2f580a221f00.json index 3a0ea31c..15adeb40 100644 --- a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_text-davinci-003_completions-0215b128-4822-4368-ac3d-2f580a221f00.json +++ b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_text-davinci-003_completions-0215b128-4822-4368-ac3d-2f580a221f00.json @@ -1,7 +1,7 @@ { "request": { "method": "POST", - "url": "//openai/deployments/text-davinci-003/completions?api-version=2024-03-01-preview" + "url": "//openai/deployments/text-davinci-003/completions?api-version=2024-07-01-preview" }, "response": { "body": "{\"id\":\"cmpl-xxx\",\"object\":\"text_completion\",\"created\":1707253062,\"model\":\"text-davinci-003\",\"prompt_filter_results\":[{\"prompt_index\":0,\"content_filter_results\":{\"hate\":{\"filtered\":false,\"severity\":\"safe\"},\"self_harm\":{\"filtered\":false,\"severity\":\"safe\"},\"sexual\":{\"filtered\":false,\"severity\":\"safe\"},\"violence\":{\"filtered\":false,\"severity\":\"safe\"}}}],\"choices\":[{\"text\":\"7949e981-c632-422f-9b76-335a2379cd83\",\"index\":0,\"finish_reason\":\"stop\",\"logprobs\":null,\"content_filter_results\":{\"hate\":{\"filtered\":false,\"severity\":\"safe\"},\"self_harm\":{\"filtered\":false,\"severity\":\"safe\"},\"sexual\":{\"filtered\":false,\"severity\":\"safe\"},\"violence\":{\"filtered\":false,\"severity\":\"safe\"}}}],\"usage\":{\"prompt_tokens\":10,\"completion_tokens\":26,\"total_tokens\":36}}", diff --git a/api-test/pom.xml b/api-test/pom.xml index 15ed5bb6..027022de 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.3-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index f53e0e0d..ebff91a9 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.3-SNAPSHOT + 1.3.0-SNAPSHOT pom https://www.github.com/microsoft/semantic-kernel @@ -15,7 +15,7 @@ Parent pom for the Semantic Kernel Project - 1.0.0-beta.8 + 1.0.0-beta.11 10.13.0 0.10.21 2.19.1 diff --git a/samples/pom.xml b/samples/pom.xml index 61d34a0b..92f61803 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.3-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index 24a745b0..6a3f835c 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.3-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 1850e6f0..f46399ab 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.2.3-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml @@ -94,6 +94,14 @@ mysql-connector-j 9.0.0 + + com.github.victools + jsonschema-generator + + + com.github.victools + jsonschema-module-jackson + diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/responseschema/Example_ChatWithResponseFormat.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/responseschema/Example_ChatWithResponseFormat.java new file mode 100644 index 00000000..5d0aad36 --- /dev/null +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/responseschema/Example_ChatWithResponseFormat.java @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.syntaxexamples.chatcompletion.responseschema; + +import com.azure.ai.openai.OpenAIAsyncClient; +import com.azure.ai.openai.OpenAIClientBuilder; +import com.azure.core.credential.AzureKeyCredential; +import com.azure.core.credential.KeyCredential; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.microsoft.semantickernel.Kernel; +import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIChatCompletion; +import com.microsoft.semantickernel.orchestration.FunctionResult; +import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; +import com.microsoft.semantickernel.orchestration.responseformat.JsonSchemaResponseFormat; +import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; + +public class Example_ChatWithResponseFormat { + + private static final String CLIENT_KEY = System.getenv("CLIENT_KEY"); + private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY"); + + // Only required if AZURE_CLIENT_KEY is set + private static final String CLIENT_ENDPOINT = System.getenv("CLIENT_ENDPOINT"); + private static final String MODEL_ID = System.getenv() + .getOrDefault("MODEL_ID", "gpt-4o"); + + public static void main(String[] args) throws InterruptedException, JsonProcessingException { + + OpenAIAsyncClient client; + + if (AZURE_CLIENT_KEY != null) { + client = new OpenAIClientBuilder() + .credential(new AzureKeyCredential(AZURE_CLIENT_KEY)) + .endpoint(CLIENT_ENDPOINT) + .buildAsyncClient(); + + } else { + client = new OpenAIClientBuilder() + .credential(new KeyCredential(CLIENT_KEY)) + .buildAsyncClient(); + } + + ChatCompletionService openAIChatCompletion = OpenAIChatCompletion.builder() + .withOpenAIAsyncClient(client) + .withModelId(MODEL_ID) + .build(); + + Kernel kernel = Kernel.builder() + .withAIService(ChatCompletionService.class, openAIChatCompletion) + .build(); + + PromptExecutionSettings promptExecutionSettings = PromptExecutionSettings.builder() + .withResponseFormat( + JsonSchemaResponseFormat.builder() + .setResponseFormat(Pet.class) + .setName("Pet") + .build()) + .build(); + + FunctionResult response = kernel + .invokePromptAsync("Generate an example pet that can be used in test code") + .withResultTypeAutoConversion(Pet.class) + .withPromptExecutionSettings(promptExecutionSettings) + .block(); + + System.out.println(new ObjectMapper() + .writerWithDefaultPrettyPrinter() + .writeValueAsString(response.getResult())); + } + +} diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/responseschema/Example_ChatWithResponseFormatToolCall.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/responseschema/Example_ChatWithResponseFormatToolCall.java new file mode 100644 index 00000000..cd227ebd --- /dev/null +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/responseschema/Example_ChatWithResponseFormatToolCall.java @@ -0,0 +1,90 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.syntaxexamples.chatcompletion.responseschema; + +import com.azure.ai.openai.OpenAIAsyncClient; +import com.azure.ai.openai.OpenAIClientBuilder; +import com.azure.core.credential.AzureKeyCredential; +import com.azure.core.credential.KeyCredential; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.microsoft.semantickernel.Kernel; +import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIChatCompletion; +import com.microsoft.semantickernel.orchestration.FunctionResult; +import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; +import com.microsoft.semantickernel.orchestration.ToolCallBehavior; +import com.microsoft.semantickernel.plugin.KernelPluginFactory; +import com.microsoft.semantickernel.samples.syntaxexamples.chatcompletion.responseschema.Pet.AnimalType; +import com.microsoft.semantickernel.samples.syntaxexamples.chatcompletion.responseschema.Pet.Weight; +import com.microsoft.semantickernel.samples.syntaxexamples.chatcompletion.responseschema.Pet.WeightUnit; +import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; +import com.microsoft.semantickernel.semanticfunctions.annotations.KernelFunctionParameter; +import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; + +public class Example_ChatWithResponseFormatToolCall { + + private static final String CLIENT_KEY = System.getenv("CLIENT_KEY"); + private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY"); + + // Only required if AZURE_CLIENT_KEY is set + private static final String CLIENT_ENDPOINT = System.getenv("CLIENT_ENDPOINT"); + private static final String MODEL_ID = System.getenv() + .getOrDefault("MODEL_ID", "gpt-4o"); + + public static void main(String[] args) throws InterruptedException, JsonProcessingException { + + OpenAIAsyncClient client; + + if (AZURE_CLIENT_KEY != null) { + client = new OpenAIClientBuilder() + .credential(new AzureKeyCredential(AZURE_CLIENT_KEY)) + .endpoint(CLIENT_ENDPOINT) + .buildAsyncClient(); + + } else { + client = new OpenAIClientBuilder() + .credential(new KeyCredential(CLIENT_KEY)) + .buildAsyncClient(); + } + + ChatCompletionService openAIChatCompletion = OpenAIChatCompletion.builder() + .withOpenAIAsyncClient(client) + .withModelId(MODEL_ID) + .build(); + + Kernel kernel = Kernel.builder() + .withAIService(ChatCompletionService.class, openAIChatCompletion) + .withPlugin(KernelPluginFactory.createFromObject(new PetStore(), "PetStore")) + .build(); + + PromptExecutionSettings promptExecutionSettings = PromptExecutionSettings.builder() + .withJsonSchemaResponseFormat(Pet.class) + .build(); + + FunctionResult response = kernel.invokePromptAsync("Get pet with id 1234") + .withResultTypeAutoConversion(Pet.class) + .withPromptExecutionSettings(promptExecutionSettings) + .withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(true)) + .block(); + + System.out.println(new ObjectMapper() + .writerWithDefaultPrettyPrinter() + .writeValueAsString(response.getResult())); + } + + public static class PetStore { + + @DefineKernelFunction(name = "getPetById", returnType = "com.microsoft.semantickernel.samples.syntaxexamples.chatcompletion.responseschema.Pet") + public Pet getPetById( + @KernelFunctionParameter(name = "id") String id) { + if (id.equals("1234")) { + return new Pet( + "Test name", + AnimalType.CAT, + 5, + new Weight(5.0, WeightUnit.KG)); + } + return null; + } + } + +} diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/responseschema/Pet.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/responseschema/Pet.java new file mode 100644 index 00000000..b88a8e53 --- /dev/null +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/responseschema/Pet.java @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.syntaxexamples.chatcompletion.responseschema; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class Pet { + + private final String name; + private final AnimalType type; + private final int age; + private final Weight weight; + + public static enum AnimalType { + CAT, DOG, FISH + } + + public static enum WeightUnit { + KG, LB + } + + public static class Weight { + + private final double value; + private final WeightUnit unit; + + @JsonCreator + public Weight( + @JsonProperty("value") double value, + @JsonProperty("unit") WeightUnit unit) { + this.value = value; + this.unit = unit; + } + + public double getValue() { + return value; + } + + public WeightUnit getUnit() { + return unit; + } + } + + @JsonCreator + public Pet( + @JsonProperty("name") String name, + @JsonProperty("type") AnimalType type, + @JsonProperty("age") int age, + @JsonProperty("weight") Weight weight) { + this.name = name; + this.type = type; + this.age = age; + this.weight = weight; + } + + public String getName() { + return name; + } + + public AnimalType getType() { + return type; + } + + public int getAge() { + return age; + } + + public Weight getWeight() { + return weight; + } +} \ No newline at end of file diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java index 2a1a49ab..ab2b72aa 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java @@ -118,17 +118,19 @@ public static void dataStorageWithAzureAISearch( OpenAITextEmbeddingGenerationService embeddingGeneration) { // Create a new Azure AI Search vector store - var azureAISearchVectorStore = com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStore.builder() + var azureAISearchVectorStore = com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStore + .builder() .withSearchIndexAsyncClient(searchClient) .withOptions(new AzureAISearchVectorStoreOptions()) .build(); String collectionName = "skgithubfiles"; - var collection = (AzureAISearchVectorStoreRecordCollection) azureAISearchVectorStore.getCollection( - collectionName, - AzureAISearchVectorStoreRecordCollectionOptions.builder() - .withRecordClass(GitHubFile.class) - .build()); + var collection = (AzureAISearchVectorStoreRecordCollection) azureAISearchVectorStore + .getCollection( + collectionName, + AzureAISearchVectorStoreRecordCollectionOptions.builder() + .withRecordClass(GitHubFile.class) + .build()); // Create collection if it does not exist and store data collection @@ -146,17 +148,17 @@ public static void dataStorageWithAzureAISearch( } var searchResult = results.get(0); System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", - searchResult.getScore(), searchResult.getRecord().link, searchResult.getRecord().description); + searchResult.getScore(), searchResult.getRecord().link, + searchResult.getRecord().description); } - private static Mono>> search( - String searchText, - AzureAISearchVectorStoreRecordCollection recordCollection, - OpenAITextEmbeddingGenerationService embeddingGeneration) { + String searchText, + AzureAISearchVectorStoreRecordCollection recordCollection, + OpenAITextEmbeddingGenerationService embeddingGeneration) { return embeddingGeneration.generateEmbeddingsAsync(Collections.singletonList(searchText)) - .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); + .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); } private static Mono> storeData( diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/resources/log4j2.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/resources/log4j2.xml index 2774f660..f34bc201 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/resources/log4j2.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/resources/log4j2.xml @@ -19,6 +19,9 @@ + + + diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index 59b998bf..d9ab68bc 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.2.3-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index 755964a2..71a74025 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.3-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index ee374eae..8917499e 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.2.3-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index 56fd812a..33e90396 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.2.3-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 94e9c598..53d501db 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.3-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index f9a1f140..c94ccdca 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.3-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index 7d3ac488..17b7143f 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.2.3-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index fd85b83a..1c13db81 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.2.3-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index 16581d2f..d0dafe5d 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.3-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariable.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariable.java index f749974e..5e79b872 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariable.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariable.java @@ -128,7 +128,7 @@ public static ContextVariable convert( if (typeOfActualReturnedType != null) { // Try the to object T converted = typeOfActualReturnedType.getConverter().toObject(contextVariableTypes, it, - requestedResultType); + requestedResultType, false); if (converted != null) { return contextVariableTypes.getVariableTypeForClass(requestedResultType) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypeConverter.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypeConverter.java index f2e9944d..ac1b9dd2 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypeConverter.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypeConverter.java @@ -123,6 +123,22 @@ public ContextVariableTypeConverter( @Nullable @SuppressWarnings("unchecked") public U toObject(ContextVariableTypes types, @Nullable Object t, Class clazz) { + return toObject(types, t, clazz, true); + } + + /** + * Use this converter to convert the object to the type of the context variable. + * + * @param the type to convert to + * @param t the object to convert + * @param clazz the class of the type to convert to + * @param logWarnings whether to log warnings + * @return the converted object + */ + @Nullable + @SuppressWarnings("unchecked") + public U toObject(ContextVariableTypes types, @Nullable Object t, Class clazz, + boolean logWarnings) { if (t == null) { return null; } @@ -145,8 +161,11 @@ public U toObject(ContextVariableTypes types, @Nullable Object t, Class c return (U) converter.get().toObject((T) t); } - LOGGER.warn(SemanticKernelResources.getString("no.converter.found.for.to"), t.getClass(), - clazz); + if (logWarnings) { + LOGGER.warn(SemanticKernelResources.getString("no.converter.found.for.to"), + t.getClass(), + clazz); + } return null; } @@ -173,7 +192,7 @@ public T fromObject(@Nullable Object s) { * constructor. * * @param types the context variable types, if {@code null} the global types are used - * @param t the type to convert + * @param t the type to convert * @return the prompt string */ public String toPromptString(@Nullable ContextVariableTypes types, @Nullable T t) { diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypes.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypes.java index 21e12e70..76ed87fa 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypes.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypes.java @@ -104,7 +104,11 @@ public static ContextVariableTypes getGlobalTypes() { * @param contextVariableTypes The collection of context variable types to copy. */ public ContextVariableTypes(ContextVariableTypes contextVariableTypes) { - this.variableTypes = new HashMap<>(contextVariableTypes.variableTypes); + if (contextVariableTypes != null) { + this.variableTypes = new HashMap<>(contextVariableTypes.variableTypes); + } else { + this.variableTypes = new HashMap<>(); + } } /** diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java index d6ca091f..405c32a1 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java @@ -8,28 +8,35 @@ public class SemanticKernelResources { + private static final String RESOURCE_BUNDLE_CLASS = "com.microsoft.semantickernel.localization.ResourceBundle"; + private static ResourceBundle RESOURCE_BUNDLE; private static Locale LOCALE; static { - LOCALE = new Locale( + LOCALE = setLocale(new Locale( System.getProperty("semantickernel.locale", - String.valueOf(Locale.getDefault().getLanguage()))); - - setLocale(LOCALE); + String.valueOf(Locale.getDefault().getLanguage())))); + RESOURCE_BUNDLE = setResourceBundle(LOCALE); } - public static void setLocale(Locale locale) { - LOCALE = locale; + public static ResourceBundle setResourceBundle(Locale locale) { ResourceBundle resourceBundle; try { resourceBundle = PropertyResourceBundle.getBundle( - "com.microsoft.semantickernel.localization.ResourceBundle", locale); + RESOURCE_BUNDLE_CLASS, locale); } catch (MissingResourceException e) { resourceBundle = PropertyResourceBundle.getBundle( - "com.microsoft.semantickernel.localization.ResourceBundle"); + RESOURCE_BUNDLE_CLASS); } RESOURCE_BUNDLE = resourceBundle; + return resourceBundle; + } + + public static Locale setLocale(Locale locale) { + LOCALE = locale; + setResourceBundle(locale); + return locale; } public static String localize(String id, String defaultValue) { diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java index bcae014d..630ce286 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java @@ -6,6 +6,7 @@ import com.microsoft.semantickernel.contextvariables.ContextVariableType; import com.microsoft.semantickernel.contextvariables.ContextVariableTypeConverter; import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; +import com.microsoft.semantickernel.contextvariables.converters.ContextVariableJacksonConverter; import com.microsoft.semantickernel.exceptions.SKException; import com.microsoft.semantickernel.hooks.KernelHook; import com.microsoft.semantickernel.hooks.KernelHooks; @@ -102,6 +103,8 @@ private static void performSubscribe( function.getName()); } + InvocationContext contextClone = new InvocationContext(context); + function .invokeAsync( kernel, @@ -110,8 +113,8 @@ private static void performSubscribe( .withVariables(arguments) .build(), null, - new InvocationContext(context)) - .handle(convertToType(variableType)) + contextClone) + .handle(convertToType(variableType, contextClone.getContextVariableTypes())) .onErrorResume(e -> { if (e instanceof NoSuchElementException) { return Mono.empty(); @@ -123,13 +126,17 @@ private static void performSubscribe( } private static BiConsumer, SynchronousSink>> convertToType( - @Nullable ContextVariableType variableType) { + @Nullable ContextVariableType variableType, + @Nullable ContextVariableTypes contextVariableTypes) { return (result, sink) -> { // If a specific result type was requested, convert the result to that type. if (variableType != null) { try { + ContextVariableTypes types = new ContextVariableTypes(contextVariableTypes); + types.putConverter(variableType.getConverter()); + sink.next(new FunctionResult<>( - ContextVariable.convert(result.getResult(), variableType), + ContextVariable.convert(result.getResult(), variableType.getClazz(), types), result.getMetadata(), result.getUnconvertedResult())); } catch (Exception e) { @@ -190,6 +197,23 @@ public FunctionInvocation withResultType(ContextVariableType resultTyp .withTypes(contextVariableTypes); } + /** + * Supply the result type of function invocation. Also registers a type converter for the given + * type using {@code} ContextVariableJacksonConverter.create}. + * + * @param resultType The arguments to supply to the function invocation. + * @param The type of the result of the function invocation. + * @return A new {@code FunctionInvocation} for fluent chaining. + */ + public FunctionInvocation withResultTypeAutoConversion(Class resultType) { + try { + return withTypeConverter(ContextVariableJacksonConverter.create(resultType)) + .withResultType(contextVariableTypes.getVariableTypeForSuperClass(resultType)); + } catch (SKException e) { + return withResultType(ContextVariableTypes.getGlobalVariableTypeForClass(resultType)); + } + } + /** * Supply the result type of function invocation. Uses the global context variable types. * diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/PromptExecutionSettings.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/PromptExecutionSettings.java index 12dc0bbc..19dfbd0d 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/PromptExecutionSettings.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/PromptExecutionSettings.java @@ -3,11 +3,15 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.microsoft.semantickernel.exceptions.SKException; +import com.microsoft.semantickernel.orchestration.responseformat.JsonObjectResponseFormat; +import com.microsoft.semantickernel.orchestration.responseformat.JsonSchemaResponseFormat; +import com.microsoft.semantickernel.orchestration.responseformat.ResponseFormat; +import com.microsoft.semantickernel.orchestration.responseformat.TextResponseFormat; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Objects; import javax.annotation.Nullable; @@ -97,7 +101,6 @@ public class PromptExecutionSettings { private final String user; private final List stopSequences; private final Map tokenSelectionBiases; - @Nullable private final ResponseFormat responseFormat; /** @@ -115,7 +118,8 @@ public class PromptExecutionSettings { * @param user The user to associate with the prompt execution. * @param stopSequences The stop sequences to use for prompt execution. * @param tokenSelectionBiases The token selection biases to use for prompt execution. - * @param responseFormat The response format to use for prompt execution {@link ResponseFormat}. + * @param responseFormat The response format to use for prompt execution + * {@link ResponseFormat}, Defaults to TextResponseFormat. */ @JsonCreator public PromptExecutionSettings( @@ -131,7 +135,7 @@ public PromptExecutionSettings( @JsonProperty(USER) String user, @Nullable @JsonProperty(STOP_SEQUENCES) List stopSequences, @Nullable @JsonProperty(TOKEN_SELECTION_BIASES) Map tokenSelectionBiases, - @Nullable @JsonProperty(RESPONSE_FORMAT) String responseFormat) { + @Nullable @JsonProperty(RESPONSE_FORMAT) ResponseFormat responseFormat) { this.serviceId = serviceId != null ? serviceId : DEFAULT_SERVICE_ID; this.modelId = modelId != null ? modelId : ""; this.temperature = clamp(temperature, 0d, 2d, DEFAULT_TEMPERATURE); @@ -150,10 +154,10 @@ public PromptExecutionSettings( : Collections.emptyMap(); this.tokenSelectionBiases.replaceAll((k, v) -> clamp(v, -100, 100, 0)); - if (responseFormat != null && !responseFormat.isEmpty()) { - this.responseFormat = ResponseFormat.valueOf(responseFormat.toUpperCase(Locale.ROOT)); + if (responseFormat == null) { + this.responseFormat = new TextResponseFormat(); } else { - this.responseFormat = null; + this.responseFormat = responseFormat; } } @@ -374,7 +378,7 @@ public boolean equals(Object obj) { * * @return The response format to use for prompt execution. */ - @Nullable + @JsonProperty(RESPONSE_FORMAT) public ResponseFormat getResponseFormat() { return responseFormat; } @@ -550,7 +554,47 @@ public Builder withTokenSelectionBiases(Map tokenSelectionBias */ public Builder withResponseFormat(ResponseFormat responseFormat) { if (responseFormat != null) { - settings.put(RESPONSE_FORMAT, responseFormat.toString()); + settings.put(RESPONSE_FORMAT, responseFormat); + } + return this; + } + + /** + * Set the response format to use for prompt execution. + * + * @param responseFormat The response format to use for prompt execution. + * @return This builder. + */ + public Builder withResponseFormat(ResponseFormat.Type responseFormat) { + switch (responseFormat) { + case JSON_OBJECT: + settings.put(RESPONSE_FORMAT, new JsonObjectResponseFormat()); + break; + case TEXT: + settings.put(RESPONSE_FORMAT, new TextResponseFormat()); + break; + case JSON_SCHEMA: + throw new SKException( + "Cannot set JSON_SCHEMA response format without a schema, use withResponseFormat(ResponseFormat responseFormat)"); + } + + return this; + } + + /** + * Set the response format to use a json schema generated for the given class. The name of + * the response format will be the name of the class. + * + * @param responseFormat The response format type. + * @return This builder. + */ + public Builder withJsonSchemaResponseFormat(Class responseFormat) { + if (responseFormat != null) { + settings.put(RESPONSE_FORMAT, + JsonSchemaResponseFormat.builder() + .setResponseFormat(responseFormat) + .setName(responseFormat.getSimpleName()) + .build()); } return this; } @@ -576,7 +620,7 @@ public PromptExecutionSettings build() { (List) settings.getOrDefault(STOP_SEQUENCES, Collections.emptyList()), (Map) settings.getOrDefault(TOKEN_SELECTION_BIASES, Collections.emptyMap()), - (String) settings.getOrDefault(RESPONSE_FORMAT, null)); + (ResponseFormat) settings.getOrDefault(RESPONSE_FORMAT, new TextResponseFormat())); } } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/ResponseFormat.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/ResponseFormat.java deleted file mode 100644 index 5442315e..00000000 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/ResponseFormat.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.orchestration; - -public enum ResponseFormat { - - /** - * Only valid for openai chat completion, with GPT-4 and gpt-3.5-turbo-1106+ models. - */ - JSON_OBJECT, TEXT; -} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonObjectResponseFormat.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonObjectResponseFormat.java new file mode 100644 index 00000000..6d9a107a --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonObjectResponseFormat.java @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.orchestration.responseformat; + +import com.fasterxml.jackson.annotation.JsonCreator; + +public class JsonObjectResponseFormat extends ResponseFormat { + + @JsonCreator + public JsonObjectResponseFormat() { + super(Type.JSON_OBJECT); + } +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonResponseSchema.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonResponseSchema.java new file mode 100644 index 00000000..be92d417 --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonResponseSchema.java @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.orchestration.responseformat; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class JsonResponseSchema extends ResponseSchema { + + private final String name; + private final String schema; + private final boolean strict; + + @JsonCreator + public JsonResponseSchema( + @JsonProperty("name") String name, + @JsonProperty("schema") String schema, + @JsonProperty("strict") boolean strict) { + this.name = name; + this.schema = schema; + this.strict = strict; + } + + public String getName() { + return name; + } + + public String getSchema() { + return schema; + } + + public boolean isStrict() { + return strict; + } + +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonSchemaResponseFormat.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonSchemaResponseFormat.java new file mode 100644 index 00000000..68547d3a --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonSchemaResponseFormat.java @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.orchestration.responseformat; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.microsoft.semantickernel.exceptions.SKException; +import javax.annotation.Nullable; + +public class JsonSchemaResponseFormat extends ResponseFormat { + + private final JsonResponseSchema jsonSchema; + + @JsonCreator + public JsonSchemaResponseFormat( + @JsonProperty("json_schema") JsonResponseSchema jsonSchema) { + super(Type.JSON_SCHEMA); + this.jsonSchema = jsonSchema; + } + + @JsonProperty("json_schema") + public JsonResponseSchema getJsonSchema() { + return jsonSchema; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + + @Nullable + private JsonResponseSchema jsonResponseSchema = null; + @Nullable + private String jsonSchema = null; + @Nullable + private String name = null; + private boolean strict = true; + + public Builder setResponseFormat(Class clazz, + ResponseSchemaGenerator responseSchemaGenerator) { + name = clazz.getSimpleName(); + return setJsonSchema(responseSchemaGenerator.generateSchema(clazz)); + } + + public Builder setResponseFormat(Class clazz) { + name = clazz.getSimpleName(); + setJsonSchema(ResponseSchemaGenerator.jacksonGenerator().generateSchema(clazz)); + return this; + } + + public Builder setJsonResponseSchema(JsonResponseSchema jsonResponseSchema) { + this.jsonResponseSchema = jsonResponseSchema; + return this; + } + + public Builder setJsonSchema(String jsonSchema) { + this.jsonSchema = jsonSchema; + return this; + } + + public Builder setName(String name) { + this.name = name; + return this; + } + + public Builder setStrict(boolean strict) { + this.strict = strict; + return this; + } + + public JsonSchemaResponseFormat build() { + + if (jsonResponseSchema != null) { + return new JsonSchemaResponseFormat(jsonResponseSchema); + } + + if (jsonSchema == null) { + throw new SKException("Response format not set"); + } + + if (name == null) { + throw new SKException("Json format name not set"); + } + + return new JsonSchemaResponseFormat(new JsonResponseSchema(name, jsonSchema, strict)); + } + } +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseFormat.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseFormat.java new file mode 100644 index 00000000..beb68760 --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseFormat.java @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.orchestration.responseformat; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo.As; +import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; + +@JsonTypeInfo(use = Id.NAME, include = As.EXISTING_PROPERTY, property = "type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = JsonSchemaResponseFormat.class, name = "json_schema", names = { + "json_schema", "JSON_SCHEMA" }), + @JsonSubTypes.Type(value = TextResponseFormat.class, name = "text", names = { "text", + "TEXT" }), + @JsonSubTypes.Type(value = JsonObjectResponseFormat.class, name = "json_object", names = { + "json_object", "JSON_OBJECT" }), +}) +public abstract class ResponseFormat { + + public static enum Type { + JSON_OBJECT, + /** + * Only valid for openai chat completion, with GPT-4 and gpt-3.5-turbo-1106+ models. + */ + JSON_SCHEMA, TEXT; + } + + private final Type type; + + public ResponseFormat(Type type) { + this.type = type; + } + + @JsonProperty("type") + public Type getType() { + return type; + } + +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseSchema.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseSchema.java new file mode 100644 index 00000000..5d140330 --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseSchema.java @@ -0,0 +1,6 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.orchestration.responseformat; + +public abstract class ResponseSchema { + +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseSchemaGenerator.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseSchemaGenerator.java new file mode 100644 index 00000000..f9f66849 --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseSchemaGenerator.java @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.orchestration.responseformat; + +import com.microsoft.semantickernel.exceptions.SKException; +import com.microsoft.semantickernel.implementation.ServiceLoadUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Interface for generating json response schemas for a given class. + */ +public interface ResponseSchemaGenerator { + + Logger LOGGER = LoggerFactory.getLogger(ResponseSchemaGenerator.class); + + /** + * Generate a json schema for the given class. + * + * @param clazz The class to generate a schema for. + * @return The json schema. + */ + public String generateSchema(Class clazz); + + /** + * Load a response schema generator based on the Jackson library, requires that + * com.github.victools:jsonschema-generator has been added to the class path. + * + * @return The response schema generator. + */ + public static ResponseSchemaGenerator jacksonGenerator() { + try { + return loadGenerator( + "com.microsoft.semantickernel.aiservices.openai.chatcompletion.responseformat.JacksonResponseFormatGenerator"); + } catch (NoClassDefFoundError e) { + LOGGER.error( + "The Jackson response schema generator relies on the optional dependencies 'com.github.victools:jsonschema-generator', and 'com.github.victools:jsonschema-module-jackson'. To use this feature, please add this dependency to your project."); + throw new SKException( + "The Jackson response schema generator relies on the optional dependency 'com.github.victools:jsonschema-generator', and 'com.github.victools:jsonschema-module-jackson'. To use this feature, please add this dependency to your project."); + } + } + + public static ResponseSchemaGenerator loadGenerator(String className) { + return ServiceLoadUtil + .findServiceLoader(ResponseSchemaGenerator.class, + className) + .get(); + } +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/TextResponseFormat.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/TextResponseFormat.java new file mode 100644 index 00000000..daf0e021 --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/TextResponseFormat.java @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.orchestration.responseformat; + +import com.fasterxml.jackson.annotation.JsonCreator; + +public class TextResponseFormat extends ResponseFormat { + + @JsonCreator + public TextResponseFormat() { + super(Type.TEXT); + } +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateConfig.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateConfig.java index 5f69ad46..93df46f3 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateConfig.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateConfig.java @@ -86,7 +86,7 @@ protected PromptTemplateConfig(String template) { */ @JsonCreator public PromptTemplateConfig( - @JsonProperty("schema") int schema, + @Nullable @JsonProperty("schema") Integer schema, @Nullable @JsonProperty("name") String name, @Nullable @JsonProperty("template") String template, @Nullable @JsonProperty(value = "template_format", defaultValue = SEMANTIC_KERNEL_TEMPLATE_FORMAT) String templateFormat, @@ -95,6 +95,9 @@ public PromptTemplateConfig( @Nullable @JsonProperty("input_variables") List inputVariables, @Nullable @JsonProperty("output_variable") OutputVariable outputVariable, @Nullable @JsonProperty("execution_settings") Map executionSettings) { + if (schema == null) { + schema = CURRENT_SCHEMA; + } this.schema = schema; this.name = name; this.template = template; diff --git a/semantickernel-api/src/test/java/com/microsoft/semantickernel/orchestration/PromptExecutionSettingsTest.java b/semantickernel-api/src/test/java/com/microsoft/semantickernel/orchestration/PromptExecutionSettingsTest.java index daec4722..9c21b115 100644 --- a/semantickernel-api/src/test/java/com/microsoft/semantickernel/orchestration/PromptExecutionSettingsTest.java +++ b/semantickernel-api/src/test/java/com/microsoft/semantickernel/orchestration/PromptExecutionSettingsTest.java @@ -5,6 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.ObjectMapper; +import com.microsoft.semantickernel.orchestration.responseformat.ResponseFormat.Type; import org.junit.jupiter.api.Test; public class PromptExecutionSettingsTest { @@ -113,7 +114,8 @@ void testJsonDeserializeAndBuilder() throws Exception { + "\"best_of\":3," + "\"results_per_prompt\":5," + "\"model_id\":\"custom-model\"," - + "\"user\":\"custom-user\"" + + "\"user\":\"custom-user\"," + + "\"response_format\" : {\"type\" : \"text\"}" + "}"; PromptExecutionSettings settingsFromJson = new ObjectMapper().readValue(json, PromptExecutionSettings.class); @@ -129,8 +131,13 @@ void testJsonDeserializeAndBuilder() throws Exception { .withResultsPerPrompt(5) .withModelId("custom-model") .withUser("custom-user") + .withResponseFormat(Type.TEXT) .build(); - assertEquals(settingsFromBuilder, settingsFromJson); + assertEquals( + new ObjectMapper().writerWithDefaultPrettyPrinter() + .writeValueAsString(settingsFromBuilder), + new ObjectMapper().writerWithDefaultPrettyPrinter() + .writeValueAsString(settingsFromJson)); } } diff --git a/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionYamlTest.java b/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionYamlTest.java index 513f4fd7..7f89d4db 100644 --- a/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionYamlTest.java +++ b/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionYamlTest.java @@ -5,6 +5,8 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.fail; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; import java.nio.file.Path; import java.util.Arrays; @@ -78,10 +80,21 @@ private void testFromPromptYaml(String templateFormat) throws Exception { assertNotNull(result); assertEquals(expResult.getName(), result.getName()); assertEquals(expResult.getDescription(), result.getDescription()); - assertEquals(expResult.getExecutionSettings(), result.getExecutionSettings()); + assertEqualsJson(expResult.getExecutionSettings(), result.getExecutionSettings()); assertEquals(expResult.getMetadata(), result.getMetadata()); } + public static void assertEqualsJson(Object a, Object b) { + try { + assertEquals(new ObjectMapper().writerWithDefaultPrettyPrinter() + .writeValueAsString(a), + new ObjectMapper().writerWithDefaultPrettyPrinter() + .writeValueAsString(b)); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + /** * Test of fromYaml method, of class KernelFunctionYaml. */ diff --git a/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateConfigTest.java b/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateConfigTest.java index ea34400a..387ae06b 100644 --- a/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateConfigTest.java +++ b/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateConfigTest.java @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.semanticfunctions; +import static com.microsoft.semantickernel.semanticfunctions.KernelFunctionYamlTest.assertEqualsJson; import static org.junit.jupiter.api.Assertions.assertEquals; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; @@ -52,7 +53,7 @@ void testInstanceMadeWithBuilderEqualsInstanceMadeWithConstructor() { }) .build(); - assertEquals(expected, result); + assertEqualsJson(expected, result); } @Test @@ -145,6 +146,6 @@ void testParseFromJson() throws Exception { + "}"; PromptTemplateConfig result = PromptTemplateConfig.parseFromJson(jsonString); - assertEquals(expected, result); + assertEqualsJson(expected, result); } } diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index dfb1eca2..31def1ad 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.2.3-SNAPSHOT + 1.3.0-SNAPSHOT pom Semantic Kernel Java BOM @@ -94,7 +94,7 @@ com.azure azure-ai-openai - 1.0.0-beta.8 + 1.0.0-beta.11 com.azure @@ -192,6 +192,21 @@ google-cloud-vertexai 1.1.0 + + + + + com.github.victools + jsonschema-generator + 4.36.0 + true + + + com.github.victools + jsonschema-module-jackson + 4.36.0 + true + diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 5467df12..62dbf018 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.3-SNAPSHOT + 1.3.0-SNAPSHOT semantickernel-experimental From 5fb5e8d17ecb37380d8386cd425d617c1304aa54 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Tue, 17 Sep 2024 18:34:13 +0100 Subject: [PATCH 044/277] Fix build --- CHANGELOG.md | 2 +- .../com/microsoft/semantickernel/tests/RenderingTest.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1209d7d..34931b26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ - Added support for Json Schema to Open AI Chat Completions - Upgraded to openai sdk 1.0.0-beta.11 -- Added convinience method `FunctionInvocation.withResultTypeAutoConversion` which sets the return type and registers a +- Added convenience method `FunctionInvocation.withResultTypeAutoConversion` which sets the return type and registers a type converter based on Jackson for the return type. ### Bug Fixes diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/RenderingTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/RenderingTest.java index 39b11182..4c24eb92 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/RenderingTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/RenderingTest.java @@ -240,10 +240,10 @@ public void canHandleIterableReturnFromFunction() { .block(); String requestBody = wm.getAllServeEvents().get(0).getRequest().getBodyAsString(); - Assertions.assertTrue(requestBody.contains("{\"role\":\"user\",\"content\":\"[]")); - Assertions.assertTrue(requestBody.contains("{\"role\":\"user\",\"content\":\"[1, 2, 3]")); - Assertions.assertTrue(requestBody.contains("{\"role\":\"user\",\"content\":\"[4, 5, 6]")); - Assertions.assertTrue(requestBody.contains("{\"role\":\"user\",\"content\":\"[7, 8, 9]")); + Assertions.assertTrue(requestBody.contains("{\"content\":\"[]")); + Assertions.assertTrue(requestBody.contains("{\"content\":\"[1, 2, 3]")); + Assertions.assertTrue(requestBody.contains("{\"content\":\"[4, 5, 6]")); + Assertions.assertTrue(requestBody.contains("{\"content\":\"[7, 8, 9]")); } private Kernel buildTextKernel() { From 6746c861dc80c416b547d8086d9871f25fca6729 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Wed, 18 Sep 2024 00:47:00 -0700 Subject: [PATCH 045/277] Add search for redis json --- .../connectors/memory/{ => jdbc}/Hotel.java | 4 +- .../JDBCVectorStoreRecordCollectionTest.java | 1 - .../memory/jdbc/JDBCVectorStoreTest.java | 1 - .../tests/connectors/memory/redis/Hotel.java | 98 ++++++++++++++++ ...ashSetVectorStoreRecordCollectionTest.java | 106 +++++++++++++----- ...isJsonVectorStoreRecordCollectionTest.java | 78 ++++++++++--- .../memory/redis/RedisVectorStoreTest.java | 2 +- ...disHashSetVectorStoreRecordCollection.java | 102 +++++++++++++++-- ...SetVectorStoreRecordCollectionOptions.java | 35 +++++- .../RedisHashSetVectorStoreRecordMapper.java | 36 ++++-- .../RedisJsonVectorStoreRecordCollection.java | 106 ++++++++++++++++-- ...sonVectorStoreRecordCollectionOptions.java | 42 ++++++- ...disVectorStoreCollectionCreateMapping.java | 8 +- ...disVectorStoreCollectionSearchMapping.java | 85 ++++++++++++++ .../filter/RedisEqualToFilterClause.java | 44 ++++++++ 15 files changed, 667 insertions(+), 81 deletions(-) rename api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/{ => jdbc}/Hotel.java (96%) create mode 100644 api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/filter/RedisEqualToFilterClause.java diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java similarity index 96% rename from api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java rename to api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java index 13ca8d17..b518e913 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java @@ -1,4 +1,4 @@ -package com.microsoft.semantickernel.tests.connectors.memory; +package com.microsoft.semantickernel.tests.connectors.memory.jdbc; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -12,7 +12,7 @@ public class Hotel { @VectorStoreRecordKeyAttribute private final String id; - @VectorStoreRecordDataAttribute + @VectorStoreRecordDataAttribute(isFilterable = true) private final String name; @VectorStoreRecordDataAttribute diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java index 9c472868..28a45e27 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -10,7 +10,6 @@ import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; -import com.microsoft.semantickernel.tests.connectors.memory.Hotel; import com.mysql.cj.jdbc.MysqlDataSource; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java index fe5ec1b9..d70e5a5b 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java @@ -6,7 +6,6 @@ import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.tests.connectors.memory.Hotel; import com.mysql.cj.jdbc.MysqlDataSource; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java new file mode 100644 index 00000000..04c2f138 --- /dev/null +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java @@ -0,0 +1,98 @@ +package com.microsoft.semantickernel.tests.connectors.memory.redis; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; + +import java.util.List; + +public class Hotel { + @VectorStoreRecordKeyAttribute + private final String id; + + @VectorStoreRecordDataAttribute(isFilterable = true) + private final String name; + + @VectorStoreRecordDataAttribute + private final int code; + + @JsonProperty("summary") + @VectorStoreRecordDataAttribute() + private final String description; + + @JsonProperty("summaryEmbedding1") + @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "euclidean") + private final List euclidean; + + @JsonProperty("summaryEmbedding2") + @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "cosineDistance") + private final List cosineDistance; + + @JsonProperty("summaryEmbedding3") + @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "dotProduct") + private final List dotProduct; + @VectorStoreRecordDataAttribute + private double rating; + + public Hotel() { + this(null, null, 0, null, null, null, null, 0.0); + } + + @JsonCreator + public Hotel( + @JsonProperty("id") String id, + @JsonProperty("name") String name, + @JsonProperty("code") int code, + @JsonProperty("summary") String description, + @JsonProperty("summaryEmbedding1") List euclidean, + @JsonProperty("summaryEmbedding2") List cosineDistance, + @JsonProperty("summaryEmbedding3") List dotProduct, + @JsonProperty("rating") double rating) { + this.id = id; + this.name = name; + this.code = code; + this.description = description; + this.euclidean = euclidean; + this.cosineDistance = euclidean; + this.dotProduct = euclidean; + this.rating = rating; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public int getCode() { + return code; + } + + public String getDescription() { + return description; + } + + public List getEuclidean() { + return euclidean; + } + + public List getCosineDistance() { + return cosineDistance; + } + + public List getDotProduct() { + return dotProduct; + } + + public double getRating() { + return rating; + } + + public void setRating(double rating) { + this.rating = rating; + } +} diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index f6849d00..0d329513 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -2,13 +2,15 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; -import com.microsoft.semantickernel.tests.connectors.memory.Hotel; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import com.redis.testcontainers.RedisContainer; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.MethodOrderer; @@ -16,7 +18,9 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; import redis.clients.jedis.JedisPooled; @@ -27,6 +31,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -86,12 +91,6 @@ static void setup() { .withFieldType(List.class) .withDimensions(8) .build()); - fields.add(VectorStoreRecordVectorField.builder() - .withName("indexedEuclidean") - .withStorageName("indexedSummaryEmbedding") - .withFieldType(List.class) - .withDimensions(8) - .build()); fields.add(VectorStoreRecordDataField.builder() .withName("rating") .withFieldType(Double.class) @@ -104,7 +103,7 @@ static void setup() { .build()); } - private RedisHashSetVectorStoreRecordCollection buildrecordCollection(@Nonnull RedisHashSetVectorStoreRecordCollectionOptions options, @Nonnull String collectionName) { + private RedisHashSetVectorStoreRecordCollection buildRecordCollection(@Nonnull RedisHashSetVectorStoreRecordCollectionOptions options, @Nonnull String collectionName) { return new RedisHashSetVectorStoreRecordCollection<>(new JedisPooled(redisContainer.getRedisURI()), collectionName, RedisHashSetVectorStoreRecordCollectionOptions.builder() .withRecordClass(options.getRecordClass()) .withVectorStoreRecordMapper(options.getVectorStoreRecordMapper()) @@ -115,26 +114,35 @@ private RedisHashSetVectorStoreRecordCollection buildrecordCollection(@No private List getHotels() { return List.of( - new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(1.0f, 2.0f, 3.0f), null, null, null, 4.0), - new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(1.0f, 2.0f, 3.0f), null, null, null, 3.0), - new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(1.0f, 2.0f, 3.0f), null, null, null, 2.0), - new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(1.0f, 2.0f, 3.0f), null, null, null, 4.0), - new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(1.0f, 2.0f, 3.0f), null, null, null, 5.0) + new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f),null, null, 4.0), + new Hotel("id_2", "Hotel2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f),null, null, 4.0), + new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f),null, null, 5.0), + new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f),null, null, 4.0), + new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f),null, null, 4.0) ); } + /** + * Search embeddings similar to the third hotel embeddings. + * In order of similarity: + * 1. Hotel 3 + * 2. Hotel 1 + * 3. Hotel 4 + */ + private static final List SEARCH_EMBEDDINGS = Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.2f); + @Order(1) @ParameterizedTest @EnumSource(RecordCollectionOptions.class) - public void buildrecordCollection(RecordCollectionOptions options) { - assertNotNull(buildrecordCollection(optionsMap.get(options), options.name())); + public void buildRecordCollection(RecordCollectionOptions options) { + assertNotNull(buildRecordCollection(optionsMap.get(options), options.name())); } @Order(2) @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void createCollectionAsync(RecordCollectionOptions options) { - RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); assertEquals(false, recordCollection.collectionExistsAsync().block()); recordCollection.createCollectionAsync().block(); @@ -143,7 +151,7 @@ public void createCollectionAsync(RecordCollectionOptions options) { @Test public void deleteCollectionAsync() { - RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(RecordCollectionOptions.DEFAULT), "deleteCollectionAsync"); + RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(RecordCollectionOptions.DEFAULT), "deleteCollectionAsync"); assertEquals(false, recordCollection.collectionExistsAsync().block()); recordCollection.createCollectionAsync().block(); @@ -154,7 +162,7 @@ public void deleteCollectionAsync() { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void upsertAndGetRecordAsync(RecordCollectionOptions options) { - RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); for (Hotel hotel : hotels) { @@ -171,7 +179,7 @@ public void upsertAndGetRecordAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getBatchAsync(RecordCollectionOptions options) { - RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); for (Hotel hotel : hotels) { @@ -193,7 +201,7 @@ public void getBatchAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void upsertBatchAsync(RecordCollectionOptions options) { - RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); List keys = recordCollection.upsertBatchAsync(hotels, null).block(); @@ -211,7 +219,7 @@ public void upsertBatchAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void deleteAsync(RecordCollectionOptions options) { - RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -226,7 +234,7 @@ public void deleteAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void deleteBatchAsync(RecordCollectionOptions options) { - RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -245,7 +253,7 @@ public void deleteBatchAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getAsyncWithVectors(RecordCollectionOptions options) { - RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -262,7 +270,7 @@ public void getAsyncWithVectors(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getBatchAsyncWithVectors(RecordCollectionOptions options) { - RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -284,7 +292,7 @@ public void getBatchAsyncWithVectors(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getAsyncWithNoVectors(RecordCollectionOptions options) { - RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -302,7 +310,7 @@ public void getAsyncWithNoVectors(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getBatchAsyncWithNoVectors(RecordCollectionOptions options) { - RedisHashSetVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -321,4 +329,50 @@ public void getBatchAsyncWithNoVectors(RecordCollectionOptions options) { assertNull(retrievedHotels.get(i).getEuclidean()); } } + + private static Stream provideSearchParameters() { + return Stream.of( + Arguments.of(RecordCollectionOptions.DEFAULT, "euclidean"), + Arguments.of(RecordCollectionOptions.DEFAULT, "cosineDistance"), + Arguments.of(RecordCollectionOptions.DEFAULT, "dotProduct"), + Arguments.of(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, "euclidean"), + Arguments.of(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, "cosineDistance"), + Arguments.of(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, "dotProduct") + ); + } + +// @ParameterizedTest +// @MethodSource("provideSearchParameters") +// public void search(RecordCollectionOptions options, String embeddingName) { +// String collectionName = "search" + embeddingName; +// RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); +// +// List hotels = getHotels(); +// recordCollection.upsertBatchAsync(hotels, null).block(); +// +// VectorSearchOptions searchOptions = VectorSearchOptions.builder() +// .withVectorFieldName(embeddingName) +// .withLimit(3) +// .build(); +// +// // Embeddings similar to the third hotel +// List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); +// assertNotNull(results); +// assertEquals(3, results.size()); +// // The third hotel should be the most similar +// assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); +// +// searchOptions = VectorSearchOptions.builder() +// .withVectorFieldName(embeddingName) +// .withOffset(1) +// .withLimit(-100) +// .build(); +// +// // Skip the first result +// results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); +// assertNotNull(results); +// assertEquals(1, results.size()); +// // The first hotel should be the most similar +// assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); +// } } diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index 04effca7..15a6f2f4 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -2,13 +2,14 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; -import com.microsoft.semantickernel.tests.connectors.memory.Hotel; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import com.redis.testcontainers.RedisContainer; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.MethodOrderer; @@ -16,7 +17,9 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; import redis.clients.jedis.JedisPooled; @@ -27,9 +30,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -87,12 +90,6 @@ static void setup() { .withFieldType(List.class) .withDimensions(8) .build()); - fields.add(VectorStoreRecordVectorField.builder() - .withName("indexedEuclidean") - .withStorageName("indexedSummaryEmbedding") - .withFieldType(List.class) - .withDimensions(8) - .build()); fields.add(VectorStoreRecordDataField.builder() .withName("rating") .withFieldType(Double.class) @@ -116,14 +113,23 @@ private RedisJsonVectorStoreRecordCollection buildrecordCollection(@Nonnu private List getHotels() { return List.of( - new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(1.0f, 2.0f, 3.0f), null, null, null, 4.0), - new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(1.0f, 2.0f, 3.0f), null, null, null, 3.0), - new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(1.0f, 2.0f, 3.0f), null, null, null, 2.0), - new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(1.0f, 2.0f, 3.0f), null, null, null, 4.0), - new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(1.0f, 2.0f, 3.0f), null, null, null, 5.0) + new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f),null, null, 4.0), + new Hotel("id_2", "Hotel2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f),null, null, 4.0), + new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f),null, null, 5.0), + new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f),null, null, 4.0), + new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f),null, null, 4.0) ); } + /** + * Search embeddings similar to the third hotel embeddings. + * In order of similarity: + * 1. Hotel 3 + * 2. Hotel 1 + * 3. Hotel 4 + */ + private static final List SEARCH_EMBEDDINGS = Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.2f); + @Order(1) @ParameterizedTest @EnumSource(RecordCollectionOptions.class) @@ -322,4 +328,50 @@ public void getBatchAsyncWithNoVectors(RecordCollectionOptions options) { assertNull(retrievedHotels.get(i).getEuclidean()); } } + + private static Stream provideSearchParameters() { + return Stream.of( + Arguments.of(RecordCollectionOptions.DEFAULT, "euclidean"), + Arguments.of(RecordCollectionOptions.DEFAULT, "cosineDistance"), + Arguments.of(RecordCollectionOptions.DEFAULT, "dotProduct"), + Arguments.of(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, "euclidean"), + Arguments.of(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, "cosineDistance"), + Arguments.of(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, "dotProduct") + ); + } + + @ParameterizedTest + @MethodSource("provideSearchParameters") + public void search(RecordCollectionOptions options, String embeddingName) { + String collectionName = "search" + embeddingName; + RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), collectionName); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions searchOptions = VectorSearchOptions.builder() + .withVectorFieldName(embeddingName) + .withLimit(3) + .build(); + + // Embeddings similar to the third hotel + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); + assertNotNull(results); + assertEquals(3, results.size()); + // The third hotel should be the most similar + assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + + searchOptions = VectorSearchOptions.builder() + .withVectorFieldName(embeddingName) + .withOffset(1) + .withLimit(-100) + .build(); + + // Skip the first result + results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); + assertNotNull(results); + assertEquals(1, results.size()); + // The first hotel should be the most similar + assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); + } } diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java index fd63b9e5..ea48a155 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java @@ -6,7 +6,7 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStore; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreOptions; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.tests.connectors.memory.Hotel; +import com.microsoft.semantickernel.tests.connectors.memory.jdbc.Hotel; import com.redis.testcontainers.RedisContainer; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java index bb9ef070..4b69537d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java @@ -1,7 +1,13 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.VectorizedSearch; +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizedSearchQuery; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; @@ -9,6 +15,8 @@ import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; @@ -18,7 +26,9 @@ import redis.clients.jedis.exceptions.JedisDataException; import redis.clients.jedis.search.IndexDefinition; import redis.clients.jedis.search.IndexOptions; +import redis.clients.jedis.search.Query; import redis.clients.jedis.search.Schema; +import redis.clients.jedis.search.SearchResult; import javax.annotation.Nonnull; import java.util.AbstractMap; @@ -33,7 +43,8 @@ import java.util.stream.Collectors; public class RedisHashSetVectorStoreRecordCollection - implements VectorStoreRecordCollection { + implements VectorStoreRecordCollection, + VectorizedSearch { private static final HashSet> supportedKeyTypes = new HashSet<>( Collections.singletonList( @@ -50,7 +61,7 @@ public class RedisHashSetVectorStoreRecordCollection private final VectorStoreRecordMapper>> vectorStoreRecordMapper; private final VectorStoreRecordDefinition recordDefinition; private final String[] dataFields; - private final ObjectMapper objectMapper = new ObjectMapper(); + private final ObjectMapper objectMapper; /** * Creates a new instance of the RedisVectorRecordStore. @@ -84,11 +95,16 @@ public RedisHashSetVectorStoreRecordCollection( new ArrayList<>(recordDefinition.getVectorFields()), supportedVectorTypes); + // If mapper is not provided, set a default one + objectMapper = options.getObjectMapper() != null ? options.getObjectMapper() + : new ObjectMapper(); + // If mapper is not provided, set a default one if (options.getVectorStoreRecordMapper() == null) { vectorStoreRecordMapper = new RedisHashSetVectorStoreRecordMapper.Builder() .withRecordClass(options.getRecordClass()) .withVectorStoreRecordDefinition(recordDefinition) + .withObjectMapper(objectMapper) .build(); } else { vectorStoreRecordMapper = options.getVectorStoreRecordMapper(); @@ -182,10 +198,17 @@ public Mono deleteCollectionAsync() { .then(); } - private String getRedisKey(String key, String collectionName) { + private String prefixKeyIfNeeded(String key, String collectionName) { return options.isPrefixCollectionName() ? collectionName + ":" + key : key; } + private String removeKeyPrefixIfNeeded(String key, String collectionName) { + if (options.isPrefixCollectionName() && key.startsWith(collectionName + ":")) { + return key.substring(collectionName.length() + 1); + } + return key; + } + private Map addDataFieldNames(List result) { Map dataFields = new HashMap<>(); for (int i = 0; i < result.size(); i++) { @@ -229,7 +252,7 @@ public Mono> getBatchAsync(List keys, Pipeline pipeline = client.pipelined(); List>> responses = new ArrayList<>(keys.size()); keys.forEach(key -> { - String redisKey = getRedisKey(key, collectionName); + String redisKey = prefixKeyIfNeeded(key, collectionName); if (options != null && options.isIncludeVectors()) { // Returns Map with the fields and values @@ -281,7 +304,7 @@ public Mono> getBatchAsync(List keys, public Mono upsertAsync(Record data, UpsertRecordOptions options) { Map.Entry> redisObject = this.vectorStoreRecordMapper .mapRecordToStorageModel(data); - String redisKey = getRedisKey(redisObject.getKey(), collectionName); + String redisKey = prefixKeyIfNeeded(redisObject.getKey(), collectionName); return Mono.fromRunnable(() -> client.hset(redisKey, redisObject.getValue())) .subscribeOn(Schedulers.boundedElastic()) @@ -303,7 +326,7 @@ public Mono> upsertBatchAsync(List data, UpsertRecordOption data.forEach(record -> { Map.Entry> redisObject = this.vectorStoreRecordMapper .mapRecordToStorageModel(record); - String redisKey = getRedisKey(redisObject.getKey(), collectionName); + String redisKey = prefixKeyIfNeeded(redisObject.getKey(), collectionName); keys.add(redisObject.getKey()); pipeline.hset(redisKey, redisObject.getValue()); @@ -323,7 +346,7 @@ public Mono> upsertBatchAsync(List data, UpsertRecordOption */ @Override public Mono deleteAsync(String key, DeleteRecordOptions options) { - String redisKey = getRedisKey(key, collectionName); + String redisKey = prefixKeyIfNeeded(key, collectionName); return Mono.fromRunnable(() -> client.del(redisKey)) .subscribeOn(Schedulers.boundedElastic()) @@ -341,7 +364,7 @@ public Mono deleteAsync(String key, DeleteRecordOptions options) { public Mono deleteBatchAsync(List strings, DeleteRecordOptions options) { Pipeline pipeline = client.pipelined(); strings.forEach(key -> { - String redisKey = getRedisKey(key, collectionName); + String redisKey = prefixKeyIfNeeded(key, collectionName); pipeline.del(redisKey); }); @@ -349,4 +372,67 @@ public Mono deleteBatchAsync(List strings, DeleteRecordOptions opt .subscribeOn(Schedulers.boundedElastic()) .then(); } + + @Override + public Mono>> searchAsync(VectorSearchQuery query) { + if (recordDefinition.getVectorFields().isEmpty()) { + return Mono + .error(new SKException("No vector fields defined. Cannot perform vector search")); + } + + return createCollectionIfNotExistsAsync().flatMap(collection -> + Mono.fromCallable(() -> { + if (query instanceof VectorizedSearchQuery) { + VectorSearchOptions options = query.getSearchOptions(); + + Query redisQuery = RedisVectorStoreCollectionSearchMapping + .buildQuery((VectorizedSearchQuery) query, recordDefinition); + SearchResult searchResult = client.ftSearch(collectionName, redisQuery); + + return searchResult.getDocuments().stream() + .map(doc -> { + Map properties = new HashMap<>(); + for (Map.Entry entry : doc.getProperties()) { + properties.put(entry.getKey(), entry.getValue()); + } + + String key = removeKeyPrefixIfNeeded(doc.getId(), collectionName); + String value = (String) properties.get("$"); + double score = Double.parseDouble((String) properties + .get(RedisVectorStoreCollectionSearchMapping.VECTOR_SCORE_FIELD)); + + try { + Map valueMap = objectMapper.readValue(value, Map.class); + + Record record = this.vectorStoreRecordMapper + .mapStorageModelToRecord( + new AbstractMap.SimpleEntry<>(key, valueMap), + new GetRecordOptions( + options != null && options.isIncludeVectors())); + + return new VectorSearchResult<>(record, score); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); + } + + throw new SKException("Unsupported query type"); + }).subscribeOn(Schedulers.boundedElastic()) + ); + } + + /** + * Vectorized search. This method searches for records that are similar to the given vector. + * + * @param vector The vector to search with. + * @param options The options to use for the search. + * @return A list of search results. + */ + @Override + public Mono>> searchAsync(List vector, + VectorSearchOptions options) { + return this.searchAsync(VectorSearchQuery.createQuery(vector, options)); + } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java index ab062708..cad713f9 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java @@ -1,10 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; +import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -19,16 +21,20 @@ public class RedisHashSetVectorStoreRecordCollectionOptions @Nullable private final VectorStoreRecordDefinition recordDefinition; private final boolean prefixCollectionName; + @Nullable + private final ObjectMapper objectMapper; private RedisHashSetVectorStoreRecordCollectionOptions( @Nonnull Class recordClass, @Nullable VectorStoreRecordMapper>> vectorStoreRecordMapper, @Nullable VectorStoreRecordDefinition recordDefinition, - boolean prefixCollectionName) { + boolean prefixCollectionName, + @Nullable ObjectMapper objectMapper) { this.recordClass = recordClass; this.vectorStoreRecordMapper = vectorStoreRecordMapper; this.recordDefinition = recordDefinition; this.prefixCollectionName = prefixCollectionName; + this.objectMapper = objectMapper; } /** @@ -89,6 +95,16 @@ public boolean isPrefixCollectionName() { return prefixCollectionName; } + /** + * Gets the object mapper. + * + * @return the object mapper + */ + @Nullable + ObjectMapper getObjectMapper() { + return objectMapper; + } + /** * Builder for {@link RedisHashSetVectorStoreRecordCollectionOptions}. * @@ -102,6 +118,8 @@ public static class Builder { @Nullable private VectorStoreRecordDefinition recordDefinition; private boolean prefixCollectionName = true; + @Nullable + private ObjectMapper objectMapper = new ObjectMapper(); /** * Sets the record class. @@ -149,6 +167,18 @@ public Builder withPrefixCollectionName(boolean prefixCollectionName) { return this; } + /** + * Sets the object mapper. + * + * @param objectMapper the object mapper + * @return the builder + */ + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withObjectMapper(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + return this; + } + /** * Builds the options. * @@ -163,7 +193,8 @@ public RedisHashSetVectorStoreRecordCollectionOptions build() { recordClass, vectorStoreRecordMapper, recordDefinition, - prefixCollectionName); + prefixCollectionName, + objectMapper); } } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java index adf5e97e..596893a3 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java @@ -12,6 +12,7 @@ import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.annotation.Nullable; import java.util.AbstractMap; @@ -50,9 +51,10 @@ public static class Builder implements SemanticKernelBuilder> { @Nullable private Class recordClass; - @Nullable private VectorStoreRecordDefinition recordDefinition; + @Nullable + private ObjectMapper objectMapper = new ObjectMapper(); /** * Sets the record class. @@ -77,6 +79,18 @@ public Builder withVectorStoreRecordDefinition( return this; } + /** + * Sets the object mapper. + * + * @param objectMapper the object mapper + * @return the builder + */ + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withObjectMapper(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + return this; + } + /** * Builds the {@link RedisHashSetVectorStoreRecordMapper}. * @@ -91,11 +105,9 @@ public RedisHashSetVectorStoreRecordMapper build() { throw new SKException("vectorStoreRecordDefinition is required"); } - ObjectMapper mapper = new ObjectMapper(); - return new RedisHashSetVectorStoreRecordMapper<>(record -> { try { - ObjectNode jsonNode = mapper.valueToTree(record); + ObjectNode jsonNode = objectMapper.valueToTree(record); String key = jsonNode .get(recordDefinition.getKeyField().getEffectiveStorageName()).asText(); jsonNode.remove(recordDefinition.getKeyField().getEffectiveStorageName()); @@ -108,7 +120,7 @@ public RedisHashSetVectorStoreRecordMapper build() { resultMap.put(field.getKey(), field.getValue().asText()); } else { resultMap.put(field.getKey(), - mapper.valueToTree(field.getValue()).toString()); + objectMapper.valueToTree(field.getValue()).toString()); } } @@ -125,9 +137,9 @@ public RedisHashSetVectorStoreRecordMapper build() { return null; } - ObjectNode jsonNode = mapper.createObjectNode(); + ObjectNode jsonNode = objectMapper.createObjectNode(); jsonNode.set(recordDefinition.getKeyField().getEffectiveStorageName(), - mapper.valueToTree(storageModel.getKey())); + objectMapper.valueToTree(storageModel.getKey())); for (VectorStoreRecordDataField field : recordDefinition.getDataFields()) { jsonNode.put(field.getEffectiveStorageName(), @@ -140,6 +152,11 @@ public RedisHashSetVectorStoreRecordMapper build() { String value = storageModel.getValue() .get(field.getEffectiveStorageName()); + // No vector found + if (value == null) { + continue; + } + Class valueType = field.getFieldType(); if (valueType.equals(String.class)) { @@ -147,12 +164,13 @@ public RedisHashSetVectorStoreRecordMapper build() { } else { // Convert the String stored in Redis back to the correct type and then put the JSON node jsonNode.set(field.getEffectiveStorageName(), - mapper.valueToTree(mapper.readValue(value, valueType))); + objectMapper + .valueToTree(objectMapper.readValue(value, valueType))); } } } - return mapper.convertValue(jsonNode, recordClass); + return objectMapper.convertValue(jsonNode, recordClass); } catch (JsonProcessingException e) { throw new SKException( "Failure to deserialize object, by default the Redis connector uses Jackson, ensure your model object can be serialized by Jackson, i.e the class is visible, has getters, constructor, annotations etc.", diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java index dc468b1c..301e7f3c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java @@ -1,9 +1,14 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.VectorizedSearch; +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizedSearchQuery; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; @@ -11,12 +16,15 @@ import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -34,10 +42,13 @@ import redis.clients.jedis.json.Path2; import redis.clients.jedis.search.IndexDefinition; import redis.clients.jedis.search.IndexOptions; +import redis.clients.jedis.search.Query; import redis.clients.jedis.search.Schema; +import redis.clients.jedis.search.SearchResult; public class RedisJsonVectorStoreRecordCollection - implements VectorStoreRecordCollection { + implements VectorStoreRecordCollection, + VectorizedSearch { private static final HashSet> supportedKeyTypes = new HashSet<>( Collections.singletonList( @@ -54,7 +65,7 @@ public class RedisJsonVectorStoreRecordCollection private final VectorStoreRecordMapper> vectorStoreRecordMapper; private final VectorStoreRecordDefinition recordDefinition; private final Path2[] dataFields; - private final ObjectMapper objectMapper = new ObjectMapper(); + private final ObjectMapper objectMapper; /** * Creates a new instance of the RedisVectorRecordStore. @@ -87,11 +98,16 @@ public RedisJsonVectorStoreRecordCollection( new ArrayList<>(recordDefinition.getVectorFields()), supportedVectorTypes); + // If object mapper is not provided, set a default one + this.objectMapper = options.getObjectMapper() != null ? options.getObjectMapper() + : new ObjectMapper(); + // If mapper is not provided, set a default one if (options.getVectorStoreRecordMapper() == null) { vectorStoreRecordMapper = new RedisJsonVectorStoreRecordMapper.Builder() .withKeyFieldName(recordDefinition.getKeyField().getEffectiveStorageName()) .withRecordClass(options.getRecordClass()) + .withObjectMapper(objectMapper) .build(); } else { vectorStoreRecordMapper = options.getVectorStoreRecordMapper(); @@ -181,10 +197,17 @@ public Mono deleteCollectionAsync() { .then(); } - private String getRedisKey(String key, String collectionName) { + private String prefixKeyIfNeeded(String key, String collectionName) { return options.isPrefixCollectionName() ? collectionName + ":" + key : key; } + private String removeKeyPrefixIfNeeded(String key, String collectionName) { + if (options.isPrefixCollectionName() && key.startsWith(collectionName + ":")) { + return key.substring(collectionName.length() + 1); + } + return key; + } + private JsonNode removeRedisPathPrefix(JSONObject object) { ObjectNode noPathPrefix = objectMapper.createObjectNode(); object.keySet().forEach(key -> { @@ -209,7 +232,7 @@ private JsonNode removeRedisPathPrefix(JSONObject object) { */ @Override public Mono getAsync(String key, GetRecordOptions options) { - String redisKey = getRedisKey(key, collectionName); + String redisKey = prefixKeyIfNeeded(key, collectionName); return Mono.defer(() -> { try { @@ -253,7 +276,7 @@ public Mono> getBatchAsync(List keys, Pipeline pipeline = client.pipelined(); List>> responses = new ArrayList<>(keys.size()); keys.forEach(key -> { - String redisKey = getRedisKey(key, collectionName); + String redisKey = prefixKeyIfNeeded(key, collectionName); if (options != null && options.isIncludeVectors()) { responses.add(new SimpleEntry<>(key, pipeline.jsonGet(redisKey))); @@ -301,7 +324,7 @@ public Mono> getBatchAsync(List keys, public Mono upsertAsync(Record data, UpsertRecordOptions options) { Entry redisObject = this.vectorStoreRecordMapper .mapRecordToStorageModel(data); - String redisKey = getRedisKey(redisObject.getKey(), collectionName); + String redisKey = prefixKeyIfNeeded(redisObject.getKey(), collectionName); return Mono.fromRunnable(() -> client.jsonSet(redisKey, redisObject.getValue())) .subscribeOn(Schedulers.boundedElastic()) @@ -323,7 +346,7 @@ public Mono> upsertBatchAsync(List data, UpsertRecordOption data.forEach(record -> { Entry redisObject = this.vectorStoreRecordMapper .mapRecordToStorageModel(record); - String redisKey = getRedisKey(redisObject.getKey(), collectionName); + String redisKey = prefixKeyIfNeeded(redisObject.getKey(), collectionName); keys.add(redisObject.getKey()); pipeline.jsonSet(redisKey, redisObject.getValue()); @@ -343,7 +366,7 @@ public Mono> upsertBatchAsync(List data, UpsertRecordOption */ @Override public Mono deleteAsync(String key, DeleteRecordOptions options) { - String redisKey = getRedisKey(key, collectionName); + String redisKey = prefixKeyIfNeeded(key, collectionName); return Mono.fromRunnable(() -> client.del(redisKey)) .subscribeOn(Schedulers.boundedElastic()) @@ -361,7 +384,7 @@ public Mono deleteAsync(String key, DeleteRecordOptions options) { public Mono deleteBatchAsync(List strings, DeleteRecordOptions options) { Pipeline pipeline = client.pipelined(); strings.forEach(key -> { - String redisKey = getRedisKey(key, collectionName); + String redisKey = prefixKeyIfNeeded(key, collectionName); pipeline.del(redisKey); }); @@ -369,4 +392,69 @@ public Mono deleteBatchAsync(List strings, DeleteRecordOptions opt .subscribeOn(Schedulers.boundedElastic()) .then(); } + + /** + * Vector search. This method searches for records that are similar to the given vector using the index defined when creating the collection. + * + * @param query The search query. + * @return A list of search results. + */ + @Override + public Mono>> searchAsync(VectorSearchQuery query) { + if (recordDefinition.getVectorFields().isEmpty()) { + return Mono + .error(new SKException("No vector fields defined. Cannot perform vector search")); + } + + return createCollectionIfNotExistsAsync().flatMap(collection -> Mono.fromCallable(() -> { + if (query instanceof VectorizedSearchQuery) { + VectorSearchOptions options = query.getSearchOptions(); + + Query redisQuery = RedisVectorStoreCollectionSearchMapping + .buildQuery((VectorizedSearchQuery) query, recordDefinition); + SearchResult searchResult = client.ftSearch(collectionName, redisQuery); + + return searchResult.getDocuments().stream() + .map(doc -> { + Map properties = new HashMap<>(); + for (Map.Entry entry : doc.getProperties()) { + properties.put(entry.getKey(), entry.getValue()); + } + + String key = removeKeyPrefixIfNeeded(doc.getId(), collectionName); + String value = (String) properties.get("$"); + double score = Double.parseDouble((String) properties + .get(RedisVectorStoreCollectionSearchMapping.VECTOR_SCORE_FIELD)); + + try { + JsonNode jsonNode = objectMapper.readTree(value); + Record record = this.vectorStoreRecordMapper + .mapStorageModelToRecord(new SimpleEntry<>(key, jsonNode), + new GetRecordOptions( + options != null && options.isIncludeVectors())); + + return new VectorSearchResult<>(record, score); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); + } + + throw new SKException("Unsupported query type"); + }).subscribeOn(Schedulers.boundedElastic())); + } + + /** + * Vectorized search. This method searches for records that are similar to the given vector. + * + * @param vector The vector to search with. + * @param options The options to use for the search. + * @return A list of search results. + */ + @Override + public Mono>> searchAsync(List vector, + VectorSearchOptions options) { + return this.searchAsync(VectorSearchQuery.createQuery(vector, options)); + } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java index 6c35fa82..9702e185 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java @@ -1,10 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; +import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -18,16 +20,19 @@ public class RedisJsonVectorStoreRecordCollectionOptions @Nullable private final VectorStoreRecordDefinition recordDefinition; private final boolean prefixCollectionName; + private final ObjectMapper objectMapper; private RedisJsonVectorStoreRecordCollectionOptions( @Nonnull Class recordClass, @Nullable VectorStoreRecordMapper> vectorStoreRecordMapper, @Nullable VectorStoreRecordDefinition recordDefinition, - boolean prefixCollectionName) { + boolean prefixCollectionName, + @Nullable ObjectMapper objectMapper) { this.recordClass = recordClass; this.vectorStoreRecordMapper = vectorStoreRecordMapper; this.recordDefinition = recordDefinition; this.prefixCollectionName = prefixCollectionName; + this.objectMapper = objectMapper; } /** @@ -88,6 +93,15 @@ public boolean isPrefixCollectionName() { return prefixCollectionName; } + /** + * Gets the object mapper. + * + * @return the object mapper + */ + ObjectMapper getObjectMapper() { + return objectMapper; + } + /** * Builder for {@link RedisJsonVectorStoreRecordCollectionOptions}. * @@ -101,6 +115,8 @@ public static class Builder { @Nullable private VectorStoreRecordDefinition recordDefinition; private boolean prefixCollectionName = true; + @Nullable + private ObjectMapper objectMapper = new ObjectMapper(); /** * Sets the record class. @@ -137,10 +153,11 @@ public Builder withRecordDefinition(VectorStoreRecordDefinition recordDe } /** - * Sets whether to prefix the collection name to the redis key. - * Default is true. - * - * @param prefixCollectionName whether to prefix the collection name to the redis key + * Sets whether the collection name should be prefixed to the key names before reading or writing to the Redis store. Default is true. + *

+ * For a record to be indexed by a specific Redis index, the key name must be prefixed with the matching prefix configured on the Redis index. + * You can either pass in keys that are already prefixed, or set this option to true to have the collection name prefixed to the key names automatically. + * @param prefixCollectionName whether to prefix the collection name to the key * @return the builder */ public Builder withPrefixCollectionName(boolean prefixCollectionName) { @@ -148,6 +165,18 @@ public Builder withPrefixCollectionName(boolean prefixCollectionName) { return this; } + /** + * Sets the object mapper to use for serialization and deserialization. + * + * @param objectMapper the object mapper + * @return the builder + */ + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withObjectMapper(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + return this; + } + /** * Builds the options. * @@ -162,7 +191,8 @@ public RedisJsonVectorStoreRecordCollectionOptions build() { recordClass, vectorStoreRecordMapper, recordDefinition, - prefixCollectionName); + prefixCollectionName, + objectMapper); } } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java index f84a8e9a..087a1981 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java @@ -89,9 +89,11 @@ public static Schema mapToSchema(List fields) { } if (dataField.getFieldType().equals(String.class)) { - schema.addTextField(getRedisPath(dataField.getEffectiveStorageName()), 1.0); + schema.addTextField(getRedisPath(dataField.getEffectiveStorageName()), 1.0) + .as(dataField.getEffectiveStorageName()); } else if (supportedFilterableNumericTypes.contains(dataField.getFieldType())) { - schema.addNumericField(getRedisPath(dataField.getEffectiveStorageName())); + schema.addNumericField(getRedisPath(dataField.getEffectiveStorageName())) + .as(dataField.getEffectiveStorageName()); } else { throw new SKException( "Unsupported field type for numeric filterable fields: " @@ -118,7 +120,7 @@ public static Schema mapToSchema(List fields) { attributes.put(RedisIndexSchemaParams.DISTANCE_METRIC, metric); schema.addVectorField(getRedisPath(vectorField.getEffectiveStorageName()), - algorithm, attributes); + algorithm, attributes).as(vectorField.getEffectiveStorageName()); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java new file mode 100644 index 00000000..1dff3ed8 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.redis; + +import com.microsoft.semantickernel.connectors.data.redis.filter.RedisEqualToFilterClause; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizedSearchQuery; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import com.microsoft.semantickernel.exceptions.SKException; +import redis.clients.jedis.search.Query; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.List; +import java.util.stream.Collectors; + +public class RedisVectorStoreCollectionSearchMapping { + static final String VECTOR_SCORE_FIELD = "vector_score"; + + public static Query buildQuery(VectorizedSearchQuery query, + VectorStoreRecordDefinition recordDefinition) { + VectorSearchOptions options = query.getSearchOptions(); + + VectorStoreRecordVectorField firstVectorField = recordDefinition.getVectorFields() + .get(0); + if (options == null) { + options = VectorSearchOptions.createDefault(firstVectorField.getName()); + } + + VectorStoreRecordVectorField vectorField = options.getVectorFieldName() == null + ? firstVectorField + : (VectorStoreRecordVectorField) recordDefinition + .getField(options.getVectorFieldName()); + + String filter = buildFilter(options.getVectorSearchFilter(), recordDefinition); + + String knn = String.format("%s=>[KNN $K @%s $BLOB AS %s]", filter, + vectorField.getEffectiveStorageName(), VECTOR_SCORE_FIELD); + Query redisQuery = new Query(knn) + .addParam("K", options.getLimit() + options.getOffset()) + .addParam("BLOB", convertListToByteArray(query.getVector())) + .limit(options.getOffset(), options.getLimit()) + .setSortBy(VECTOR_SCORE_FIELD, true) + .dialect(2); + + if (options.isIncludeVectors()) { + redisQuery.returnFields(recordDefinition.getDataFields().stream() + .map(VectorStoreRecordDataField::getEffectiveStorageName) + .toArray(String[]::new)); + } + + return redisQuery; + } + + public static byte[] convertListToByteArray(List embeddings) { + ByteBuffer bytes = ByteBuffer.allocate(Float.BYTES * embeddings.size()); + bytes.order(ByteOrder.LITTLE_ENDIAN); + embeddings.iterator().forEachRemaining(bytes::putFloat); + return bytes.array(); + } + + public static String buildFilter(VectorSearchFilter vectorSearchFilter, + VectorStoreRecordDefinition recordDefinition) { + if (vectorSearchFilter == null + || vectorSearchFilter.getFilterClauses().isEmpty()) { + return "*"; + } + + return String.format("(%s)", + vectorSearchFilter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof RedisEqualToFilterClause) { + RedisEqualToFilterClause equalToFilterClause = (RedisEqualToFilterClause) filterClause; + return new RedisEqualToFilterClause( + recordDefinition.getField(equalToFilterClause.getFieldName()) + .getEffectiveStorageName(), + equalToFilterClause.getValue()).getFilter(); + } else { + throw new SKException("Unsupported filter clause type '" + + filterClause.getClass().getSimpleName() + "'."); + } + }).collect(Collectors.joining(" "))); + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/filter/RedisEqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/filter/RedisEqualToFilterClause.java new file mode 100644 index 00000000..b6ecf43f --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/filter/RedisEqualToFilterClause.java @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.redis.filter; + +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.filter.EqualToFilterClause; +import com.microsoft.semantickernel.exceptions.SKException; + +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; + +public class RedisEqualToFilterClause extends EqualToFilterClause { + public RedisEqualToFilterClause(String fieldName, Object value) { + super(fieldName, value); + } + + /** + * Gets the filter string. + * + * @return The filter. + */ + @Override + public String getFilter() { + String fieldName = getFieldName(); + Object value = getValue(); + String formattedValue; + + if (value instanceof String) { + formattedValue = String.format("\"%s\"", value); + } else if (value instanceof Boolean) { + formattedValue = value.toString().toLowerCase(); + } else if (value instanceof Integer || value instanceof Long || value instanceof Float + || value instanceof Double) { + formattedValue = value.toString(); + } else if (value instanceof OffsetDateTime) { + formattedValue = ((OffsetDateTime) value) + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); + } else { + throw new SKException("Unsupported filter value type '" + + value.getClass().getSimpleName() + "'."); + } + + return String.format("@%s:%s", fieldName, formattedValue); + } +} From d95c6328851007c5a606bc05a76fdb799b3d5af7 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:19:26 +0100 Subject: [PATCH 046/277] Remove broken badge --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index b748c348..2f00d959 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ [![Builds](https://github.com/microsoft/semantic-kernel-java/actions/workflows/java-build.yml/badge.svg?branch=main)](https://github.com/microsoft/semantic-kernel-java/actions/workflows/java-build.yml) -[![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.microsoft.semantic-kernel/semantickernel-api/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.microsoft.semantic-kernel/semantickernel-api) [![License: MIT](https://img.shields.io/github/license/microsoft/semantic-kernel)](https://github.com/microsoft/semantic-kernel-java/blob/main/LICENSE) [![Discord](https://img.shields.io/discord/1063152441819942922?label=Discord&logo=discord&logoColor=white&color=d82679)](https://aka.ms/SKDiscord) From 9e19b4da33d0023f99300504a892d1f3aa58a53e Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 19 Sep 2024 01:01:22 -0700 Subject: [PATCH 047/277] Add search for Redis Hash Set --- ...ashSetVectorStoreRecordCollectionTest.java | 103 ++++++++----- ...isJsonVectorStoreRecordCollectionTest.java | 48 ++++++- .../pom.xml | 2 +- semantickernel-bom/pom.xml | 2 +- ...disHashSetVectorStoreRecordCollection.java | 135 +++++++++--------- ...SetVectorStoreRecordCollectionOptions.java | 10 +- .../RedisHashSetVectorStoreRecordMapper.java | 77 +++++----- .../RedisJsonVectorStoreRecordCollection.java | 16 ++- .../RedisJsonVectorStoreRecordMapper.java | 40 ++++-- ...disVectorStoreCollectionCreateMapping.java | 17 ++- ...disVectorStoreCollectionSearchMapping.java | 45 ++++-- .../options/VectorSearchOptions.java | 1 + 12 files changed, 315 insertions(+), 181 deletions(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index 0d329513..18dc4a84 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -189,7 +189,7 @@ public void getBatchAsync(RecordCollectionOptions options) { List ids = new ArrayList<>(); hotels.forEach(hotel -> ids.add(hotel.getId())); - List retrievedHotels = recordCollection.getBatchAsync(ids, null).block(); + List retrievedHotels = recordCollection.getBatchAsync(ids, new GetRecordOptions(true)).block(); assertNotNull(retrievedHotels); assertEquals(hotels.size(), retrievedHotels.size()); @@ -341,38 +341,71 @@ private static Stream provideSearchParameters() { ); } -// @ParameterizedTest -// @MethodSource("provideSearchParameters") -// public void search(RecordCollectionOptions options, String embeddingName) { -// String collectionName = "search" + embeddingName; -// RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); -// -// List hotels = getHotels(); -// recordCollection.upsertBatchAsync(hotels, null).block(); -// -// VectorSearchOptions searchOptions = VectorSearchOptions.builder() -// .withVectorFieldName(embeddingName) -// .withLimit(3) -// .build(); -// -// // Embeddings similar to the third hotel -// List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); -// assertNotNull(results); -// assertEquals(3, results.size()); -// // The third hotel should be the most similar -// assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); -// -// searchOptions = VectorSearchOptions.builder() -// .withVectorFieldName(embeddingName) -// .withOffset(1) -// .withLimit(-100) -// .build(); -// -// // Skip the first result -// results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); -// assertNotNull(results); -// assertEquals(1, results.size()); -// // The first hotel should be the most similar -// assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); -// } + @ParameterizedTest + @MethodSource("provideSearchParameters") + public void search(RecordCollectionOptions options, String embeddingName) { + String collectionName = "search" + embeddingName; + RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions searchOptions = VectorSearchOptions.builder() + .withVectorFieldName(embeddingName) + .build(); + + // Embeddings similar to the third hotel + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); + assertNotNull(results); + assertEquals(VectorSearchOptions.DEFAULT_RESULT_LIMIT, results.size()); + // The third hotel should be the most similar + assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + assertNull(results.get(0).getRecord().getEuclidean()); + } + + @ParameterizedTest + @MethodSource("provideSearchParameters") + public void searchWithVectors(RecordCollectionOptions options, String embeddingName) { + String collectionName = "search" + embeddingName; + RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions searchOptions = VectorSearchOptions.builder() + .withVectorFieldName(embeddingName) + .withIncludeVectors(true) + .build(); + + // Embeddings similar to the third hotel + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); + assertNotNull(results); + assertEquals(VectorSearchOptions.DEFAULT_RESULT_LIMIT, results.size()); + // The third hotel should be the most similar + assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + assertNotNull(results.get(0).getRecord().getEuclidean()); + } + + @ParameterizedTest + @MethodSource("provideSearchParameters") + public void searchWithOffSet(RecordCollectionOptions options, String embeddingName) { + String collectionName = "search" + embeddingName; + RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions searchOptions = VectorSearchOptions.builder() + .withVectorFieldName(embeddingName) + .withOffset(1) + .withLimit(4) + .build(); + + // Embeddings similar to the third hotel + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); + assertNotNull(results); + assertEquals(4, results.size()); + // The first hotel should be the most similar + assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); + } } diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index 15a6f2f4..358c0238 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -351,27 +351,61 @@ public void search(RecordCollectionOptions options, String embeddingName) { VectorSearchOptions searchOptions = VectorSearchOptions.builder() .withVectorFieldName(embeddingName) - .withLimit(3) .build(); // Embeddings similar to the third hotel List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); assertNotNull(results); - assertEquals(3, results.size()); + assertEquals(VectorSearchOptions.DEFAULT_RESULT_LIMIT, results.size()); // The third hotel should be the most similar assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + assertNull(results.get(0).getRecord().getEuclidean()); + } + + @ParameterizedTest + @MethodSource("provideSearchParameters") + public void searchWithVectors(RecordCollectionOptions options, String embeddingName) { + String collectionName = "search" + embeddingName; + RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), collectionName); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions searchOptions = VectorSearchOptions.builder() + .withVectorFieldName(embeddingName) + .withIncludeVectors(true) + .build(); + + // Embeddings similar to the third hotel + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); + assertNotNull(results); + assertEquals(VectorSearchOptions.DEFAULT_RESULT_LIMIT, results.size()); + // The third hotel should be the most similar + assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + assertNotNull(results.get(0).getRecord().getEuclidean()); + } + + @ParameterizedTest + @MethodSource("provideSearchParameters") + public void searchWithOffSet(RecordCollectionOptions options, String embeddingName) { + String collectionName = "search" + embeddingName; + RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), collectionName); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); - searchOptions = VectorSearchOptions.builder() + VectorSearchOptions searchOptions = VectorSearchOptions.builder() .withVectorFieldName(embeddingName) .withOffset(1) - .withLimit(-100) + .withLimit(4) .build(); - // Skip the first result - results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); + // Embeddings similar to the third hotel + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); assertNotNull(results); - assertEquals(1, results.size()); + assertEquals(4, results.size()); // The first hotel should be the most similar assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); } + } diff --git a/connectors/semantickernel-connectors-memory-redis/pom.xml b/connectors/semantickernel-connectors-memory-redis/pom.xml index 51a340d0..463e076f 100644 --- a/connectors/semantickernel-connectors-memory-redis/pom.xml +++ b/connectors/semantickernel-connectors-memory-redis/pom.xml @@ -41,7 +41,7 @@ redis.clients jedis - 5.1.0 + 5.2.0-beta5 provided diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index dfb1eca2..ce747561 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -120,7 +120,7 @@ redis.clients jedis - 5.1.0 + 5.2.0-beta5 diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java index 4b69537d..0f7ae6cb 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java @@ -1,8 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorizedSearch; @@ -10,7 +8,6 @@ import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizedSearchQuery; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; -import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; @@ -18,19 +15,21 @@ import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.apache.commons.lang3.tuple.Pair; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; import redis.clients.jedis.JedisPooled; import redis.clients.jedis.Pipeline; import redis.clients.jedis.Response; import redis.clients.jedis.exceptions.JedisDataException; +import redis.clients.jedis.search.FTSearchParams; import redis.clients.jedis.search.IndexDefinition; import redis.clients.jedis.search.IndexOptions; -import redis.clients.jedis.search.Query; import redis.clients.jedis.search.Schema; import redis.clients.jedis.search.SearchResult; import javax.annotation.Nonnull; +import java.nio.charset.StandardCharsets; import java.util.AbstractMap; import java.util.ArrayList; import java.util.Arrays; @@ -58,9 +57,9 @@ public class RedisHashSetVectorStoreRecordCollection private final JedisPooled client; private final String collectionName; private final RedisHashSetVectorStoreRecordCollectionOptions options; - private final VectorStoreRecordMapper>> vectorStoreRecordMapper; + private final VectorStoreRecordMapper>> vectorStoreRecordMapper; private final VectorStoreRecordDefinition recordDefinition; - private final String[] dataFields; + private final byte[][] dataFields; private final ObjectMapper objectMapper; /** @@ -111,10 +110,11 @@ public RedisHashSetVectorStoreRecordCollection( } // Creates a list of paths to retrieve from Redis when no vectors are requested - // Paths are in the format of $.field - this.dataFields = recordDefinition.getDataFields().stream() - .map(VectorStoreRecordDataField::getEffectiveStorageName) - .toArray(String[]::new); + this.dataFields = new byte[recordDefinition.getDataFields().size()][]; + for (int i = 0; i < recordDefinition.getDataFields().size(); i++) { + this.dataFields[i] = stringToBytes( + recordDefinition.getDataFields().get(i).getEffectiveStorageName()); + } } /** @@ -156,7 +156,7 @@ public Mono collectionExistsAsync() { public Mono> createCollectionAsync() { return Mono.fromRunnable(() -> { Schema schema = RedisVectorStoreCollectionCreateMapping - .mapToSchema(recordDefinition.getAllFields()); + .mapToSchema(recordDefinition.getAllFields(), false); IndexDefinition indexDefinition = new IndexDefinition(IndexDefinition.Type.HASH) .setPrefixes(collectionName + ":"); @@ -209,8 +209,12 @@ private String removeKeyPrefixIfNeeded(String key, String collectionName) { return key; } - private Map addDataFieldNames(List result) { - Map dataFields = new HashMap<>(); + private byte[] stringToBytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + + private Map addDataFieldNames(List result) { + Map dataFields = new HashMap<>(); for (int i = 0; i < result.size(); i++) { if (result.get(i) == null) { continue; @@ -255,12 +259,15 @@ public Mono> getBatchAsync(List keys, String redisKey = prefixKeyIfNeeded(key, collectionName); if (options != null && options.isIncludeVectors()) { - // Returns Map with the fields and values - responses.add(new AbstractMap.SimpleEntry<>(key, pipeline.hgetAll(redisKey))); + // Returns Map with the fields and values + responses + .add(new AbstractMap.SimpleEntry<>(key, + pipeline.hgetAll(stringToBytes(redisKey)))); } else { // Returns List with the values of the fields responses - .add(new AbstractMap.SimpleEntry<>(key, pipeline.hmget(redisKey, dataFields))); + .add(new AbstractMap.SimpleEntry<>(key, + pipeline.hmget(stringToBytes(redisKey), dataFields))); } }); @@ -271,19 +278,19 @@ public Mono> getBatchAsync(List keys, return Mono.just(responses.stream() .map(entry -> { if (options != null && options.isIncludeVectors()) { - // Results directly in a Map + // Results directly in a Map return this.vectorStoreRecordMapper .mapStorageModelToRecord( new AbstractMap.SimpleEntry<>(entry.getKey(), - (Map) entry.getValue().get()), + (Map) entry.getValue().get()), options); } - // Results in a List with the values of the fields + // Results in a List with the values of the fields return this.vectorStoreRecordMapper .mapStorageModelToRecord( new AbstractMap.SimpleEntry<>(entry.getKey(), - addDataFieldNames((List) entry.getValue().get())), + addDataFieldNames((List) entry.getValue().get())), options); }) .collect(Collectors.toList())); @@ -302,11 +309,11 @@ public Mono> getBatchAsync(List keys, */ @Override public Mono upsertAsync(Record data, UpsertRecordOptions options) { - Map.Entry> redisObject = this.vectorStoreRecordMapper + Map.Entry> redisObject = this.vectorStoreRecordMapper .mapRecordToStorageModel(data); String redisKey = prefixKeyIfNeeded(redisObject.getKey(), collectionName); - return Mono.fromRunnable(() -> client.hset(redisKey, redisObject.getValue())) + return Mono.fromRunnable(() -> client.hset(stringToBytes(redisKey), redisObject.getValue())) .subscribeOn(Schedulers.boundedElastic()) .thenReturn(redisObject.getKey()); } @@ -324,12 +331,12 @@ public Mono> upsertBatchAsync(List data, UpsertRecordOption List keys = new ArrayList<>(data.size()); data.forEach(record -> { - Map.Entry> redisObject = this.vectorStoreRecordMapper + Map.Entry> redisObject = this.vectorStoreRecordMapper .mapRecordToStorageModel(record); String redisKey = prefixKeyIfNeeded(redisObject.getKey(), collectionName); keys.add(redisObject.getKey()); - pipeline.hset(redisKey, redisObject.getValue()); + pipeline.hset(stringToBytes(redisKey), redisObject.getValue()); }); return Mono.fromRunnable(pipeline::sync) @@ -380,47 +387,47 @@ public Mono>> searchAsync(VectorSearchQuery quer .error(new SKException("No vector fields defined. Cannot perform vector search")); } - return createCollectionIfNotExistsAsync().flatMap(collection -> - Mono.fromCallable(() -> { - if (query instanceof VectorizedSearchQuery) { - VectorSearchOptions options = query.getSearchOptions(); - - Query redisQuery = RedisVectorStoreCollectionSearchMapping - .buildQuery((VectorizedSearchQuery) query, recordDefinition); - SearchResult searchResult = client.ftSearch(collectionName, redisQuery); - - return searchResult.getDocuments().stream() - .map(doc -> { - Map properties = new HashMap<>(); - for (Map.Entry entry : doc.getProperties()) { - properties.put(entry.getKey(), entry.getValue()); - } - - String key = removeKeyPrefixIfNeeded(doc.getId(), collectionName); - String value = (String) properties.get("$"); - double score = Double.parseDouble((String) properties - .get(RedisVectorStoreCollectionSearchMapping.VECTOR_SCORE_FIELD)); - - try { - Map valueMap = objectMapper.readValue(value, Map.class); - - Record record = this.vectorStoreRecordMapper - .mapStorageModelToRecord( - new AbstractMap.SimpleEntry<>(key, valueMap), - new GetRecordOptions( - options != null && options.isIncludeVectors())); - - return new VectorSearchResult<>(record, score); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }) - .collect(Collectors.toList()); - } + return createCollectionIfNotExistsAsync().flatMap(collection -> Mono.fromCallable(() -> { + if (query instanceof VectorizedSearchQuery) { + VectorSearchOptions options = query.getSearchOptions(); + + Pair ftSearchParams = RedisVectorStoreCollectionSearchMapping + .buildQuery((VectorizedSearchQuery) query, recordDefinition, + RedisStorageType.HASH_SET); + + SearchResult searchResult = client.ftSearch(collectionName, + ftSearchParams.getLeft(), ftSearchParams.getRight()); + + return searchResult.getDocuments().stream() + .map(doc -> { + String key = removeKeyPrefixIfNeeded(doc.getId(), collectionName); + double score = 0; + + // Convert from Map to Map + Map storage = new HashMap<>(); + for (Map.Entry entry : doc.getProperties()) { + if (entry.getValue() instanceof byte[]) { + storage.put(stringToBytes(entry.getKey()), + (byte[]) entry.getValue()); + } else if (entry.getKey().equals( + RedisVectorStoreCollectionSearchMapping.VECTOR_SCORE_FIELD)) { + score = Double.parseDouble((String) entry.getValue()); + } + } + + Record record = this.vectorStoreRecordMapper + .mapStorageModelToRecord( + new AbstractMap.SimpleEntry<>(key, storage), + new GetRecordOptions( + options != null && options.isIncludeVectors())); + + return new VectorSearchResult<>(record, score); + }) + .collect(Collectors.toList()); + } - throw new SKException("Unsupported query type"); - }).subscribeOn(Schedulers.boundedElastic()) - ); + throw new SKException("Unsupported query type"); + }).subscribeOn(Schedulers.boundedElastic())); } /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java index cad713f9..52dc404b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java @@ -17,7 +17,7 @@ public class RedisHashSetVectorStoreRecordCollectionOptions implements VectorStoreRecordCollectionOptions { private final Class recordClass; @Nullable - private final VectorStoreRecordMapper>> vectorStoreRecordMapper; + private final VectorStoreRecordMapper>> vectorStoreRecordMapper; @Nullable private final VectorStoreRecordDefinition recordDefinition; private final boolean prefixCollectionName; @@ -26,7 +26,7 @@ public class RedisHashSetVectorStoreRecordCollectionOptions private RedisHashSetVectorStoreRecordCollectionOptions( @Nonnull Class recordClass, - @Nullable VectorStoreRecordMapper>> vectorStoreRecordMapper, + @Nullable VectorStoreRecordMapper>> vectorStoreRecordMapper, @Nullable VectorStoreRecordDefinition recordDefinition, boolean prefixCollectionName, @Nullable ObjectMapper objectMapper) { @@ -82,7 +82,7 @@ public VectorStoreRecordDefinition getRecordDefinition() { * @return the vector store record mapper */ @Nullable - public VectorStoreRecordMapper>> getVectorStoreRecordMapper() { + public VectorStoreRecordMapper>> getVectorStoreRecordMapper() { return vectorStoreRecordMapper; } @@ -112,7 +112,7 @@ ObjectMapper getObjectMapper() { */ public static class Builder { @Nullable - private VectorStoreRecordMapper>> vectorStoreRecordMapper; + private VectorStoreRecordMapper>> vectorStoreRecordMapper; @Nullable private Class recordClass; @Nullable @@ -139,7 +139,7 @@ public Builder withRecordClass(Class recordClass) { * @return the builder */ public Builder withVectorStoreRecordMapper( - VectorStoreRecordMapper>> vectorStoreRecordMapper) { + VectorStoreRecordMapper>> vectorStoreRecordMapper) { this.vectorStoreRecordMapper = vectorStoreRecordMapper; return this; } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java index 596893a3..01a5531f 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java @@ -4,31 +4,37 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.AbstractMap; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.function.BiFunction; import java.util.function.Function; public class RedisHashSetVectorStoreRecordMapper - extends VectorStoreRecordMapper>> { + extends VectorStoreRecordMapper>> { private RedisHashSetVectorStoreRecordMapper( - Function>> toStorageModelMapper, - BiFunction>, GetRecordOptions, Record> toRecordMapper) { + Function>> toStorageModelMapper, + BiFunction>, GetRecordOptions, Record> toRecordMapper) { super(toStorageModelMapper, toRecordMapper); } @@ -108,23 +114,32 @@ public RedisHashSetVectorStoreRecordMapper build() { return new RedisHashSetVectorStoreRecordMapper<>(record -> { try { ObjectNode jsonNode = objectMapper.valueToTree(record); + String key = jsonNode .get(recordDefinition.getKeyField().getEffectiveStorageName()).asText(); jsonNode.remove(recordDefinition.getKeyField().getEffectiveStorageName()); - Map resultMap = new HashMap<>(); - Iterator> fields = jsonNode.fields(); - while (fields.hasNext()) { - Map.Entry field = fields.next(); - if (field.getValue().isTextual()) { - resultMap.put(field.getKey(), field.getValue().asText()); - } else { - resultMap.put(field.getKey(), - objectMapper.valueToTree(field.getValue()).toString()); + Map storage = new HashMap<>(); + for (VectorStoreRecordDataField field : recordDefinition.getDataFields()) { + JsonNode value = jsonNode.get(field.getEffectiveStorageName()); + if (value != null) { + storage.put( + field.getEffectiveStorageName().getBytes(StandardCharsets.UTF_8), + objectMapper.writeValueAsBytes(value)); + } + } + for (VectorStoreRecordVectorField field : recordDefinition.getVectorFields()) { + ArrayNode value = (ArrayNode) jsonNode.get(field.getEffectiveStorageName()); + List vector = objectMapper.convertValue(value, List.class); + if (value != null) { + storage.put( + field.getEffectiveStorageName().getBytes(StandardCharsets.UTF_8), + RedisVectorStoreCollectionSearchMapping + .convertListToByteArray(vector)); } } - return new AbstractMap.SimpleEntry<>(key, resultMap); + return new AbstractMap.SimpleEntry<>(key, storage); } catch (Exception e) { throw new SKException( "Failure to serialize object, by default the Redis connector uses Jackson, ensure your model object can be serialized by Jackson, i.e the class is visible, has getters, constructor, annotations etc.", @@ -141,31 +156,27 @@ public RedisHashSetVectorStoreRecordMapper build() { jsonNode.set(recordDefinition.getKeyField().getEffectiveStorageName(), objectMapper.valueToTree(storageModel.getKey())); + // byte[] as key is not useful, convert to String + Map storage = new HashMap<>(); + storageModel.getValue() + .forEach((k, v) -> storage.put(new String(k, StandardCharsets.UTF_8), v)); + for (VectorStoreRecordDataField field : recordDefinition.getDataFields()) { - jsonNode.put(field.getEffectiveStorageName(), - storageModel.getValue().get(field.getEffectiveStorageName())); + byte[] value = storage.get(field.getEffectiveStorageName()); + if (value != null) { + jsonNode.set(field.getEffectiveStorageName(), + objectMapper.valueToTree( + objectMapper.readValue(value, field.getFieldType()))); + } } - if (options != null && options.isIncludeVectors()) { for (VectorStoreRecordVectorField field : recordDefinition .getVectorFields()) { - String value = storageModel.getValue() - .get(field.getEffectiveStorageName()); - - // No vector found - if (value == null) { - continue; - } - - Class valueType = field.getFieldType(); - - if (valueType.equals(String.class)) { - jsonNode.put(field.getEffectiveStorageName(), value); - } else { - // Convert the String stored in Redis back to the correct type and then put the JSON node + byte[] value = storage.get(field.getEffectiveStorageName()); + if (value != null) { jsonNode.set(field.getEffectiveStorageName(), - objectMapper - .valueToTree(objectMapper.readValue(value, valueType))); + objectMapper.valueToTree(RedisVectorStoreCollectionSearchMapping + .convertByteArrayToList(value))); } } } @@ -175,6 +186,8 @@ public RedisHashSetVectorStoreRecordMapper build() { throw new SKException( "Failure to deserialize object, by default the Redis connector uses Jackson, ensure your model object can be serialized by Jackson, i.e the class is visible, has getters, constructor, annotations etc.", e); + } catch (IOException e) { + throw new RuntimeException(e); } }); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java index 301e7f3c..739ddf1a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java @@ -31,6 +31,8 @@ import java.util.Map.Entry; import java.util.stream.Collectors; import javax.annotation.Nonnull; + +import org.apache.commons.lang3.tuple.Pair; import org.json.JSONArray; import org.json.JSONObject; import reactor.core.publisher.Mono; @@ -40,6 +42,7 @@ import redis.clients.jedis.Response; import redis.clients.jedis.exceptions.JedisDataException; import redis.clients.jedis.json.Path2; +import redis.clients.jedis.search.FTSearchParams; import redis.clients.jedis.search.IndexDefinition; import redis.clients.jedis.search.IndexOptions; import redis.clients.jedis.search.Query; @@ -105,8 +108,8 @@ public RedisJsonVectorStoreRecordCollection( // If mapper is not provided, set a default one if (options.getVectorStoreRecordMapper() == null) { vectorStoreRecordMapper = new RedisJsonVectorStoreRecordMapper.Builder() - .withKeyFieldName(recordDefinition.getKeyField().getEffectiveStorageName()) .withRecordClass(options.getRecordClass()) + .withRecordDefinition(recordDefinition) .withObjectMapper(objectMapper) .build(); } else { @@ -160,7 +163,7 @@ public Mono collectionExistsAsync() { public Mono> createCollectionAsync() { return Mono.fromRunnable(() -> { Schema schema = RedisVectorStoreCollectionCreateMapping - .mapToSchema(recordDefinition.getAllFields()); + .mapToSchema(recordDefinition.getAllFields(), true); IndexDefinition indexDefinition = new IndexDefinition(IndexDefinition.Type.JSON) .setPrefixes(collectionName + ":"); @@ -410,9 +413,12 @@ public Mono>> searchAsync(VectorSearchQuery quer if (query instanceof VectorizedSearchQuery) { VectorSearchOptions options = query.getSearchOptions(); - Query redisQuery = RedisVectorStoreCollectionSearchMapping - .buildQuery((VectorizedSearchQuery) query, recordDefinition); - SearchResult searchResult = client.ftSearch(collectionName, redisQuery); + Pair ftSearchParams = RedisVectorStoreCollectionSearchMapping + .buildQuery((VectorizedSearchQuery) query, recordDefinition, + RedisStorageType.JSON); + + SearchResult searchResult = client.ftSearch(collectionName, + ftSearchParams.getLeft(), ftSearchParams.getRight()); return searchResult.getDocuments().stream() .map(doc -> { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java index 201d8246..c71389cf 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java @@ -5,6 +5,8 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -42,30 +44,31 @@ public static Builder builder() { public static class Builder implements SemanticKernelBuilder> { @Nullable - private String keyFieldName; - @Nullable private Class recordClass; + @Nullable + private VectorStoreRecordDefinition recordDefinition; + private ObjectMapper objectMapper = new ObjectMapper(); /** - * Sets the key field name in the record. + * Sets the record class. * - * @param keyFieldName the key field + * @param recordClass the record class * @return the builder */ - public Builder withKeyFieldName(String keyFieldName) { - this.keyFieldName = keyFieldName; + public Builder withRecordClass(Class recordClass) { + this.recordClass = recordClass; return this; } /** - * Sets the record class. + * Sets the record definition. * - * @param recordClass the record class + * @param recordDefinition the record definition * @return the builder */ - public Builder withRecordClass(Class recordClass) { - this.recordClass = recordClass; + public Builder withRecordDefinition(VectorStoreRecordDefinition recordDefinition) { + this.recordDefinition = recordDefinition; return this; } @@ -88,15 +91,16 @@ public Builder withObjectMapper(ObjectMapper objectMapper) { */ @Override public RedisJsonVectorStoreRecordMapper build() { - if (keyFieldName == null) { - throw new SKException("keyFieldName is required"); - } if (recordClass == null) { throw new SKException("recordClass is required"); } + if (recordDefinition == null) { + throw new SKException("recordDefinition is required"); + } return new RedisJsonVectorStoreRecordMapper<>(record -> { try { + String keyFieldName = recordDefinition.getKeyField().getEffectiveStorageName(); ObjectNode jsonNode = objectMapper.valueToTree(record); String key = jsonNode.get(keyFieldName).asText(); jsonNode.remove(keyFieldName); @@ -109,9 +113,19 @@ public RedisJsonVectorStoreRecordMapper build() { } }, (storageModel, options) -> { try { + String keyFieldName = recordDefinition.getKeyField().getEffectiveStorageName(); ObjectNode jsonNode = objectMapper.valueToTree(storageModel.getValue()); // Add the key back to the record jsonNode.put(keyFieldName, storageModel.getKey()); + // Make sure to exclude the vectors if needed + if (options == null || !options.isIncludeVectors()) { + for (VectorStoreRecordVectorField vectorField : recordDefinition + .getVectorFields()) { + if (jsonNode.has(vectorField.getEffectiveStorageName())) { + jsonNode.remove(vectorField.getEffectiveStorageName()); + } + } + } return objectMapper.convertValue(jsonNode, recordClass); } catch (Exception e) { throw new SKException( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java index 087a1981..e0cf037d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java @@ -66,11 +66,12 @@ private static Schema.VectorField.VectorAlgo getAlgorithmConfig( } } - private static String getRedisPath(String name) { - return "$." + name; + private static String getRedisPath(String name, boolean withRedisJsonRoot) { + return withRedisJsonRoot ? "$." + name : name; } - public static Schema mapToSchema(List fields) { + public static Schema mapToSchema(List fields, + boolean withRedisJsonRoot) { Schema schema = new Schema(); for (VectorStoreRecordField field : fields) { @@ -89,10 +90,13 @@ public static Schema mapToSchema(List fields) { } if (dataField.getFieldType().equals(String.class)) { - schema.addTextField(getRedisPath(dataField.getEffectiveStorageName()), 1.0) + schema.addTextField( + getRedisPath(dataField.getEffectiveStorageName(), withRedisJsonRoot), 1.0) .as(dataField.getEffectiveStorageName()); } else if (supportedFilterableNumericTypes.contains(dataField.getFieldType())) { - schema.addNumericField(getRedisPath(dataField.getEffectiveStorageName())) + schema + .addNumericField( + getRedisPath(dataField.getEffectiveStorageName(), withRedisJsonRoot)) .as(dataField.getEffectiveStorageName()); } else { throw new SKException( @@ -119,7 +123,8 @@ public static Schema mapToSchema(List fields) { attributes.put(RedisIndexSchemaParams.DIMENSIONS, vectorField.getDimensions()); attributes.put(RedisIndexSchemaParams.DISTANCE_METRIC, metric); - schema.addVectorField(getRedisPath(vectorField.getEffectiveStorageName()), + schema.addVectorField( + getRedisPath(vectorField.getEffectiveStorageName(), withRedisJsonRoot), algorithm, attributes).as(vectorField.getEffectiveStorageName()); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java index 1dff3ed8..2fb88789 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java @@ -9,6 +9,9 @@ import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import com.microsoft.semantickernel.exceptions.SKException; +import org.apache.commons.lang3.tuple.Pair; +import redis.clients.jedis.args.SortingOrder; +import redis.clients.jedis.search.FTSearchParams; import redis.clients.jedis.search.Query; import java.nio.ByteBuffer; @@ -19,12 +22,11 @@ public class RedisVectorStoreCollectionSearchMapping { static final String VECTOR_SCORE_FIELD = "vector_score"; - public static Query buildQuery(VectorizedSearchQuery query, - VectorStoreRecordDefinition recordDefinition) { + public static Pair buildQuery(VectorizedSearchQuery query, + VectorStoreRecordDefinition recordDefinition, + RedisStorageType storageType) { VectorSearchOptions options = query.getSearchOptions(); - - VectorStoreRecordVectorField firstVectorField = recordDefinition.getVectorFields() - .get(0); + VectorStoreRecordVectorField firstVectorField = recordDefinition.getVectorFields().get(0); if (options == null) { options = VectorSearchOptions.createDefault(firstVectorField.getName()); } @@ -38,20 +40,30 @@ public static Query buildQuery(VectorizedSearchQuery query, String knn = String.format("%s=>[KNN $K @%s $BLOB AS %s]", filter, vectorField.getEffectiveStorageName(), VECTOR_SCORE_FIELD); - Query redisQuery = new Query(knn) + + FTSearchParams searchParams = new FTSearchParams() .addParam("K", options.getLimit() + options.getOffset()) .addParam("BLOB", convertListToByteArray(query.getVector())) .limit(options.getOffset(), options.getLimit()) - .setSortBy(VECTOR_SCORE_FIELD, true) + .sortBy(VECTOR_SCORE_FIELD, SortingOrder.ASC) .dialect(2); - if (options.isIncludeVectors()) { - redisQuery.returnFields(recordDefinition.getDataFields().stream() - .map(VectorStoreRecordDataField::getEffectiveStorageName) - .toArray(String[]::new)); + // For hash set storage is possible to select what fields to return without them being filterable + if (storageType == RedisStorageType.HASH_SET) { + // We also need to tell Redis to return the fields without decoding them + // Vector fields specially need to be returned as raw bytes + searchParams.returnField(VECTOR_SCORE_FIELD, false); + for (VectorStoreRecordDataField dataField : recordDefinition.getDataFields()) { + searchParams.returnField(dataField.getEffectiveStorageName(), false); + } + if (options.isIncludeVectors()) { + for (VectorStoreRecordVectorField vector : recordDefinition.getVectorFields()) { + searchParams.returnField(vector.getEffectiveStorageName(), false); + } + } } - return redisQuery; + return Pair.of(knn, searchParams); } public static byte[] convertListToByteArray(List embeddings) { @@ -61,6 +73,15 @@ public static byte[] convertListToByteArray(List embeddings) { return bytes.array(); } + public static List convertByteArrayToList(byte[] bytes) { + ByteBuffer buffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN); + List embeddings = new java.util.ArrayList<>(); + while (buffer.hasRemaining()) { + embeddings.add(buffer.getFloat()); + } + return embeddings; + } + public static String buildFilter(VectorSearchFilter vectorSearchFilter, VectorStoreRecordDefinition recordDefinition) { if (vectorSearchFilter == null diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java index acaddf9b..28c0a6d7 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java @@ -103,6 +103,7 @@ public static class Builder implements SemanticKernelBuilder Date: Thu, 19 Sep 2024 01:10:05 -0700 Subject: [PATCH 048/277] Update --- ...ashSetVectorStoreRecordCollectionTest.java | 2 +- ...isJsonVectorStoreRecordCollectionTest.java | 36 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index 18dc4a84..9517f23f 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -115,7 +115,7 @@ private RedisHashSetVectorStoreRecordCollection buildRecordCollection(@No private List getHotels() { return List.of( new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f),null, null, 4.0), - new Hotel("id_2", "Hotel2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f),null, null, 4.0), + new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f),null, null, 4.0), new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f),null, null, 5.0), new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f),null, null, 4.0), new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f),null, null, 4.0) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index 358c0238..cef31c7a 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -102,7 +102,7 @@ static void setup() { .build()); } - private RedisJsonVectorStoreRecordCollection buildrecordCollection(@Nonnull RedisJsonVectorStoreRecordCollectionOptions options, @Nonnull String collectionName) { + private RedisJsonVectorStoreRecordCollection buildRecordCollection(@Nonnull RedisJsonVectorStoreRecordCollectionOptions options, @Nonnull String collectionName) { return new RedisJsonVectorStoreRecordCollection<>(new JedisPooled(redisContainer.getRedisURI()), collectionName, RedisJsonVectorStoreRecordCollectionOptions.builder() .withRecordClass(options.getRecordClass()) .withVectorStoreRecordMapper(options.getVectorStoreRecordMapper()) @@ -114,7 +114,7 @@ private RedisJsonVectorStoreRecordCollection buildrecordCollection(@Nonnu private List getHotels() { return List.of( new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f),null, null, 4.0), - new Hotel("id_2", "Hotel2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f),null, null, 4.0), + new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f),null, null, 4.0), new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f),null, null, 5.0), new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f),null, null, 4.0), new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f),null, null, 4.0) @@ -133,15 +133,15 @@ private List getHotels() { @Order(1) @ParameterizedTest @EnumSource(RecordCollectionOptions.class) - public void buildrecordCollection(RecordCollectionOptions options) { - assertNotNull(buildrecordCollection(optionsMap.get(options), options.name())); + public void buildRecordCollection(RecordCollectionOptions options) { + assertNotNull(buildRecordCollection(optionsMap.get(options), options.name())); } @Order(2) @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void createCollectionAsync(RecordCollectionOptions options) { - RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); assertEquals(false, recordCollection.collectionExistsAsync().block()); recordCollection.createCollectionAsync().block(); @@ -150,7 +150,7 @@ public void createCollectionAsync(RecordCollectionOptions options) { @Test public void deleteCollectionAsync() { - RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(RecordCollectionOptions.DEFAULT), "deleteCollectionAsync"); + RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(RecordCollectionOptions.DEFAULT), "deleteCollectionAsync"); assertEquals(false, recordCollection.collectionExistsAsync().block()); recordCollection.createCollectionAsync().block(); @@ -161,7 +161,7 @@ public void deleteCollectionAsync() { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void upsertAndGetRecordAsync(RecordCollectionOptions options) { - RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); for (Hotel hotel : hotels) { @@ -178,7 +178,7 @@ public void upsertAndGetRecordAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getBatchAsync(RecordCollectionOptions options) { - RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); for (Hotel hotel : hotels) { @@ -200,7 +200,7 @@ public void getBatchAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void upsertBatchAsync(RecordCollectionOptions options) { - RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); List keys = recordCollection.upsertBatchAsync(hotels, null).block(); @@ -218,7 +218,7 @@ public void upsertBatchAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void deleteAsync(RecordCollectionOptions options) { - RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -233,7 +233,7 @@ public void deleteAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void deleteBatchAsync(RecordCollectionOptions options) { - RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -252,7 +252,7 @@ public void deleteBatchAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getAsyncWithVectors(RecordCollectionOptions options) { - RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -269,7 +269,7 @@ public void getAsyncWithVectors(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getBatchAsyncWithVectors(RecordCollectionOptions options) { - RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -291,7 +291,7 @@ public void getBatchAsyncWithVectors(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getAsyncWithNoVectors(RecordCollectionOptions options) { - RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -309,7 +309,7 @@ public void getAsyncWithNoVectors(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getBatchAsyncWithNoVectors(RecordCollectionOptions options) { - RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), options.name()); + RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -344,7 +344,7 @@ private static Stream provideSearchParameters() { @MethodSource("provideSearchParameters") public void search(RecordCollectionOptions options, String embeddingName) { String collectionName = "search" + embeddingName; - RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), collectionName); + RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -366,7 +366,7 @@ public void search(RecordCollectionOptions options, String embeddingName) { @MethodSource("provideSearchParameters") public void searchWithVectors(RecordCollectionOptions options, String embeddingName) { String collectionName = "search" + embeddingName; - RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), collectionName); + RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -389,7 +389,7 @@ public void searchWithVectors(RecordCollectionOptions options, String embeddingN @MethodSource("provideSearchParameters") public void searchWithOffSet(RecordCollectionOptions options, String embeddingName) { String collectionName = "search" + embeddingName; - RedisJsonVectorStoreRecordCollection recordCollection = buildrecordCollection(optionsMap.get(options), collectionName); + RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); From 8f3be158cc1df51ceac8a7df829dafeac99b8e71 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 19 Sep 2024 01:16:50 -0700 Subject: [PATCH 049/277] Add distance function to custom definitions --- .../RedisHashSetVectorStoreRecordCollectionTest.java | 6 +++--- .../RedisJsonVectorStoreRecordCollectionTest.java | 10 +++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index 9517f23f..da957ec5 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -344,7 +344,7 @@ private static Stream provideSearchParameters() { @ParameterizedTest @MethodSource("provideSearchParameters") public void search(RecordCollectionOptions options, String embeddingName) { - String collectionName = "search" + embeddingName; + String collectionName = "search" + options.name() + embeddingName; RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); @@ -366,7 +366,7 @@ public void search(RecordCollectionOptions options, String embeddingName) { @ParameterizedTest @MethodSource("provideSearchParameters") public void searchWithVectors(RecordCollectionOptions options, String embeddingName) { - String collectionName = "search" + embeddingName; + String collectionName = "search" + options.name() + embeddingName; RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); @@ -389,7 +389,7 @@ public void searchWithVectors(RecordCollectionOptions options, String embeddingN @ParameterizedTest @MethodSource("provideSearchParameters") public void searchWithOffSet(RecordCollectionOptions options, String embeddingName) { - String collectionName = "search" + embeddingName; + String collectionName = "search" + options.name() + embeddingName; RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index cef31c7a..df58bd38 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -3,6 +3,7 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; @@ -75,18 +76,21 @@ static void setup() { fields.add(VectorStoreRecordVectorField.builder() .withName("euclidean") .withStorageName("summaryEmbedding1") + .withDistanceFunction(DistanceFunction.EUCLIDEAN_DISTANCE) .withFieldType(List.class) .withDimensions(8) .build()); fields.add(VectorStoreRecordVectorField.builder() .withName("cosineDistance") .withStorageName("summaryEmbedding2") + .withDistanceFunction(DistanceFunction.COSINE_DISTANCE) .withFieldType(List.class) .withDimensions(8) .build()); fields.add(VectorStoreRecordVectorField.builder() .withName("dotProduct") .withStorageName("summaryEmbedding3") + .withDistanceFunction(DistanceFunction.DOT_PRODUCT) .withFieldType(List.class) .withDimensions(8) .build()); @@ -343,7 +347,7 @@ private static Stream provideSearchParameters() { @ParameterizedTest @MethodSource("provideSearchParameters") public void search(RecordCollectionOptions options, String embeddingName) { - String collectionName = "search" + embeddingName; + String collectionName = "search" + options.name() + embeddingName; RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); @@ -365,7 +369,7 @@ public void search(RecordCollectionOptions options, String embeddingName) { @ParameterizedTest @MethodSource("provideSearchParameters") public void searchWithVectors(RecordCollectionOptions options, String embeddingName) { - String collectionName = "search" + embeddingName; + String collectionName = "search" + options.name() + embeddingName; RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); @@ -388,7 +392,7 @@ public void searchWithVectors(RecordCollectionOptions options, String embeddingN @ParameterizedTest @MethodSource("provideSearchParameters") public void searchWithOffSet(RecordCollectionOptions options, String embeddingName) { - String collectionName = "search" + embeddingName; + String collectionName = "search" + options.name() + embeddingName; RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); From 384c7ac7fd6b43252c3e294f86d1dd53a43062da Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 19 Sep 2024 01:23:04 -0700 Subject: [PATCH 050/277] Update collection name --- .../redis/RedisHashSetVectorStoreRecordCollectionTest.java | 4 ++-- .../redis/RedisJsonVectorStoreRecordCollectionTest.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index da957ec5..2df36e47 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -366,7 +366,7 @@ public void search(RecordCollectionOptions options, String embeddingName) { @ParameterizedTest @MethodSource("provideSearchParameters") public void searchWithVectors(RecordCollectionOptions options, String embeddingName) { - String collectionName = "search" + options.name() + embeddingName; + String collectionName = "searchWithVectors" + options.name() + embeddingName; RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); @@ -389,7 +389,7 @@ public void searchWithVectors(RecordCollectionOptions options, String embeddingN @ParameterizedTest @MethodSource("provideSearchParameters") public void searchWithOffSet(RecordCollectionOptions options, String embeddingName) { - String collectionName = "search" + options.name() + embeddingName; + String collectionName = "searchWithOffSet" + options.name() + embeddingName; RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index df58bd38..22f6cbc5 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -369,7 +369,7 @@ public void search(RecordCollectionOptions options, String embeddingName) { @ParameterizedTest @MethodSource("provideSearchParameters") public void searchWithVectors(RecordCollectionOptions options, String embeddingName) { - String collectionName = "search" + options.name() + embeddingName; + String collectionName = "searchWithVectors" + options.name() + embeddingName; RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); @@ -392,7 +392,7 @@ public void searchWithVectors(RecordCollectionOptions options, String embeddingN @ParameterizedTest @MethodSource("provideSearchParameters") public void searchWithOffSet(RecordCollectionOptions options, String embeddingName) { - String collectionName = "search" + options.name() + embeddingName; + String collectionName = "searchWithOffSet" + options.name() + embeddingName; RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); From 09e0388f656d20e5e29d94a1504f4e86114424fd Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 19 Sep 2024 01:44:47 -0700 Subject: [PATCH 051/277] Fix image version --- ...ashSetVectorStoreRecordCollectionTest.java | 2 +- ...isJsonVectorStoreRecordCollectionTest.java | 2 +- .../memory/AzureAISearchVectorStore.java | 26 ++++++++++--------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index 2df36e47..ded4d2c2 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -41,7 +41,7 @@ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class RedisHashSetVectorStoreRecordCollectionTest { - @Container private static final RedisContainer redisContainer = new RedisContainer("redis/redis-stack:latest"); + @Container private static final RedisContainer redisContainer = new RedisContainer("redis/redis-stack:7.4.0-v0"); private static final Map> optionsMap = new HashMap<>(); diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index 22f6cbc5..ba74c491 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -41,7 +41,7 @@ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class RedisJsonVectorStoreRecordCollectionTest { - @Container private static final RedisContainer redisContainer = new RedisContainer("redis/redis-stack:latest"); + @Container private static final RedisContainer redisContainer = new RedisContainer("redis/redis-stack:7.4.0-v0"); private static final Map> optionsMap = new HashMap<>(); diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java index 2a1a49ab..ab2b72aa 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java @@ -118,17 +118,19 @@ public static void dataStorageWithAzureAISearch( OpenAITextEmbeddingGenerationService embeddingGeneration) { // Create a new Azure AI Search vector store - var azureAISearchVectorStore = com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStore.builder() + var azureAISearchVectorStore = com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStore + .builder() .withSearchIndexAsyncClient(searchClient) .withOptions(new AzureAISearchVectorStoreOptions()) .build(); String collectionName = "skgithubfiles"; - var collection = (AzureAISearchVectorStoreRecordCollection) azureAISearchVectorStore.getCollection( - collectionName, - AzureAISearchVectorStoreRecordCollectionOptions.builder() - .withRecordClass(GitHubFile.class) - .build()); + var collection = (AzureAISearchVectorStoreRecordCollection) azureAISearchVectorStore + .getCollection( + collectionName, + AzureAISearchVectorStoreRecordCollectionOptions.builder() + .withRecordClass(GitHubFile.class) + .build()); // Create collection if it does not exist and store data collection @@ -146,17 +148,17 @@ public static void dataStorageWithAzureAISearch( } var searchResult = results.get(0); System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", - searchResult.getScore(), searchResult.getRecord().link, searchResult.getRecord().description); + searchResult.getScore(), searchResult.getRecord().link, + searchResult.getRecord().description); } - private static Mono>> search( - String searchText, - AzureAISearchVectorStoreRecordCollection recordCollection, - OpenAITextEmbeddingGenerationService embeddingGeneration) { + String searchText, + AzureAISearchVectorStoreRecordCollection recordCollection, + OpenAITextEmbeddingGenerationService embeddingGeneration) { return embeddingGeneration.generateEmbeddingsAsync(Collections.singletonList(searchText)) - .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); + .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); } private static Mono> storeData( From 87e9e4dea011a631a1aa35294e86943f1d112b6f Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 19 Sep 2024 12:25:51 -0700 Subject: [PATCH 052/277] Change collectionName --- .../redis/RedisHashSetVectorStoreRecordCollectionTest.java | 6 +++--- .../redis/RedisJsonVectorStoreRecordCollectionTest.java | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index ded4d2c2..86afc85a 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -344,7 +344,7 @@ private static Stream provideSearchParameters() { @ParameterizedTest @MethodSource("provideSearchParameters") public void search(RecordCollectionOptions options, String embeddingName) { - String collectionName = "search" + options.name() + embeddingName; + String collectionName = "search"; RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); @@ -366,7 +366,7 @@ public void search(RecordCollectionOptions options, String embeddingName) { @ParameterizedTest @MethodSource("provideSearchParameters") public void searchWithVectors(RecordCollectionOptions options, String embeddingName) { - String collectionName = "searchWithVectors" + options.name() + embeddingName; + String collectionName = "search"; RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); @@ -389,7 +389,7 @@ public void searchWithVectors(RecordCollectionOptions options, String embeddingN @ParameterizedTest @MethodSource("provideSearchParameters") public void searchWithOffSet(RecordCollectionOptions options, String embeddingName) { - String collectionName = "searchWithOffSet" + options.name() + embeddingName; + String collectionName = "search"; RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index ba74c491..3c0d0ac8 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -347,7 +347,7 @@ private static Stream provideSearchParameters() { @ParameterizedTest @MethodSource("provideSearchParameters") public void search(RecordCollectionOptions options, String embeddingName) { - String collectionName = "search" + options.name() + embeddingName; + String collectionName = "search"; RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); @@ -369,7 +369,7 @@ public void search(RecordCollectionOptions options, String embeddingName) { @ParameterizedTest @MethodSource("provideSearchParameters") public void searchWithVectors(RecordCollectionOptions options, String embeddingName) { - String collectionName = "searchWithVectors" + options.name() + embeddingName; + String collectionName = "search"; RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); @@ -392,7 +392,7 @@ public void searchWithVectors(RecordCollectionOptions options, String embeddingN @ParameterizedTest @MethodSource("provideSearchParameters") public void searchWithOffSet(RecordCollectionOptions options, String embeddingName) { - String collectionName = "searchWithOffSet" + options.name() + embeddingName; + String collectionName = "search"; RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); From 649099a1df2e320f067a8ffa2fff834390481e8e Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 19 Sep 2024 14:26:56 -0700 Subject: [PATCH 053/277] Add sleep to guarantee indexed data in tests --- ...ashSetVectorStoreRecordCollectionTest.java | 81 +++++++++++------ ...isJsonVectorStoreRecordCollectionTest.java | 87 ++++++++++++------- ...disHashSetVectorStoreRecordCollection.java | 2 +- .../RedisHashSetVectorStoreRecordMapper.java | 7 +- .../filter/RedisEqualToFilterClause.java | 4 +- 5 files changed, 115 insertions(+), 66 deletions(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index 86afc85a..d6d270e5 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -2,7 +2,6 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; @@ -41,7 +40,7 @@ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class RedisHashSetVectorStoreRecordCollectionTest { - @Container private static final RedisContainer redisContainer = new RedisContainer("redis/redis-stack:7.4.0-v0"); + @Container private static final RedisContainer redisContainer = new RedisContainer("redis/redis-stack:latest"); private static final Map> optionsMap = new HashMap<>(); @@ -101,9 +100,29 @@ static void setup() { .withRecordClass(Hotel.class) .withRecordDefinition(recordDefinition) .build()); + + // Search configuration + List hotels = getHotels(); + + for (RecordCollectionOptions options : RecordCollectionOptions.values()) { + String collectionName = getCollectionName("search", options); + RedisHashSetVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); + + recordCollection.createCollectionAsync().block(); + assertEquals(true, recordCollection.collectionExistsAsync().block()); + + recordCollection.upsertBatchAsync(hotels, null).block(); + } + + // Wait for data to be indexed + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } - private RedisHashSetVectorStoreRecordCollection buildRecordCollection(@Nonnull RedisHashSetVectorStoreRecordCollectionOptions options, @Nonnull String collectionName) { + private static RedisHashSetVectorStoreRecordCollection createCollection(@Nonnull RedisHashSetVectorStoreRecordCollectionOptions options, @Nonnull String collectionName) { return new RedisHashSetVectorStoreRecordCollection<>(new JedisPooled(redisContainer.getRedisURI()), collectionName, RedisHashSetVectorStoreRecordCollectionOptions.builder() .withRecordClass(options.getRecordClass()) .withVectorStoreRecordMapper(options.getVectorStoreRecordMapper()) @@ -112,7 +131,7 @@ private RedisHashSetVectorStoreRecordCollection buildRecordCollection(@No .build()); } - private List getHotels() { + private static List getHotels() { return List.of( new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f),null, null, 4.0), new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f),null, null, 4.0), @@ -131,18 +150,16 @@ private List getHotels() { */ private static final List SEARCH_EMBEDDINGS = Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.2f); - @Order(1) - @ParameterizedTest - @EnumSource(RecordCollectionOptions.class) - public void buildRecordCollection(RecordCollectionOptions options) { - assertNotNull(buildRecordCollection(optionsMap.get(options), options.name())); + private static String getCollectionName(String id, RecordCollectionOptions options) { + return id + options.name(); } - @Order(2) + @Order(1) @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void createCollectionAsync(RecordCollectionOptions options) { - RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); + String collectionName = getCollectionName("createCollectionAsync", options); + RedisHashSetVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); assertEquals(false, recordCollection.collectionExistsAsync().block()); recordCollection.createCollectionAsync().block(); @@ -151,7 +168,8 @@ public void createCollectionAsync(RecordCollectionOptions options) { @Test public void deleteCollectionAsync() { - RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(RecordCollectionOptions.DEFAULT), "deleteCollectionAsync"); + String collectionName = getCollectionName("deleteCollectionAsync", RecordCollectionOptions.DEFAULT); + RedisHashSetVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(RecordCollectionOptions.DEFAULT), collectionName); assertEquals(false, recordCollection.collectionExistsAsync().block()); recordCollection.createCollectionAsync().block(); @@ -162,7 +180,8 @@ public void deleteCollectionAsync() { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void upsertAndGetRecordAsync(RecordCollectionOptions options) { - RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); + String collectionName = getCollectionName("upsertAndGetRecordAsync", options); + RedisHashSetVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); for (Hotel hotel : hotels) { @@ -179,7 +198,8 @@ public void upsertAndGetRecordAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getBatchAsync(RecordCollectionOptions options) { - RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); + String collectionName = getCollectionName("getBatchAsync", options); + RedisHashSetVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); for (Hotel hotel : hotels) { @@ -201,7 +221,8 @@ public void getBatchAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void upsertBatchAsync(RecordCollectionOptions options) { - RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); + String collectionName = getCollectionName("upsertBatchAsync", options); + RedisHashSetVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); List keys = recordCollection.upsertBatchAsync(hotels, null).block(); @@ -219,7 +240,8 @@ public void upsertBatchAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void deleteAsync(RecordCollectionOptions options) { - RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); + String collectionName = getCollectionName("deleteAsync", options); + RedisHashSetVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -234,7 +256,8 @@ public void deleteAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void deleteBatchAsync(RecordCollectionOptions options) { - RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); + String collectionName = getCollectionName("deleteBatchAsync", options); + RedisHashSetVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -253,7 +276,8 @@ public void deleteBatchAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getAsyncWithVectors(RecordCollectionOptions options) { - RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); + String collectionName = getCollectionName("getAsyncWithVectors", options); + RedisHashSetVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -270,7 +294,8 @@ public void getAsyncWithVectors(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getBatchAsyncWithVectors(RecordCollectionOptions options) { - RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); + String collectionName = getCollectionName("getBatchAsyncWithVectors", options); + RedisHashSetVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -292,7 +317,8 @@ public void getBatchAsyncWithVectors(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getAsyncWithNoVectors(RecordCollectionOptions options) { - RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); + String collectionName = getCollectionName("getAsyncWithNoVectors", options); + RedisHashSetVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -310,7 +336,8 @@ public void getAsyncWithNoVectors(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getBatchAsyncWithNoVectors(RecordCollectionOptions options) { - RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); + String collectionName = getCollectionName("getBatchAsyncWithNoVectors", options); + RedisHashSetVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -344,8 +371,8 @@ private static Stream provideSearchParameters() { @ParameterizedTest @MethodSource("provideSearchParameters") public void search(RecordCollectionOptions options, String embeddingName) { - String collectionName = "search"; - RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); + String collectionName = getCollectionName("search", options); + RedisHashSetVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -366,8 +393,8 @@ public void search(RecordCollectionOptions options, String embeddingName) { @ParameterizedTest @MethodSource("provideSearchParameters") public void searchWithVectors(RecordCollectionOptions options, String embeddingName) { - String collectionName = "search"; - RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); + String collectionName = getCollectionName("search", options); + RedisHashSetVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -389,8 +416,8 @@ public void searchWithVectors(RecordCollectionOptions options, String embeddingN @ParameterizedTest @MethodSource("provideSearchParameters") public void searchWithOffSet(RecordCollectionOptions options, String embeddingName) { - String collectionName = "search"; - RedisHashSetVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); + String collectionName = getCollectionName("search", options); + RedisHashSetVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index 3c0d0ac8..6eab5a1d 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -3,7 +3,6 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; -import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; @@ -41,7 +40,7 @@ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class RedisJsonVectorStoreRecordCollectionTest { - @Container private static final RedisContainer redisContainer = new RedisContainer("redis/redis-stack:7.4.0-v0"); + @Container private static final RedisContainer redisContainer = new RedisContainer("redis/redis-stack:latest"); private static final Map> optionsMap = new HashMap<>(); @@ -76,21 +75,18 @@ static void setup() { fields.add(VectorStoreRecordVectorField.builder() .withName("euclidean") .withStorageName("summaryEmbedding1") - .withDistanceFunction(DistanceFunction.EUCLIDEAN_DISTANCE) .withFieldType(List.class) .withDimensions(8) .build()); fields.add(VectorStoreRecordVectorField.builder() .withName("cosineDistance") .withStorageName("summaryEmbedding2") - .withDistanceFunction(DistanceFunction.COSINE_DISTANCE) .withFieldType(List.class) .withDimensions(8) .build()); fields.add(VectorStoreRecordVectorField.builder() .withName("dotProduct") .withStorageName("summaryEmbedding3") - .withDistanceFunction(DistanceFunction.DOT_PRODUCT) .withFieldType(List.class) .withDimensions(8) .build()); @@ -104,9 +100,29 @@ static void setup() { .withRecordClass(Hotel.class) .withRecordDefinition(recordDefinition) .build()); + + // Search configuration + List hotels = getHotels(); + + for (RecordCollectionOptions options : RecordCollectionOptions.values()) { + String collectionName = getCollectionName("search", options); + RedisJsonVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); + + recordCollection.createCollectionAsync().block(); + assertEquals(true, recordCollection.collectionExistsAsync().block()); + + recordCollection.upsertBatchAsync(hotels, null).block(); + } + + // Wait for data to be indexed + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } - private RedisJsonVectorStoreRecordCollection buildRecordCollection(@Nonnull RedisJsonVectorStoreRecordCollectionOptions options, @Nonnull String collectionName) { + private static RedisJsonVectorStoreRecordCollection createCollection(@Nonnull RedisJsonVectorStoreRecordCollectionOptions options, @Nonnull String collectionName) { return new RedisJsonVectorStoreRecordCollection<>(new JedisPooled(redisContainer.getRedisURI()), collectionName, RedisJsonVectorStoreRecordCollectionOptions.builder() .withRecordClass(options.getRecordClass()) .withVectorStoreRecordMapper(options.getVectorStoreRecordMapper()) @@ -115,7 +131,7 @@ private RedisJsonVectorStoreRecordCollection buildRecordCollection(@Nonnu .build()); } - private List getHotels() { + private static List getHotels() { return List.of( new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f),null, null, 4.0), new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f),null, null, 4.0), @@ -134,18 +150,16 @@ private List getHotels() { */ private static final List SEARCH_EMBEDDINGS = Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.2f); - @Order(1) - @ParameterizedTest - @EnumSource(RecordCollectionOptions.class) - public void buildRecordCollection(RecordCollectionOptions options) { - assertNotNull(buildRecordCollection(optionsMap.get(options), options.name())); + private static String getCollectionName(String id, RecordCollectionOptions options) { + return id + options.name(); } - @Order(2) + @Order(1) @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void createCollectionAsync(RecordCollectionOptions options) { - RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); + String collectionName = getCollectionName("createCollectionAsync", options); + RedisJsonVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); assertEquals(false, recordCollection.collectionExistsAsync().block()); recordCollection.createCollectionAsync().block(); @@ -154,7 +168,8 @@ public void createCollectionAsync(RecordCollectionOptions options) { @Test public void deleteCollectionAsync() { - RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(RecordCollectionOptions.DEFAULT), "deleteCollectionAsync"); + String collectionName = getCollectionName("deleteCollectionAsync", RecordCollectionOptions.DEFAULT); + RedisJsonVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(RecordCollectionOptions.DEFAULT), collectionName); assertEquals(false, recordCollection.collectionExistsAsync().block()); recordCollection.createCollectionAsync().block(); @@ -165,7 +180,8 @@ public void deleteCollectionAsync() { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void upsertAndGetRecordAsync(RecordCollectionOptions options) { - RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); + String collectionName = getCollectionName("upsertAndGetRecordAsync", options); + RedisJsonVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); for (Hotel hotel : hotels) { @@ -182,7 +198,8 @@ public void upsertAndGetRecordAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getBatchAsync(RecordCollectionOptions options) { - RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); + String collectionName = getCollectionName("getBatchAsync", options); + RedisJsonVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); for (Hotel hotel : hotels) { @@ -192,7 +209,7 @@ public void getBatchAsync(RecordCollectionOptions options) { List ids = new ArrayList<>(); hotels.forEach(hotel -> ids.add(hotel.getId())); - List retrievedHotels = recordCollection.getBatchAsync(ids, null).block(); + List retrievedHotels = recordCollection.getBatchAsync(ids, new GetRecordOptions(true)).block(); assertNotNull(retrievedHotels); assertEquals(hotels.size(), retrievedHotels.size()); @@ -204,7 +221,8 @@ public void getBatchAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void upsertBatchAsync(RecordCollectionOptions options) { - RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); + String collectionName = getCollectionName("upsertBatchAsync", options); + RedisJsonVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); List keys = recordCollection.upsertBatchAsync(hotels, null).block(); @@ -222,7 +240,8 @@ public void upsertBatchAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void deleteAsync(RecordCollectionOptions options) { - RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); + String collectionName = getCollectionName("deleteAsync", options); + RedisJsonVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -237,7 +256,8 @@ public void deleteAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void deleteBatchAsync(RecordCollectionOptions options) { - RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); + String collectionName = getCollectionName("deleteBatchAsync", options); + RedisJsonVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -256,7 +276,8 @@ public void deleteBatchAsync(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getAsyncWithVectors(RecordCollectionOptions options) { - RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); + String collectionName = getCollectionName("getAsyncWithVectors", options); + RedisJsonVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -273,7 +294,8 @@ public void getAsyncWithVectors(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getBatchAsyncWithVectors(RecordCollectionOptions options) { - RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); + String collectionName = getCollectionName("getBatchAsyncWithVectors", options); + RedisJsonVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -295,7 +317,8 @@ public void getBatchAsyncWithVectors(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getAsyncWithNoVectors(RecordCollectionOptions options) { - RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); + String collectionName = getCollectionName("getAsyncWithNoVectors", options); + RedisJsonVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -313,7 +336,8 @@ public void getAsyncWithNoVectors(RecordCollectionOptions options) { @ParameterizedTest @EnumSource(RecordCollectionOptions.class) public void getBatchAsyncWithNoVectors(RecordCollectionOptions options) { - RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), options.name()); + String collectionName = getCollectionName("getBatchAsyncWithNoVectors", options); + RedisJsonVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -347,8 +371,8 @@ private static Stream provideSearchParameters() { @ParameterizedTest @MethodSource("provideSearchParameters") public void search(RecordCollectionOptions options, String embeddingName) { - String collectionName = "search"; - RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); + String collectionName = getCollectionName("search", options); + RedisJsonVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -369,8 +393,8 @@ public void search(RecordCollectionOptions options, String embeddingName) { @ParameterizedTest @MethodSource("provideSearchParameters") public void searchWithVectors(RecordCollectionOptions options, String embeddingName) { - String collectionName = "search"; - RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); + String collectionName = getCollectionName("search", options); + RedisJsonVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -392,8 +416,8 @@ public void searchWithVectors(RecordCollectionOptions options, String embeddingN @ParameterizedTest @MethodSource("provideSearchParameters") public void searchWithOffSet(RecordCollectionOptions options, String embeddingName) { - String collectionName = "search"; - RedisJsonVectorStoreRecordCollection recordCollection = buildRecordCollection(optionsMap.get(options), collectionName); + String collectionName = getCollectionName("search", options); + RedisJsonVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(options), collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -411,5 +435,4 @@ public void searchWithOffSet(RecordCollectionOptions options, String embeddingNa // The first hotel should be the most similar assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); } - } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java index 0f7ae6cb..6802ad03 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java @@ -209,7 +209,7 @@ private String removeKeyPrefixIfNeeded(String key, String collectionName) { return key; } - private byte[] stringToBytes(String value) { + static byte[] stringToBytes(String value) { return value.getBytes(StandardCharsets.UTF_8); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java index 01a5531f..2f91ea11 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java @@ -22,13 +22,14 @@ import java.nio.charset.StandardCharsets; import java.util.AbstractMap; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.function.BiFunction; import java.util.function.Function; +import static com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollection.stringToBytes; + public class RedisHashSetVectorStoreRecordMapper extends VectorStoreRecordMapper>> { @@ -124,7 +125,7 @@ public RedisHashSetVectorStoreRecordMapper build() { JsonNode value = jsonNode.get(field.getEffectiveStorageName()); if (value != null) { storage.put( - field.getEffectiveStorageName().getBytes(StandardCharsets.UTF_8), + stringToBytes(field.getEffectiveStorageName()), objectMapper.writeValueAsBytes(value)); } } @@ -133,7 +134,7 @@ public RedisHashSetVectorStoreRecordMapper build() { List vector = objectMapper.convertValue(value, List.class); if (value != null) { storage.put( - field.getEffectiveStorageName().getBytes(StandardCharsets.UTF_8), + stringToBytes(field.getEffectiveStorageName()), RedisVectorStoreCollectionSearchMapping .convertListToByteArray(vector)); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/filter/RedisEqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/filter/RedisEqualToFilterClause.java index b6ecf43f..aa823ef9 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/filter/RedisEqualToFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/filter/RedisEqualToFilterClause.java @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis.filter; -import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; import com.microsoft.semantickernel.data.filter.EqualToFilterClause; import com.microsoft.semantickernel.exceptions.SKException; @@ -28,8 +27,7 @@ public String getFilter() { formattedValue = String.format("\"%s\"", value); } else if (value instanceof Boolean) { formattedValue = value.toString().toLowerCase(); - } else if (value instanceof Integer || value instanceof Long || value instanceof Float - || value instanceof Double) { + } else if (value instanceof Number) { formattedValue = value.toString(); } else if (value instanceof OffsetDateTime) { formattedValue = ((OffsetDateTime) value) From 25f06301cac464058ac43ec8d9f9b817e6b1ce34 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 19 Sep 2024 15:39:57 -0700 Subject: [PATCH 054/277] Update docs --- .../data/redis/RedisHashSetVectorStoreRecordCollection.java | 5 +++-- .../data/redis/RedisJsonVectorStoreRecordCollection.java | 2 +- .../data/redis/RedisVectorStoreCollectionCreateMapping.java | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java index 6802ad03..ae2c7d23 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java @@ -156,7 +156,7 @@ public Mono collectionExistsAsync() { public Mono> createCollectionAsync() { return Mono.fromRunnable(() -> { Schema schema = RedisVectorStoreCollectionCreateMapping - .mapToSchema(recordDefinition.getAllFields(), false); + .mapToSchema(recordDefinition.getAllFields(), RedisStorageType.HASH_SET); IndexDefinition indexDefinition = new IndexDefinition(IndexDefinition.Type.HASH) .setPrefixes(collectionName + ":"); @@ -264,7 +264,7 @@ public Mono> getBatchAsync(List keys, .add(new AbstractMap.SimpleEntry<>(key, pipeline.hgetAll(stringToBytes(redisKey)))); } else { - // Returns List with the values of the fields + // Returns List with the values of the fields responses .add(new AbstractMap.SimpleEntry<>(key, pipeline.hmget(stringToBytes(redisKey), dataFields))); @@ -411,6 +411,7 @@ public Mono>> searchAsync(VectorSearchQuery quer (byte[]) entry.getValue()); } else if (entry.getKey().equals( RedisVectorStoreCollectionSearchMapping.VECTOR_SCORE_FIELD)) { + // Score is stored as a string in one of the fields score = Double.parseDouble((String) entry.getValue()); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java index 739ddf1a..994e12ea 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java @@ -163,7 +163,7 @@ public Mono collectionExistsAsync() { public Mono> createCollectionAsync() { return Mono.fromRunnable(() -> { Schema schema = RedisVectorStoreCollectionCreateMapping - .mapToSchema(recordDefinition.getAllFields(), true); + .mapToSchema(recordDefinition.getAllFields(), RedisStorageType.JSON); IndexDefinition indexDefinition = new IndexDefinition(IndexDefinition.Type.JSON) .setPrefixes(collectionName + ":"); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java index e0cf037d..2613be90 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java @@ -71,8 +71,9 @@ private static String getRedisPath(String name, boolean withRedisJsonRoot) { } public static Schema mapToSchema(List fields, - boolean withRedisJsonRoot) { + RedisStorageType storageType) { Schema schema = new Schema(); + boolean withRedisJsonRoot = storageType == RedisStorageType.JSON; for (VectorStoreRecordField field : fields) { if (field instanceof VectorStoreRecordKeyField) { From 4135ad511d86dd5b73e2ee38a66e851f67c83e3a Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Mon, 23 Sep 2024 12:53:30 -0700 Subject: [PATCH 055/277] Clean Vector search interfaces --- ...reAISearchVectorStoreRecordCollection.java | 108 ++++++------- .../jdbc/JDBCVectorStoreQueryProvider.java | 149 ++++++++---------- .../jdbc/JDBCVectorStoreRecordCollection.java | 15 +- .../jdbc/SQLVectorStoreQueryProvider.java | 8 +- .../PostgreSQLVectorStoreQueryProvider.java | 131 ++++++++------- ...disHashSetVectorStoreRecordCollection.java | 92 +++++------ .../RedisJsonVectorStoreRecordCollection.java | 92 +++++------ ...disVectorStoreCollectionSearchMapping.java | 11 +- .../data/vectorsearch/VectorSearch.java | 12 -- ...earch.java => VectorizableTextSearch.java} | 2 +- .../data/vectorsearch/VectorizedSearch.java | 2 +- .../queries/VectorSearchQuery.java | 34 ---- .../queries/VectorSearchQueryType.java | 20 --- .../queries/VectorizableTextSearchQuery.java | 30 ---- .../queries/VectorizedSearchQuery.java | 30 ---- 15 files changed, 274 insertions(+), 462 deletions(-) delete mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearch.java rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/{VectorizableSearch.java => VectorizableTextSearch.java} (89%) delete mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java delete mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQueryType.java delete mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizableTextSearchQuery.java delete mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index e20e24d8..fad24fe5 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -14,7 +14,7 @@ import com.azure.search.documents.models.VectorQuery; import com.azure.search.documents.models.VectorizableTextQuery; import com.azure.search.documents.models.VectorizedQuery; -import com.microsoft.semantickernel.data.vectorsearch.VectorizableSearch; +import com.microsoft.semantickernel.data.vectorsearch.VectorizableTextSearch; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorizedSearch; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; @@ -28,9 +28,6 @@ import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; -import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; -import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizableTextSearchQuery; -import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizedSearchQuery; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.time.OffsetDateTime; @@ -49,7 +46,7 @@ public class AzureAISearchVectorStoreRecordCollection implements VectorStoreRecordCollection, VectorizedSearch, - VectorizableSearch { + VectorizableTextSearch { private static final HashSet> supportedKeyTypes = new HashSet<>( Collections.singletonList( @@ -277,8 +274,25 @@ public Mono deleteBatchAsync(List keys, DeleteRecordOptions option }).collect(Collectors.toList())).then(); } - private Mono>> searchAndMapAsync(SearchOptions searchOptions, + private Mono>> searchAndMapAsync( + List vectorQueries, VectorSearchOptions options, GetRecordOptions getRecordOptions) { + + String filter = AzureAISearchVectorStoreCollectionSearchMapping + .buildFilterString(options.getVectorSearchFilter(), recordDefinition); + + SearchOptions searchOptions = new SearchOptions() + .setFilter(filter) + .setTop(options.getLimit()) + .setSkip(options.getOffset()) + .setScoringParameters() + .setVectorSearchOptions(new com.azure.search.documents.models.VectorSearchOptions() + .setQueries(vectorQueries)); + + if (!options.isIncludeVectors()) { + searchOptions.setSelect(nonVectorFields.toArray(new String[0])); + } + VectorStoreRecordMapper mapper = this.options .getVectorStoreRecordMapper(); @@ -300,70 +314,32 @@ record = response.getDocument(this.options.getRecordClass()); } /** - * Search the vector store for records that match the given embedding and filter. + * Vectorizable text search. This method searches for records that are similar to the given text. * - * @param query The vector search query. + * @param searchText The text to search with. + * @param options The options to use for the search. * @return A list of search results. */ @Override - public Mono>> searchAsync(VectorSearchQuery query) { + public Mono>> searchAsync(String searchText, + VectorSearchOptions options) { if (firstVectorFieldName == null) { throw new SKException("No vector fields defined. Cannot perform vector search"); } - VectorSearchOptions options = query.getSearchOptions(); if (options == null) { options = VectorSearchOptions.createDefault(firstVectorFieldName); } List vectorQueries = new ArrayList<>(); - - if (query instanceof VectorizedSearchQuery) { - vectorQueries.add(new VectorizedQuery(((VectorizedSearchQuery) query).getVector()) - .setFields(recordDefinition.getField(options.getVectorFieldName() != null - ? options.getVectorFieldName() - : firstVectorFieldName).getEffectiveStorageName()) - .setKNearestNeighborsCount(options.getLimit())); - } else if (query instanceof VectorizableTextSearchQuery) { - vectorQueries - .add(new VectorizableTextQuery(((VectorizableTextSearchQuery) query).getQueryText()) - .setFields(recordDefinition.getField(options.getVectorFieldName() != null - ? options.getVectorFieldName() - : firstVectorFieldName).getEffectiveStorageName()) - .setKNearestNeighborsCount(options.getLimit())); - } else { - throw new SKException("Unsupported query type: " + query.getQueryType()); - } - - String filter = AzureAISearchVectorStoreCollectionSearchMapping - .buildFilterString(options.getVectorSearchFilter(), recordDefinition); - - SearchOptions searchOptions = new SearchOptions() - .setFilter(filter) - .setTop(options.getLimit()) - .setSkip(options.getOffset()) - .setScoringParameters() - .setVectorSearchOptions(new com.azure.search.documents.models.VectorSearchOptions() - .setQueries(vectorQueries)); - - if (!options.isIncludeVectors()) { - searchOptions.setSelect(nonVectorFields.toArray(new String[0])); - } - - return searchAndMapAsync(searchOptions, new GetRecordOptions(options.isIncludeVectors())); - } - - /** - * Vectorizable text search. This method searches for records that are similar to the given text. - * - * @param searchText The text to search with. - * @param options The options to use for the search. - * @return A list of search results. - */ - @Override - public Mono>> searchAsync(String searchText, - VectorSearchOptions options) { - return searchAsync(VectorSearchQuery.createQuery(searchText, options)); + vectorQueries.add(new VectorizableTextQuery(searchText) + .setFields(recordDefinition.getField(options.getVectorFieldName() != null + ? options.getVectorFieldName() + : firstVectorFieldName).getEffectiveStorageName()) + .setKNearestNeighborsCount(options.getLimit())); + + return searchAndMapAsync(vectorQueries, options, + new GetRecordOptions(options.isIncludeVectors())); } /** @@ -376,6 +352,22 @@ public Mono>> searchAsync(String searchText, @Override public Mono>> searchAsync(List vector, VectorSearchOptions options) { - return searchAsync(VectorSearchQuery.createQuery(vector, options)); + if (firstVectorFieldName == null) { + throw new SKException("No vector fields defined. Cannot perform vector search"); + } + + if (options == null) { + options = VectorSearchOptions.createDefault(firstVectorFieldName); + } + + List vectorQueries = new ArrayList<>(); + vectorQueries.add(new VectorizedQuery(vector) + .setFields(recordDefinition.getField(options.getVectorFieldName() != null + ? options.getVectorFieldName() + : firstVectorFieldName).getEffectiveStorageName()) + .setKNearestNeighborsCount(options.getLimit())); + + return searchAndMapAsync(vectorQueries, options, + new GetRecordOptions(options.isIncludeVectors())); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index c051638d..61316d67 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -6,8 +6,6 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.microsoft.semantickernel.data.vectorsearch.VectorOperations; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; -import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; -import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizedSearchQuery; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; @@ -471,101 +469,94 @@ protected List getRecordsWithFilter(String collectionName, * * @param the record type * @param collectionName the collection name - * @param query the vectorized search query, containing the vector and search options + * @param vector the vector to search with + * @param options the search options * @param recordDefinition the record definition * @param mapper the mapper, responsible for mapping the result set to the record type. * @return the search results */ @Override public List> search(String collectionName, - VectorSearchQuery query, VectorStoreRecordDefinition recordDefinition, + List vector, VectorSearchOptions options, + VectorStoreRecordDefinition recordDefinition, VectorStoreRecordMapper mapper) { if (recordDefinition.getVectorFields().isEmpty()) { throw new SKException("No vector fields defined. Cannot perform vector search"); } - if (query instanceof VectorizedSearchQuery) { - VectorizedSearchQuery vectorizedSearchQuery = (VectorizedSearchQuery) query; - VectorSearchOptions options = query.getSearchOptions(); + VectorStoreRecordVectorField firstVectorField = recordDefinition.getVectorFields() + .get(0); + if (options == null) { + options = VectorSearchOptions.createDefault(firstVectorField.getName()); + } - VectorStoreRecordVectorField firstVectorField = recordDefinition.getVectorFields() - .get(0); - if (options == null) { - options = VectorSearchOptions.createDefault(firstVectorField.getName()); + VectorStoreRecordVectorField vectorField = options.getVectorFieldName() == null + ? firstVectorField + : (VectorStoreRecordVectorField) recordDefinition + .getField(options.getVectorFieldName()); + + String filter = SQLVectorStoreRecordCollectionSearchMapping + .buildFilter(options.getVectorSearchFilter(), recordDefinition); + List parameters = SQLVectorStoreRecordCollectionSearchMapping + .getFilterParameters(options.getVectorSearchFilter()); + + List records = getRecordsWithFilter(collectionName, recordDefinition, mapper, + new GetRecordOptions(true), filter, parameters); + List> results = new ArrayList<>(); + + DistanceFunction distanceFunction = vectorField.getDistanceFunction() == null + ? DistanceFunction.EUCLIDEAN_DISTANCE + : vectorField.getDistanceFunction(); + + for (Record record : records) { + List recordVector; + try { + String json = new ObjectMapper().writeValueAsString(record); + ArrayNode arrayNode = (ArrayNode) new ObjectMapper().readTree(json) + .get(vectorField.getEffectiveStorageName()); + + recordVector = Stream.iterate(0, i -> i + 1) + .limit(arrayNode.size()) + .map(i -> arrayNode.get(i).floatValue()) + .collect(Collectors.toList()); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); } - VectorStoreRecordVectorField vectorField = options.getVectorFieldName() == null - ? firstVectorField - : (VectorStoreRecordVectorField) recordDefinition - .getField(options.getVectorFieldName()); - - String filter = SQLVectorStoreRecordCollectionSearchMapping - .buildFilter(options.getVectorSearchFilter(), recordDefinition); - List parameters = SQLVectorStoreRecordCollectionSearchMapping - .getFilterParameters(options.getVectorSearchFilter()); - - List records = getRecordsWithFilter(collectionName, recordDefinition, mapper, - new GetRecordOptions(true), filter, parameters); - List> results = new ArrayList<>(); - - DistanceFunction distanceFunction = vectorField.getDistanceFunction() == null - ? DistanceFunction.EUCLIDEAN_DISTANCE - : vectorField.getDistanceFunction(); - - for (Record record : records) { - List vector; - try { - String json = new ObjectMapper().writeValueAsString(record); - ArrayNode arrayNode = (ArrayNode) new ObjectMapper().readTree(json) - .get(vectorField.getEffectiveStorageName()); - - vector = Stream.iterate(0, i -> i + 1) - .limit(arrayNode.size()) - .map(i -> arrayNode.get(i).floatValue()) - .collect(Collectors.toList()); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - - double score; - switch (distanceFunction) { - case COSINE_SIMILARITY: - score = VectorOperations.cosineSimilarity(vectorizedSearchQuery.getVector(), - vector); - break; - case COSINE_DISTANCE: - score = VectorOperations.cosineDistance(vectorizedSearchQuery.getVector(), - vector); - break; - case EUCLIDEAN_DISTANCE: - score = VectorOperations - .euclideanDistance(vectorizedSearchQuery.getVector(), vector); - break; - case DOT_PRODUCT: - score = VectorOperations.dot(vectorizedSearchQuery.getVector(), vector); - break; - default: - throw new SKException("Unsupported distance function"); - } - - results.add(new VectorSearchResult<>(record, score)); + double score; + switch (distanceFunction) { + case COSINE_SIMILARITY: + score = VectorOperations.cosineSimilarity(vector, recordVector); + break; + case COSINE_DISTANCE: + score = VectorOperations.cosineDistance(vector, recordVector); + break; + case EUCLIDEAN_DISTANCE: + score = VectorOperations + .euclideanDistance(vector, recordVector); + break; + case DOT_PRODUCT: + score = VectorOperations.dot(vector, recordVector); + break; + default: + throw new SKException("Unsupported distance function"); } - Comparator> comparator = Comparator - .comparingDouble(VectorSearchResult::getScore); - // Higher scores are better - if (distanceFunction == DistanceFunction.COSINE_SIMILARITY - || distanceFunction == DistanceFunction.DOT_PRODUCT) { - comparator = comparator.reversed(); - } - return results.stream() - .sorted(comparator) - .skip(options.getOffset()) - .limit(options.getLimit()) - .collect(Collectors.toList()); + results.add(new VectorSearchResult<>(record, score)); } - throw new SKException("Unsupported query type"); + Comparator> comparator = Comparator + .comparingDouble(VectorSearchResult::getScore); + // Higher scores are better + if (distanceFunction == DistanceFunction.COSINE_SIMILARITY + || distanceFunction == DistanceFunction.DOT_PRODUCT) { + comparator = comparator.reversed(); + } + return results.stream() + .sorted(comparator) + .skip(options.getOffset()) + .limit(options.getLimit()) + .collect(Collectors.toList()); } /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java index 2247264c..a62d495e 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -7,7 +7,6 @@ import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorizedSearch; -import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; @@ -298,14 +297,6 @@ public Mono prepareAsync() { .subscribeOn(Schedulers.boundedElastic()).then(); } - @Override - public Mono>> searchAsync(VectorSearchQuery query) { - return Mono.fromCallable( - () -> queryProvider.search(this.collectionName, query, recordDefinition, - vectorStoreRecordMapper)) - .subscribeOn(Schedulers.boundedElastic()); - } - /** * Vectorized search. This method searches for records that are similar to the given vector. * @@ -316,7 +307,11 @@ public Mono>> searchAsync(VectorSearchQuery quer @Override public Mono>> searchAsync(List vector, VectorSearchOptions vectorSearchOptions) { - return this.searchAsync(VectorSearchQuery.createQuery(vector, vectorSearchOptions)); + return Mono.fromCallable( + () -> queryProvider.search(this.collectionName, vector, vectorSearchOptions, + recordDefinition, + vectorStoreRecordMapper)) + .subscribeOn(Schedulers.boundedElastic()); } public static class Builder diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java index 8210d81c..2c7109f8 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java @@ -3,12 +3,12 @@ import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; -import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import java.sql.ResultSet; import java.util.List; @@ -138,13 +138,15 @@ void deleteRecords(String collectionName, List keys, * * @param the record type * @param collectionName the collection name - * @param query the vectorized search query, containing the vector and search options + * @param vector the vector to search with + * @param options the vector search options * @param recordDefinition the record definition * @param mapper the mapper, responsible for mapping the result set to the record type. * @return the search results */ List> search(String collectionName, - VectorSearchQuery query, + List vector, + VectorSearchOptions options, VectorStoreRecordDefinition recordDefinition, VectorStoreRecordMapper mapper); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index 46f27dd5..df4f0127 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -8,8 +8,6 @@ import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreRecordCollectionSearchMapping; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; -import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; -import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizedSearchQuery; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; @@ -329,94 +327,89 @@ public void upsertRecords(String collectionName, List records, * * @param the record type * @param collectionName the collection name - * @param query the vectorized search query, containing the vector and search options + * @param vector the vector to search with + * @param options the search options * @param recordDefinition the record definition * @param mapper the mapper, responsible for mapping the result set to the record type. * @return the search results */ @Override public List> search(String collectionName, - VectorSearchQuery query, VectorStoreRecordDefinition recordDefinition, + List vector, VectorSearchOptions options, + VectorStoreRecordDefinition recordDefinition, VectorStoreRecordMapper mapper) { if (recordDefinition.getVectorFields().isEmpty()) { throw new SKException("No vector fields defined. Cannot perform vector search"); } - if (query instanceof VectorizedSearchQuery) { - VectorizedSearchQuery vectorizedSearchQuery = (VectorizedSearchQuery) query; - VectorSearchOptions options = query.getSearchOptions(); + VectorStoreRecordVectorField firstVectorField = recordDefinition.getVectorFields() + .get(0); + if (options == null) { + options = VectorSearchOptions.createDefault(firstVectorField.getName()); + } - VectorStoreRecordVectorField firstVectorField = recordDefinition.getVectorFields() - .get(0); - if (options == null) { - options = VectorSearchOptions.createDefault(firstVectorField.getName()); - } + VectorStoreRecordVectorField vectorField = options.getVectorFieldName() == null + ? firstVectorField + : (VectorStoreRecordVectorField) recordDefinition + .getField(options.getVectorFieldName()); - VectorStoreRecordVectorField vectorField = options.getVectorFieldName() == null - ? firstVectorField - : (VectorStoreRecordVectorField) recordDefinition - .getField(options.getVectorFieldName()); - - PostgreSQLVectorIndexKind indexKind = PostgreSQLVectorIndexKind - .fromIndexKind(vectorField.getIndexKind()); - PostgreSQLVectorDistanceFunction distanceFunction = PostgreSQLVectorDistanceFunction - .fromDistanceFunction(vectorField.getDistanceFunction()); - - // If indexKind is not specified, there is no index associated to the vector field - // and pgvector performs exact nearest neighbor search. - // If indexKind is specified, a distance function is required. - if (indexKind != null && distanceFunction == null) { - throw new SKException( - "Distance function is required for vector field: " + vectorField.getName()); - } + PostgreSQLVectorIndexKind indexKind = PostgreSQLVectorIndexKind + .fromIndexKind(vectorField.getIndexKind()); + PostgreSQLVectorDistanceFunction distanceFunction = PostgreSQLVectorDistanceFunction + .fromDistanceFunction(vectorField.getDistanceFunction()); - String filter = SQLVectorStoreRecordCollectionSearchMapping.buildFilter( - options.getVectorSearchFilter(), - recordDefinition); - List parameters = SQLVectorStoreRecordCollectionSearchMapping - .getFilterParameters(options.getVectorSearchFilter()); - - String filterClause = filter.isEmpty() ? "" : "WHERE " + filter; - String searchQuery = formatQuery( - "SELECT %s, %s %s ?::vector AS score FROM %s %s ORDER BY score LIMIT ? OFFSET ?", - getQueryColumnsFromFields( - options.isIncludeVectors() ? recordDefinition.getAllFields() - : recordDefinition.getNonVectorFields()), - validateSQLidentifier(vectorField.getEffectiveStorageName()), - distanceFunction == null ? PostgreSQLVectorDistanceFunction.L2.getOperator() - : distanceFunction.getOperator(), - getCollectionTableName(collectionName), - filterClause); + // If indexKind is not specified, there is no index associated to the vector field + // and pgvector performs exact nearest neighbor search. + // If indexKind is specified, a distance function is required. + if (indexKind != null && distanceFunction == null) { + throw new SKException( + "Distance function is required for vector field: " + vectorField.getName()); + } - try (Connection connection = dataSource.getConnection(); - PreparedStatement statement = connection.prepareStatement(searchQuery)) { - int parameterIndex = 1; + String filter = SQLVectorStoreRecordCollectionSearchMapping.buildFilter( + options.getVectorSearchFilter(), + recordDefinition); + List parameters = SQLVectorStoreRecordCollectionSearchMapping + .getFilterParameters(options.getVectorSearchFilter()); + + String filterClause = filter.isEmpty() ? "" : "WHERE " + filter; + String searchQuery = formatQuery( + "SELECT %s, %s %s ?::vector AS score FROM %s %s ORDER BY score LIMIT ? OFFSET ?", + getQueryColumnsFromFields( + options.isIncludeVectors() ? recordDefinition.getAllFields() + : recordDefinition.getNonVectorFields()), + validateSQLidentifier(vectorField.getEffectiveStorageName()), + distanceFunction == null ? PostgreSQLVectorDistanceFunction.L2.getOperator() + : distanceFunction.getOperator(), + getCollectionTableName(collectionName), + filterClause); - statement.setString(parameterIndex++, - objectMapper.writeValueAsString(vectorizedSearchQuery.getVector())); - for (Object parameter : parameters) { - statement.setObject(parameterIndex++, parameter); - } - statement.setInt(parameterIndex++, options.getLimit()); - statement.setInt(parameterIndex, options.getOffset()); + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(searchQuery)) { + int parameterIndex = 1; - List> records = new ArrayList<>(); - ResultSet resultSet = statement.executeQuery(); + statement.setString(parameterIndex++, + objectMapper.writeValueAsString(vector)); + for (Object parameter : parameters) { + statement.setObject(parameterIndex++, parameter); + } + statement.setInt(parameterIndex++, options.getLimit()); + statement.setInt(parameterIndex, options.getOffset()); - while (resultSet.next()) { - records.add(new VectorSearchResult<>( - mapper.mapStorageModelToRecord(resultSet, - new GetRecordOptions(options.isIncludeVectors())), - resultSet.getDouble("score"))); - } + List> records = new ArrayList<>(); + ResultSet resultSet = statement.executeQuery(); - return records; - } catch (SQLException | JsonProcessingException e) { - throw new SKException("Failed to search records", e); + while (resultSet.next()) { + records.add(new VectorSearchResult<>( + mapper.mapStorageModelToRecord(resultSet, + new GetRecordOptions(options.isIncludeVectors())), + resultSet.getDouble("score"))); } - } - throw new SKException("Unsupported query type"); + return records; + } catch (SQLException | JsonProcessingException e) { + throw new SKException("Failed to search records", e); + } } public static class Builder diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java index ae2c7d23..80a39492 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java @@ -4,8 +4,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorizedSearch; -import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; -import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizedSearchQuery; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; @@ -380,67 +378,55 @@ public Mono deleteBatchAsync(List strings, DeleteRecordOptions opt .then(); } + /** + * Vectorized search. This method searches for records that are similar to the given vector. + * + * @param vector The vector to search with. + * @param options The options to use for the search. + * @return A list of search results. + */ @Override - public Mono>> searchAsync(VectorSearchQuery query) { + public Mono>> searchAsync(List vector, + VectorSearchOptions options) { if (recordDefinition.getVectorFields().isEmpty()) { return Mono .error(new SKException("No vector fields defined. Cannot perform vector search")); } return createCollectionIfNotExistsAsync().flatMap(collection -> Mono.fromCallable(() -> { - if (query instanceof VectorizedSearchQuery) { - VectorSearchOptions options = query.getSearchOptions(); - - Pair ftSearchParams = RedisVectorStoreCollectionSearchMapping - .buildQuery((VectorizedSearchQuery) query, recordDefinition, - RedisStorageType.HASH_SET); - - SearchResult searchResult = client.ftSearch(collectionName, - ftSearchParams.getLeft(), ftSearchParams.getRight()); - - return searchResult.getDocuments().stream() - .map(doc -> { - String key = removeKeyPrefixIfNeeded(doc.getId(), collectionName); - double score = 0; - - // Convert from Map to Map - Map storage = new HashMap<>(); - for (Map.Entry entry : doc.getProperties()) { - if (entry.getValue() instanceof byte[]) { - storage.put(stringToBytes(entry.getKey()), - (byte[]) entry.getValue()); - } else if (entry.getKey().equals( - RedisVectorStoreCollectionSearchMapping.VECTOR_SCORE_FIELD)) { - // Score is stored as a string in one of the fields - score = Double.parseDouble((String) entry.getValue()); - } + Pair ftSearchParams = RedisVectorStoreCollectionSearchMapping + .buildQuery(vector, options, recordDefinition, RedisStorageType.HASH_SET); + + SearchResult searchResult = client.ftSearch(collectionName, + ftSearchParams.getLeft(), ftSearchParams.getRight()); + + return searchResult.getDocuments().stream() + .map(doc -> { + String key = removeKeyPrefixIfNeeded(doc.getId(), collectionName); + double score = 0; + + // Convert from Map to Map + Map storage = new HashMap<>(); + for (Map.Entry entry : doc.getProperties()) { + if (entry.getValue() instanceof byte[]) { + storage.put(stringToBytes(entry.getKey()), + (byte[]) entry.getValue()); + } else if (entry.getKey().equals( + RedisVectorStoreCollectionSearchMapping.VECTOR_SCORE_FIELD)) { + // Score is stored as a string in one of the fields + score = Double.parseDouble((String) entry.getValue()); } + } - Record record = this.vectorStoreRecordMapper - .mapStorageModelToRecord( - new AbstractMap.SimpleEntry<>(key, storage), - new GetRecordOptions( - options != null && options.isIncludeVectors())); - - return new VectorSearchResult<>(record, score); - }) - .collect(Collectors.toList()); - } + Record record = this.vectorStoreRecordMapper + .mapStorageModelToRecord( + new AbstractMap.SimpleEntry<>(key, storage), + new GetRecordOptions( + options != null && options.isIncludeVectors())); - throw new SKException("Unsupported query type"); + return new VectorSearchResult<>(record, score); + }) + .collect(Collectors.toList()); }).subscribeOn(Schedulers.boundedElastic())); } - - /** - * Vectorized search. This method searches for records that are similar to the given vector. - * - * @param vector The vector to search with. - * @param options The options to use for the search. - * @return A list of search results. - */ - @Override - public Mono>> searchAsync(List vector, - VectorSearchOptions options) { - return this.searchAsync(VectorSearchQuery.createQuery(vector, options)); - } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java index 994e12ea..d2df1766 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java @@ -7,8 +7,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorizedSearch; -import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; -import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizedSearchQuery; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; @@ -397,70 +395,52 @@ public Mono deleteBatchAsync(List strings, DeleteRecordOptions opt } /** - * Vector search. This method searches for records that are similar to the given vector using the index defined when creating the collection. + * Vectorized search. This method searches for records that are similar to the given vector. * - * @param query The search query. + * @param vector The vector to search with. + * @param options The options to use for the search. * @return A list of search results. */ @Override - public Mono>> searchAsync(VectorSearchQuery query) { + public Mono>> searchAsync(List vector, + VectorSearchOptions options) { if (recordDefinition.getVectorFields().isEmpty()) { return Mono .error(new SKException("No vector fields defined. Cannot perform vector search")); } return createCollectionIfNotExistsAsync().flatMap(collection -> Mono.fromCallable(() -> { - if (query instanceof VectorizedSearchQuery) { - VectorSearchOptions options = query.getSearchOptions(); - - Pair ftSearchParams = RedisVectorStoreCollectionSearchMapping - .buildQuery((VectorizedSearchQuery) query, recordDefinition, - RedisStorageType.JSON); - - SearchResult searchResult = client.ftSearch(collectionName, - ftSearchParams.getLeft(), ftSearchParams.getRight()); - - return searchResult.getDocuments().stream() - .map(doc -> { - Map properties = new HashMap<>(); - for (Map.Entry entry : doc.getProperties()) { - properties.put(entry.getKey(), entry.getValue()); - } - - String key = removeKeyPrefixIfNeeded(doc.getId(), collectionName); - String value = (String) properties.get("$"); - double score = Double.parseDouble((String) properties - .get(RedisVectorStoreCollectionSearchMapping.VECTOR_SCORE_FIELD)); - - try { - JsonNode jsonNode = objectMapper.readTree(value); - Record record = this.vectorStoreRecordMapper - .mapStorageModelToRecord(new SimpleEntry<>(key, jsonNode), - new GetRecordOptions( - options != null && options.isIncludeVectors())); - - return new VectorSearchResult<>(record, score); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }) - .collect(Collectors.toList()); - } - - throw new SKException("Unsupported query type"); + Pair ftSearchParams = RedisVectorStoreCollectionSearchMapping + .buildQuery(vector, options, recordDefinition, RedisStorageType.JSON); + + SearchResult searchResult = client.ftSearch(collectionName, + ftSearchParams.getLeft(), ftSearchParams.getRight()); + + return searchResult.getDocuments().stream() + .map(doc -> { + Map properties = new HashMap<>(); + for (Map.Entry entry : doc.getProperties()) { + properties.put(entry.getKey(), entry.getValue()); + } + + String key = removeKeyPrefixIfNeeded(doc.getId(), collectionName); + String value = (String) properties.get("$"); + double score = Double.parseDouble((String) properties + .get(RedisVectorStoreCollectionSearchMapping.VECTOR_SCORE_FIELD)); + + try { + JsonNode jsonNode = objectMapper.readTree(value); + Record record = this.vectorStoreRecordMapper + .mapStorageModelToRecord(new SimpleEntry<>(key, jsonNode), + new GetRecordOptions( + options != null && options.isIncludeVectors())); + + return new VectorSearchResult<>(record, score); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); }).subscribeOn(Schedulers.boundedElastic())); } - - /** - * Vectorized search. This method searches for records that are similar to the given vector. - * - * @param vector The vector to search with. - * @param options The options to use for the search. - * @return A list of search results. - */ - @Override - public Mono>> searchAsync(List vector, - VectorSearchOptions options) { - return this.searchAsync(VectorSearchQuery.createQuery(vector, options)); - } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java index 2fb88789..41d1b790 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java @@ -3,7 +3,6 @@ import com.microsoft.semantickernel.connectors.data.redis.filter.RedisEqualToFilterClause; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; -import com.microsoft.semantickernel.data.vectorsearch.queries.VectorizedSearchQuery; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; @@ -22,10 +21,10 @@ public class RedisVectorStoreCollectionSearchMapping { static final String VECTOR_SCORE_FIELD = "vector_score"; - public static Pair buildQuery(VectorizedSearchQuery query, + public static Pair buildQuery(List vector, + VectorSearchOptions options, VectorStoreRecordDefinition recordDefinition, RedisStorageType storageType) { - VectorSearchOptions options = query.getSearchOptions(); VectorStoreRecordVectorField firstVectorField = recordDefinition.getVectorFields().get(0); if (options == null) { options = VectorSearchOptions.createDefault(firstVectorField.getName()); @@ -43,7 +42,7 @@ public static Pair buildQuery(VectorizedSearchQuery quer FTSearchParams searchParams = new FTSearchParams() .addParam("K", options.getLimit() + options.getOffset()) - .addParam("BLOB", convertListToByteArray(query.getVector())) + .addParam("BLOB", convertListToByteArray(vector)) .limit(options.getOffset(), options.getLimit()) .sortBy(VECTOR_SCORE_FIELD, SortingOrder.ASC) .dialect(2); @@ -57,8 +56,8 @@ public static Pair buildQuery(VectorizedSearchQuery quer searchParams.returnField(dataField.getEffectiveStorageName(), false); } if (options.isIncludeVectors()) { - for (VectorStoreRecordVectorField vector : recordDefinition.getVectorFields()) { - searchParams.returnField(vector.getEffectiveStorageName(), false); + for (VectorStoreRecordVectorField v : recordDefinition.getVectorFields()) { + searchParams.returnField(v.getEffectiveStorageName(), false); } } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearch.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearch.java deleted file mode 100644 index 288b7768..00000000 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearch.java +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorsearch; - -import com.microsoft.semantickernel.data.vectorsearch.queries.VectorSearchQuery; -import reactor.core.publisher.Mono; - -import java.util.List; - -public interface VectorSearch { - - Mono>> searchAsync(VectorSearchQuery query); -} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableSearch.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java similarity index 89% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableSearch.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java index 37847b75..ab7c779e 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableSearch.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java @@ -6,7 +6,7 @@ import java.util.List; -public interface VectorizableSearch extends VectorSearch { +public interface VectorizableTextSearch { /** * Vectorizable text search. This method searches for records that are similar to the given text. * diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java index 51ab175a..1c9274a9 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java @@ -6,7 +6,7 @@ import java.util.List; -public interface VectorizedSearch extends VectorSearch { +public interface VectorizedSearch { /** * Vectorized search. This method searches for records that are similar to the given vector. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java deleted file mode 100644 index 694950d1..00000000 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQuery.java +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorsearch.queries; - -import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; - -import java.util.List; - -public class VectorSearchQuery { - private final VectorSearchQueryType queryType; - private final VectorSearchOptions searchOptions; - - public VectorSearchQuery(VectorSearchQueryType queryType, VectorSearchOptions searchOptions) { - this.queryType = queryType; - this.searchOptions = searchOptions; - } - - public VectorSearchQueryType getQueryType() { - return queryType; - } - - public VectorSearchOptions getSearchOptions() { - return searchOptions; - } - - public static VectorizedSearchQuery createQuery(List vector, - VectorSearchOptions options) { - return new VectorizedSearchQuery(vector, options); - } - - public static VectorizableTextSearchQuery createQuery(String searchText, - VectorSearchOptions options) { - return new VectorizableTextSearchQuery(searchText, options); - } -} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQueryType.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQueryType.java deleted file mode 100644 index 3a64ad62..00000000 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorSearchQueryType.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorsearch.queries; - -public enum VectorSearchQueryType { - - VECTORIZED_SEARCH_QUERY("VectorizedSearchQuery"), VECTORIZABLE_TEXT_SEARCH_QUERY( - "VectorizableTextSearchQuery"), HYBRID_TEXT_VECTORIZED_SEARCH_QUERY( - "HybridTextVectorizedSearchQuery"), HYBRID_VECTORIZABLE_TEXT_SEARCH_QUERY( - "HybridVectorizableTextSearchQuery"); - - private final String value; - - VectorSearchQueryType(String value) { - this.value = value; - } - - public String getValue() { - return value; - } -} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizableTextSearchQuery.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizableTextSearchQuery.java deleted file mode 100644 index 1df9b6ec..00000000 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizableTextSearchQuery.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorsearch.queries; - -import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; - -import javax.annotation.Nullable; - -public class VectorizableTextSearchQuery extends VectorSearchQuery { - - private final String queryText; - - @Nullable - private final VectorSearchOptions searchOptions; - - public VectorizableTextSearchQuery(String queryText, - @Nullable VectorSearchOptions searchOptions) { - super(VectorSearchQueryType.VECTORIZABLE_TEXT_SEARCH_QUERY, searchOptions); - this.queryText = queryText; - this.searchOptions = searchOptions; - } - - public String getQueryText() { - return queryText; - } - - @Nullable - public VectorSearchOptions getSearchOptions() { - return searchOptions; - } -} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java deleted file mode 100644 index 2a33c07d..00000000 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/queries/VectorizedSearchQuery.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorsearch.queries; - -import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; - -import javax.annotation.Nullable; -import java.util.Collections; -import java.util.List; - -public class VectorizedSearchQuery extends VectorSearchQuery { - - private final List vector; - @Nullable - private final VectorSearchOptions searchOptions; - - public VectorizedSearchQuery(List vector, VectorSearchOptions searchOptions) { - super(VectorSearchQueryType.VECTORIZED_SEARCH_QUERY, null); - - this.vector = Collections.unmodifiableList(vector); - this.searchOptions = searchOptions; - } - - public List getVector() { - return vector; - } - - public VectorSearchOptions getSearchOptions() { - return searchOptions; - } -} From 7e4b348658c5c2f7a583b188d9af190bb003e3a6 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 24 Sep 2024 17:50:51 -0700 Subject: [PATCH 056/277] Add VolatileVectorStoreRecordCollection search --- .../JDBCVectorStoreRecordCollectionTest.java | 4 +- ...ashSetVectorStoreRecordCollectionTest.java | 2 +- ...isJsonVectorStoreRecordCollectionTest.java | 2 +- .../jdbc/JDBCVectorStoreQueryProvider.java | 52 +--------- ...ileVectorStoreCollectionSearchMapping.java | 48 +++++++++ .../VolatileVectorStoreRecordCollection.java | 67 ++++++++++++- ...ileVectorStoreRecordCollectionOptions.java | 34 ++++++- .../data/filter/EqualToFilterClause.java | 13 ++- .../data/vectorsearch/VectorOperations.java | 97 ++++++++++++++++-- .../VectorStoreRecordCollection.java | 3 +- .../microsoft/semantickernel/data/Hotel.java | 64 ++++++++++-- ...latileVectorStoreRecordCollectionTest.java | 99 +++++++++++++++++-- 12 files changed, 399 insertions(+), 86 deletions(-) create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java index 28a45e27..874a469c 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -100,9 +100,7 @@ public void buildRecordCollection(QueryProvider provider) { } private List getHotels() { - ArrayList embeddings = new ArrayList<>(); - - return List.of( + return Arrays.asList( new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f),null, null, null, 4.0), new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f),null, null, null, 4.0), new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f),null, null, null, 5.0), diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index d6d270e5..37d9b383 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -132,7 +132,7 @@ private static RedisHashSetVectorStoreRecordCollection createCollection(@ } private static List getHotels() { - return List.of( + return Arrays.asList( new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f),null, null, 4.0), new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f),null, null, 4.0), new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f),null, null, 5.0), diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index 6eab5a1d..0203f7ce 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -132,7 +132,7 @@ private static RedisJsonVectorStoreRecordCollection createCollection(@Non } private static List getHotels() { - return List.of( + return Arrays.asList( new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f),null, null, 4.0), new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f),null, null, 4.0), new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f),null, null, 5.0), diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index 61316d67..bb8e8d2a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -502,61 +502,13 @@ public List> search(String collectionName, List records = getRecordsWithFilter(collectionName, recordDefinition, mapper, new GetRecordOptions(true), filter, parameters); - List> results = new ArrayList<>(); DistanceFunction distanceFunction = vectorField.getDistanceFunction() == null ? DistanceFunction.EUCLIDEAN_DISTANCE : vectorField.getDistanceFunction(); - for (Record record : records) { - List recordVector; - try { - String json = new ObjectMapper().writeValueAsString(record); - ArrayNode arrayNode = (ArrayNode) new ObjectMapper().readTree(json) - .get(vectorField.getEffectiveStorageName()); - - recordVector = Stream.iterate(0, i -> i + 1) - .limit(arrayNode.size()) - .map(i -> arrayNode.get(i).floatValue()) - .collect(Collectors.toList()); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - - double score; - switch (distanceFunction) { - case COSINE_SIMILARITY: - score = VectorOperations.cosineSimilarity(vector, recordVector); - break; - case COSINE_DISTANCE: - score = VectorOperations.cosineDistance(vector, recordVector); - break; - case EUCLIDEAN_DISTANCE: - score = VectorOperations - .euclideanDistance(vector, recordVector); - break; - case DOT_PRODUCT: - score = VectorOperations.dot(vector, recordVector); - break; - default: - throw new SKException("Unsupported distance function"); - } - - results.add(new VectorSearchResult<>(record, score)); - } - - Comparator> comparator = Comparator - .comparingDouble(VectorSearchResult::getScore); - // Higher scores are better - if (distanceFunction == DistanceFunction.COSINE_SIMILARITY - || distanceFunction == DistanceFunction.DOT_PRODUCT) { - comparator = comparator.reversed(); - } - return results.stream() - .sorted(comparator) - .skip(options.getOffset()) - .limit(options.getLimit()) - .collect(Collectors.toList()); + return VectorOperations.exactSimilaritySearch(records, vector, vectorField, + distanceFunction, options); } /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java new file mode 100644 index 00000000..79a1326d --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.microsoft.semantickernel.data.filter.EqualToFilterClause; +import com.microsoft.semantickernel.data.filter.FilterClause; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.exceptions.SKException; + +import java.util.List; +import java.util.stream.Collectors; + +public class VolatileVectorStoreCollectionSearchMapping { + + public static List filterRecords(List records, + VectorSearchFilter filter, + VectorStoreRecordDefinition recordDefinition, ObjectMapper objectMapper) { + if (filter == null || filter.getFilterClauses().isEmpty()) { + return records; + } + + return records.stream().filter( + record -> { + JsonNode recordNode = objectMapper.valueToTree(record); + + for (FilterClause filterClause : filter.getFilterClauses()) { + if (filterClause instanceof EqualToFilterClause) { + EqualToFilterClause equalToFilterClause = (EqualToFilterClause) filterClause; + VectorStoreRecordField field = recordDefinition + .getField(equalToFilterClause.getFieldName()); + + Object value = objectMapper.convertValue( + recordNode.get(field.getEffectiveStorageName()), field.getFieldType()); + if (!equalToFilterClause.getValue().equals(value)) { + return false; + } + } else { + throw new SKException(String.format("Unsupported filter clause type '%s'.", + filterClause.getClass().getSimpleName())); + } + } + return true; + }).collect(Collectors.toList()); + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java index d8b23d39..ec82700d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java @@ -1,21 +1,36 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.microsoft.semantickernel.data.filter.FilterClause; +import com.microsoft.semantickernel.data.vectorsearch.VectorOperations; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import com.microsoft.semantickernel.exceptions.SKException; + +import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; +import java.util.stream.Stream; + import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; public class VolatileVectorStoreRecordCollection implements VectorStoreRecordCollection { @@ -33,7 +48,6 @@ public VolatileVectorStoreRecordCollection(String collectionName, this.collectionName = collectionName; this.options = options; this.collections = new ConcurrentHashMap<>(); - this.objectMapper = new ObjectMapper(); if (options.getRecordDefinition() != null) { this.recordDefinition = options.getRecordDefinition(); @@ -42,6 +56,12 @@ public VolatileVectorStoreRecordCollection(String collectionName, .fromRecordClass(this.options.getRecordClass()); } + if (options.getObjectMapper() == null) { + this.objectMapper = new ObjectMapper(); + } else { + this.objectMapper = options.getObjectMapper(); + } + // Validate the key type VectorStoreRecordDefinition.validateSupportedTypes( Collections.singletonList(recordDefinition.getKeyField()), @@ -222,4 +242,49 @@ private Map getCollection() { } return (Map) collections.get(collectionName); } + + private List arrayNodeToFloatList(ArrayNode arrayNode) { + return Stream.iterate(0, i -> i + 1) + .limit(arrayNode.size()) + .map(i -> arrayNode.get(i).floatValue()) + .collect(Collectors.toList()); + } + + /** + * Vectorized search. This method searches for records that are similar to the given vector. + * + * @param vector The vector to search with. + * @param options The options to use for the search. + * @return A list of search results. + */ + @Override + public Mono>> searchAsync(List vector, + final VectorSearchOptions options) { + if (recordDefinition.getVectorFields().isEmpty()) { + throw new SKException("No vector fields defined. Cannot perform vector search"); + } + + return Mono.fromCallable(() -> { + VectorStoreRecordVectorField firstVectorField = recordDefinition.getVectorFields() + .get(0); + VectorSearchOptions effectiveOptions = options == null + ? VectorSearchOptions.createDefault(firstVectorField.getName()) + : options; + + VectorStoreRecordVectorField vectorField = effectiveOptions.getVectorFieldName() == null + ? firstVectorField + : (VectorStoreRecordVectorField) recordDefinition + .getField(effectiveOptions.getVectorFieldName()); + + DistanceFunction distanceFunction = vectorField.getDistanceFunction() == null + ? DistanceFunction.EUCLIDEAN_DISTANCE + : vectorField.getDistanceFunction(); + + List records = VolatileVectorStoreCollectionSearchMapping.filterRecords( + new ArrayList<>(getCollection().values()), effectiveOptions.getVectorSearchFilter(), + recordDefinition, objectMapper); + return VectorOperations.exactSimilaritySearch(records, vector, vectorField, + distanceFunction, effectiveOptions); + }).subscribeOn(Schedulers.boundedElastic()); + } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java index 64e95db6..592e7653 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java @@ -1,8 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data; +import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -12,6 +14,8 @@ public class VolatileVectorStoreRecordCollectionOptions private final Class recordClass; @Nullable private final VectorStoreRecordDefinition recordDefinition; + @Nullable + private final ObjectMapper objectMapper; /** * Creates a new instance of the Volatile vector store record collection options. @@ -19,10 +23,12 @@ public class VolatileVectorStoreRecordCollectionOptions * @param recordClass The record class. * @param recordDefinition The record definition. */ + @SuppressFBWarnings("EI_EXPOSE_REP2") // ObjectMapper only has package visibility public VolatileVectorStoreRecordCollectionOptions(@Nonnull Class recordClass, - @Nullable VectorStoreRecordDefinition recordDefinition) { + @Nullable VectorStoreRecordDefinition recordDefinition, ObjectMapper objectMapper) { this.recordClass = recordClass; this.recordDefinition = recordDefinition; + this.objectMapper = objectMapper; } /** @@ -54,6 +60,15 @@ public Class getRecordClass() { return recordClass; } + /** + * Gets the object mapper. + * + * @return the object mapper + */ + ObjectMapper getObjectMapper() { + return objectMapper; + } + /** * Gets the record definition. * @@ -73,6 +88,8 @@ public static class Builder { private Class recordClass; @Nullable private VectorStoreRecordDefinition recordDefinition; + @Nullable + private ObjectMapper objectMapper; /** * Sets the record class. @@ -96,6 +113,18 @@ public Builder withRecordDefinition(VectorStoreRecordDefinition recordDe return this; } + /** + * Sets the object mapper. + * + * @param objectMapper the object mapper + * @return the builder + */ + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withObjectMapper(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + return this; + } + /** * Builds the options. * @@ -106,7 +135,8 @@ public VolatileVectorStoreRecordCollectionOptions build() { throw new IllegalArgumentException("recordClass is required"); } - return new VolatileVectorStoreRecordCollectionOptions<>(recordClass, recordDefinition); + return new VolatileVectorStoreRecordCollectionOptions<>(recordClass, recordDefinition, + objectMapper); } } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java index bbf8fb77..3590b252 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.filter; -public abstract class EqualToFilterClause implements FilterClause { +public class EqualToFilterClause implements FilterClause { private final String fieldName; private final Object value; @@ -28,4 +28,15 @@ public String getFieldName() { public Object getValue() { return value; } + + /** + * Gets the filter string. + * + * @return The filter. + */ + @Override + public String getFilter() { + throw new UnsupportedOperationException(String.format( + "Not implemented. Use one of %s derived classes.", this.getClass().getSimpleName())); + } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java index d1e36fab..485c16c3 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java @@ -1,10 +1,21 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.vectorsearch; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import com.microsoft.semantickernel.exceptions.SKException; + import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; +import java.util.stream.Stream; public final class VectorOperations { @@ -21,7 +32,7 @@ public static float cosineSimilarity(@Nonnull List x, @Nonnull List x, @Nonnull List x, @Nonnull List x, @Nonnull List divide(@Nonnull List vector, float divisor) { Objects.requireNonNull(vector); if (Float.isNaN(divisor)) { - throw new IllegalArgumentException("Divisor cannot be NaN"); + throw new SKException("Divisor cannot be NaN"); } if (divisor == 0f) { - throw new IllegalArgumentException("Divisor cannot be zero"); + throw new SKException("Divisor cannot be zero"); } return vector.stream().map(x -> x / divisor).collect(Collectors.toList()); @@ -103,7 +114,7 @@ public static float dot(@Nonnull List x, @Nonnull List y) { Objects.requireNonNull(y); if (x.size() != y.size()) { - throw new IllegalArgumentException("Vectors lengths must be equal"); + throw new SKException("Vectors lengths must be equal"); } float result = 0; @@ -135,10 +146,10 @@ public static float euclideanLength(@Nonnull List vector) { public static List multiply(@Nonnull List vector, float multiplier) { Objects.requireNonNull(vector); if (Float.isNaN(multiplier)) { - throw new IllegalArgumentException("Multiplier cannot be NaN"); + throw new SKException("Multiplier cannot be NaN"); } if (Float.isInfinite(multiplier)) { - throw new IllegalArgumentException("Multiplier cannot be infinite"); + throw new SKException("Multiplier cannot be infinite"); } return vector.stream().map(x -> x * multiplier).collect(Collectors.toList()); @@ -154,4 +165,74 @@ public static List normalize(@Nonnull List vector) { Objects.requireNonNull(vector); return divide(vector, euclideanLength(vector)); } + + /** + * Performs an exact similarity search on a list of records using a vector field. + * + * @param records The records to search. + * @param vector The vector to search for. + * @param vectorField The vector field to use for the search. + * @param distanceFunction The distance function to use for the search. + * @param options The search options. + * @param The type of the records. + * @return The search results. + */ + public static List> exactSimilaritySearch( + List records, + List vector, + VectorStoreRecordVectorField vectorField, + DistanceFunction distanceFunction, + VectorSearchOptions options) { + List> results = new ArrayList<>(); + + for (Record record : records) { + List recordVector; + try { + String json = new ObjectMapper().writeValueAsString(record); + ArrayNode arrayNode = (ArrayNode) new ObjectMapper().readTree(json) + .get(vectorField.getEffectiveStorageName()); + + recordVector = Stream.iterate(0, i -> i + 1) + .limit(arrayNode.size()) + .map(i -> arrayNode.get(i).floatValue()) + .collect(Collectors.toList()); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + double score; + switch (distanceFunction) { + case COSINE_SIMILARITY: + score = cosineSimilarity(vector, recordVector); + break; + case COSINE_DISTANCE: + score = cosineDistance(vector, recordVector); + break; + case EUCLIDEAN_DISTANCE: + score = euclideanDistance(vector, recordVector); + break; + case DOT_PRODUCT: + score = dot(vector, recordVector); + break; + default: + throw new SKException("Unsupported distance function"); + } + + results.add(new VectorSearchResult<>(record, score)); + } + + Comparator> comparator = Comparator + .comparingDouble(VectorSearchResult::getScore); + // Higher scores are better + if (distanceFunction == DistanceFunction.COSINE_SIMILARITY + || distanceFunction == DistanceFunction.DOT_PRODUCT) { + comparator = comparator.reversed(); + } + + return results.stream() + .sorted(comparator) + .skip(options.getOffset()) + .limit(options.getLimit()) + .collect(Collectors.toList()); + } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java index 7efe19bb..38655fbc 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java @@ -1,13 +1,14 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.vectorstorage; +import com.microsoft.semantickernel.data.vectorsearch.VectorizedSearch; import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import java.util.List; import reactor.core.publisher.Mono; -public interface VectorStoreRecordCollection { +public interface VectorStoreRecordCollection extends VectorizedSearch { /** * Gets the name of the collection. diff --git a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java index 9da92923..5255407d 100644 --- a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java +++ b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; @@ -10,28 +12,58 @@ public class Hotel { @VectorStoreRecordKeyAttribute private final String id; - @VectorStoreRecordDataAttribute + + @VectorStoreRecordDataAttribute(isFilterable = true) private final String name; + @VectorStoreRecordDataAttribute private final int code; + + @JsonProperty("summary") @VectorStoreRecordDataAttribute() private final String description; - @VectorStoreRecordVectorAttribute(dimensions = 3) - private final List descriptionEmbedding; + + @JsonProperty("summaryEmbedding1") + @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "euclidean") + private final List euclidean; + + @JsonProperty("summaryEmbedding2") + @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "cosineDistance") + private final List cosineDistance; + + @JsonProperty("summaryEmbedding3") + @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "cosineSimilarity") + private final List cosineSimilarity; + + @JsonProperty("summaryEmbedding4") + @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "dotProduct") + private final List dotProduct; @VectorStoreRecordDataAttribute - private final double rating; + private double rating; public Hotel() { - this(null, null, 0, null, null, 0.0); + this(null, null, 0, null, null, null, null, null, 0.0); } - public Hotel(String id, String name, int code, String description, - List descriptionEmbedding, double rating) { + @JsonCreator + public Hotel( + @JsonProperty("id") String id, + @JsonProperty("name") String name, + @JsonProperty("code") int code, + @JsonProperty("summary") String description, + @JsonProperty("summaryEmbedding1") List euclidean, + @JsonProperty("summaryEmbedding2") List cosineDistance, + @JsonProperty("summaryEmbedding2") List cosineSimilarity, + @JsonProperty("summaryEmbedding3") List dotProduct, + @JsonProperty("rating") double rating) { this.id = id; this.name = name; this.code = code; this.description = description; - this.descriptionEmbedding = descriptionEmbedding; + this.euclidean = euclidean; + this.cosineDistance = euclidean; + this.cosineSimilarity = euclidean; + this.dotProduct = euclidean; this.rating = rating; } @@ -51,11 +83,23 @@ public String getDescription() { return description; } - public List getDescriptionEmbedding() { - return descriptionEmbedding; + public List getEuclidean() { + return euclidean; + } + + public List getCosineDistance() { + return cosineDistance; + } + + public List getDotProduct() { + return dotProduct; } public double getRating() { return rating; } + + public void setRating(double rating) { + this.rating = rating; + } } diff --git a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java index 915b2166..d2084967 100644 --- a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java +++ b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java @@ -6,12 +6,24 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; + +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollection; +import com.microsoft.semantickernel.connectors.data.jdbc.filter.SQLEqualToFilterClause; +import com.microsoft.semantickernel.data.filter.EqualToFilterClause; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; public class VolatileVectorStoreRecordCollectionTest { @@ -33,20 +45,35 @@ public void clearCollection() { recordCollection.createCollectionAsync().block(); } - private List getHotels() { + private static List getHotels() { return Arrays.asList( - new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(1.0f, 2.0f, 3.0f), + new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", + Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), null, null, null, + 4.0), + new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", + Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), null, null, null, 4.0), - new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(1.0f, 2.0f, 3.0f), - 3.0), - new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(1.0f, 2.0f, 3.0f), + new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", + Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), null, null, null, 5.0), - new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(1.0f, 2.0f, 3.0f), + new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", + Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), null, null, null, 4.0), - new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(1.0f, 2.0f, 3.0f), - 5.0)); + new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", + Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), null, null, null, + 4.0)); } + /** + * Search embeddings similar to the third hotel embeddings. + * In order of similarity: + * 1. Hotel 3 + * 2. Hotel 1 + * 3. Hotel 4 + */ + private static final List SEARCH_EMBEDDINGS = Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, + -0.8f, 1.1f, -2.2f, 8.2f); + @Test public void createAndDeleteCollectionAsync() { assertEquals(true, recordCollection.collectionExistsAsync().block()); @@ -126,4 +153,60 @@ public void deleteBatchAsync() { assertNull(recordCollection.getAsync(key, null).block()); } } + + @ParameterizedTest + @EnumSource(DistanceFunction.class) + public void exactSearch(DistanceFunction distanceFunction) { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions options = VectorSearchOptions.builder() + .withVectorFieldName(distanceFunction.getValue()) + .withLimit(3) + .build(); + + // Embeddings similar to the third hotel + List> results = recordCollection + .searchAsync(SEARCH_EMBEDDINGS, options).block(); + assertNotNull(results); + assertEquals(3, results.size()); + // The third hotel should be the most similar + assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + + options = VectorSearchOptions.builder() + .withVectorFieldName(distanceFunction.getValue()) + .withOffset(1) + .withLimit(-100) + .build(); + + // Skip the first result + results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, options).block(); + assertNotNull(results); + assertEquals(1, results.size()); + // The first hotel should be the most similar + assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); + } + + @ParameterizedTest + @EnumSource(DistanceFunction.class) + public void searchWithFilter(DistanceFunction distanceFunction) { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions options = VectorSearchOptions.builder() + .withVectorFieldName(distanceFunction.getValue()) + .withLimit(3) + .withVectorSearchFilter( + VectorSearchFilter.builder() + .withEqualToFilterClause(new EqualToFilterClause("rating", 4.0)).build()) + .build(); + + // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned + List> results = recordCollection + .searchAsync(SEARCH_EMBEDDINGS, options).block(); + assertNotNull(results); + assertEquals(3, results.size()); + // The first hotel should be the most similar + assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); + } } From 5306263c7b480ec7f9676a2f477ab2d2d2290db6 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 24 Sep 2024 18:47:57 -0700 Subject: [PATCH 057/277] Add SQLiteVectorStoreQueryProvider --- api-test/integration-tests/pom.xml | 5 + .../JDBCVectorStoreRecordCollectionTest.java | 39 +++-- semantickernel-experimental/pom.xml | 5 + .../SQLiteVectorStoreQueryProvider.java | 159 ++++++++++++++++++ 4 files changed, 194 insertions(+), 14 deletions(-) create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/sqlite/SQLiteVectorStoreQueryProvider.java diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 6def803a..dcf2f922 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -78,6 +78,11 @@ postgresql 42.7.3 + + org.xerial + sqlite-jdbc + 3.46.1.0 + org.testcontainers diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java index 874a469c..a4e5ba26 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -6,6 +6,7 @@ import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.filter.SQLEqualToFilterClause; +import com.microsoft.semantickernel.connectors.data.sqlite.SQLiteVectorStoreQueryProvider; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; @@ -17,6 +18,7 @@ import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import org.postgresql.ds.PGSimpleDataSource; +import org.sqlite.SQLiteDataSource; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; @@ -47,7 +49,8 @@ public class JDBCVectorStoreRecordCollectionTest { public enum QueryProvider { MySQL, - PostgreSQL + PostgreSQL, + SQLite } private JDBCVectorStoreRecordCollection buildRecordCollection(QueryProvider provider, @Nonnull String collectionName) { @@ -75,6 +78,14 @@ private JDBCVectorStoreRecordCollection buildRecordCollection(QueryProvid .withDataSource(dataSource) .build(); break; + case SQLite: + SQLiteDataSource sqliteDataSource = new SQLiteDataSource(); + sqliteDataSource.setUrl("jdbc:sqlite:file:testdb"); + dataSource = sqliteDataSource; + queryProvider = SQLiteVectorStoreQueryProvider.builder() + .withDataSource(sqliteDataSource) + .build(); + break; default: throw new IllegalArgumentException("Unknown query provider: " + provider); } @@ -131,7 +142,7 @@ public void upsertAndGetRecordAsync(QueryProvider provider) { // Upsert the first time for (Hotel hotel : hotels) { - Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block(); + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block(); assertNotNull(retrievedHotel); assertEquals(hotel.getId(), retrievedHotel.getId()); assertEquals(hotel.getRating(), retrievedHotel.getRating()); @@ -146,7 +157,7 @@ public void upsertAndGetRecordAsync(QueryProvider provider) { } for (Hotel hotel : hotels) { - Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block(); + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block(); assertNotNull(retrievedHotel); assertEquals(hotel.getId(), retrievedHotel.getId()); assertEquals(1.0, retrievedHotel.getRating()); @@ -157,7 +168,7 @@ public void upsertAndGetRecordAsync(QueryProvider provider) { @EnumSource(QueryProvider.class) public void getBatchAsync(QueryProvider provider) { String collectionName = "getBatchAsync"; - JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); List hotels = getHotels(); for (Hotel hotel : hotels) { @@ -169,7 +180,7 @@ public void getBatchAsync(QueryProvider provider) { keys.add(hotel.getId()); } - List retrievedHotels = recordCollection.getBatchAsync(keys, null).block(); + List retrievedHotels = recordCollection.getBatchAsync(keys, null).block(); assertNotNull(retrievedHotels); assertEquals(hotels.size(), retrievedHotels.size()); } @@ -178,7 +189,7 @@ public void getBatchAsync(QueryProvider provider) { @EnumSource(QueryProvider.class) public void upsertBatchAndGetBatchAsync(QueryProvider provider) { String collectionName = "upsertBatchAndGetBatchAsync"; - JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -188,7 +199,7 @@ public void upsertBatchAndGetBatchAsync(QueryProvider provider) { keys.add(hotel.getId()); } - List retrievedHotels = recordCollection.getBatchAsync(keys, null).block(); + List retrievedHotels = recordCollection.getBatchAsync(keys, null).block(); assertNotNull(retrievedHotels); assertEquals(hotels.size(), retrievedHotels.size()); } @@ -209,7 +220,7 @@ public void insertAndReplaceAsync(QueryProvider provider) { keys.add(hotel.getId()); } - List retrievedHotels = recordCollection.getBatchAsync(keys, null).block(); + List retrievedHotels = recordCollection.getBatchAsync(keys, null).block(); assertNotNull(retrievedHotels); assertEquals(hotels.size(), retrievedHotels.size()); } @@ -225,7 +236,7 @@ public void deleteRecordAsync(QueryProvider provider) { for (Hotel hotel : hotels) { recordCollection.deleteAsync(hotel.getId(), null).block(); - Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block(); + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block(); assertNull(retrievedHotel); } } @@ -247,7 +258,7 @@ public void deleteBatchAsync(QueryProvider provider) { recordCollection.deleteBatchAsync(keys, null).block(); for (String key : keys) { - Hotel retrievedHotel = recordCollection.getAsync(key, null).block(); + Hotel retrievedHotel = recordCollection.getAsync(key, null).block(); assertNull(retrievedHotel); } } @@ -266,7 +277,7 @@ public void getWithNoVectors(QueryProvider provider) { .build(); for (Hotel hotel : hotels) { - Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), options).block(); + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), options).block(); assertNotNull(retrievedHotel); assertEquals(hotel.getId(), retrievedHotel.getId()); assertNull(retrievedHotel.getEuclidean()); @@ -277,7 +288,7 @@ public void getWithNoVectors(QueryProvider provider) { .build(); for (Hotel hotel : hotels) { - Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), options).block(); + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), options).block(); assertNotNull(retrievedHotel); assertEquals(hotel.getId(), retrievedHotel.getId()); assertNotNull(retrievedHotel.getEuclidean()); @@ -302,7 +313,7 @@ public void getBatchWithNoVectors(QueryProvider provider) { keys.add(hotel.getId()); } - List retrievedHotels = recordCollection.getBatchAsync(keys, options).block(); + List retrievedHotels = recordCollection.getBatchAsync(keys, options).block(); assertNotNull(retrievedHotels); assertEquals(hotels.size(), retrievedHotels.size()); @@ -314,7 +325,7 @@ public void getBatchWithNoVectors(QueryProvider provider) { .includeVectors(true) .build(); - retrievedHotels = recordCollection.getBatchAsync(keys, options).block(); + retrievedHotels = recordCollection.getBatchAsync(keys, options).block(); assertNotNull(retrievedHotels); assertEquals(hotels.size(), retrievedHotels.size()); diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 5467df12..3f72b313 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -112,6 +112,11 @@ postgresql 42.7.3 + + org.xerial + sqlite-jdbc + 3.46.1.0 + diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/sqlite/SQLiteVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/sqlite/SQLiteVectorStoreQueryProvider.java new file mode 100644 index 00000000..c501c99f --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/sqlite/SQLiteVectorStoreQueryProvider.java @@ -0,0 +1,159 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.sqlite; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; +import com.microsoft.semantickernel.exceptions.SKException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +import javax.annotation.Nonnull; +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.List; +import java.util.stream.Collectors; + +public class SQLiteVectorStoreQueryProvider extends + JDBCVectorStoreQueryProvider implements SQLVectorStoreQueryProvider { + + private final DataSource dataSource; + private final ObjectMapper objectMapper; + + @SuppressFBWarnings("EI_EXPOSE_REP2") + private SQLiteVectorStoreQueryProvider( + @Nonnull DataSource dataSource, + @Nonnull String collectionsTable, + @Nonnull String prefixForCollectionTables, + @Nonnull ObjectMapper objectMapper) { + super(dataSource, collectionsTable, prefixForCollectionTables); + this.dataSource = dataSource; + this.objectMapper = objectMapper; + } + + /** + * Creates a new builder. + * @return the builder + */ + public static Builder builder() { + return new Builder(); + } + + private void setUpsertStatementValues(PreparedStatement statement, Object record, + List fields) { + JsonNode jsonNode = objectMapper.valueToTree(record); + + for (int i = 0; i < fields.size(); ++i) { + VectorStoreRecordField field = fields.get(i); + try { + JsonNode valueNode = jsonNode.get(field.getEffectiveStorageName()); + + if (field instanceof VectorStoreRecordVectorField) { + // Convert the vector field to a string + if (!field.getFieldType().equals(String.class)) { + statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); + continue; + } + } + + statement.setObject(i + 1, + objectMapper.convertValue(valueNode, field.getFieldType())); + } catch (SQLException | JsonProcessingException e) { + throw new RuntimeException(e); + } + } + } + + /** + * Upserts records into the collection. + * @param collectionName the collection name + * @param records the records to upsert + * @param recordDefinition the record definition + * @param options the upsert options + * @throws SKException if the upsert fails + */ + @Override + @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") // SQL query is generated dynamically with valid identifiers + public void upsertRecords(String collectionName, List records, + VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { + List fields = recordDefinition.getAllFields(); + + String query = formatQuery("INSERT OR REPLACE INTO %s (%s) VALUES (%s)", + getCollectionTableName(collectionName), + getQueryColumnsFromFields(fields), + getWildcardString(fields.size())); + + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(query)) { + for (Object record : records) { + setUpsertStatementValues(statement, record, recordDefinition.getAllFields()); + statement.addBatch(); + } + + statement.executeBatch(); + } catch (SQLException e) { + throw new SKException("Failed to upsert records", e); + } + } + + public static class Builder + extends JDBCVectorStoreQueryProvider.Builder { + private DataSource dataSource; + private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; + private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; + private ObjectMapper objectMapper = new ObjectMapper(); + + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withDataSource(DataSource dataSource) { + this.dataSource = dataSource; + return this; + } + + /** + * Sets the collections table name. + * @param collectionsTable the collections table name + * @return the builder + */ + public Builder withCollectionsTable(String collectionsTable) { + this.collectionsTable = validateSQLidentifier(collectionsTable); + return this; + } + + /** + * Sets the prefix for collection tables. + * @param prefixForCollectionTables the prefix for collection tables + * @return the builder + */ + public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { + this.prefixForCollectionTables = validateSQLidentifier(prefixForCollectionTables); + return this; + } + + /** + * Sets the object mapper. + * @param objectMapper the object mapper + * @return the builder + */ + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withObjectMapper(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + return this; + } + + public SQLiteVectorStoreQueryProvider build() { + if (dataSource == null) { + throw new SKException("DataSource is required"); + } + + return new SQLiteVectorStoreQueryProvider(dataSource, collectionsTable, + prefixForCollectionTables, objectMapper); + } + } +} From 7c478122df22a5ba5aa5eee756ad76e85b5d80c1 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 24 Sep 2024 18:56:13 -0700 Subject: [PATCH 058/277] Fix JDBCVectorStoreTest --- .../connectors/memory/jdbc/JDBCVectorStoreTest.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java index d70e5a5b..33928262 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java @@ -6,10 +6,12 @@ import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.connectors.data.sqlite.SQLiteVectorStoreQueryProvider; import com.mysql.cj.jdbc.MysqlDataSource; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.postgresql.ds.PGSimpleDataSource; +import org.sqlite.SQLiteDataSource; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; @@ -59,6 +61,14 @@ private JDBCVectorStore buildVectorStore(QueryProvider provider) { .withDataSource(dataSource) .build(); break; + case SQLite: + SQLiteDataSource sqliteDataSource = new SQLiteDataSource(); + sqliteDataSource.setUrl("jdbc:sqlite:file:test"); + dataSource = sqliteDataSource; + queryProvider = SQLiteVectorStoreQueryProvider.builder() + .withDataSource(sqliteDataSource) + .build(); + break; default: throw new IllegalArgumentException("Unknown query provider: " + provider); } From db814b1fbb403138bf66235224fbbf7bcd4ad5b8 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Wed, 25 Sep 2024 11:16:08 -0700 Subject: [PATCH 059/277] Update vector store examples and add some bug fixes --- .../tests/connectors/memory/jdbc/Hotel.java | 3 +- ...ashSetVectorStoreRecordCollectionTest.java | 3 + ...isJsonVectorStoreRecordCollectionTest.java | 3 + .../samples/syntaxexamples/RunAll.java | 4 +- ....java => InMemoryVolatileVectorStore.java} | 47 ++++++++++--- ...java => VectorStoreWithAzureAISearch.java} | 23 +++--- ...aStorage.java => VectorStoreWithJDBC.java} | 65 ++++++++++++----- ...Storage.java => VectorStoreWithRedis.java} | 70 ++++++++++++------- .../jdbc/JDBCVectorStoreRecordCollection.java | 2 +- ...disHashSetVectorStoreRecordCollection.java | 9 ++- ...disVectorStoreCollectionSearchMapping.java | 4 +- 11 files changed, 158 insertions(+), 75 deletions(-) rename samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/{InMemory_DataStorage.java => InMemoryVolatileVectorStore.java} (77%) rename samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/{AzureAISearchVectorStore.java => VectorStoreWithAzureAISearch.java} (92%) rename samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/{JDBC_DataStorage.java => VectorStoreWithJDBC.java} (78%) rename samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/{Redis_DataStorage.java => VectorStoreWithRedis.java} (75%) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java index b518e913..d476b70a 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java @@ -9,6 +9,7 @@ import java.util.List; public class Hotel { + @JsonProperty("hotelId") @VectorStoreRecordKeyAttribute private final String id; @@ -46,7 +47,7 @@ public Hotel() { @JsonCreator public Hotel( - @JsonProperty("id") String id, + @JsonProperty("hotelId") String id, @JsonProperty("name") String name, @JsonProperty("code") int code, @JsonProperty("summary") String description, diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index 37d9b383..975e7679 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -33,6 +33,7 @@ import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -387,6 +388,8 @@ public void search(RecordCollectionOptions options, String embeddingName) { assertEquals(VectorSearchOptions.DEFAULT_RESULT_LIMIT, results.size()); // The third hotel should be the most similar assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + // Score should be different than zero + assertNotEquals(0.0, results.get(0).getScore()); assertNull(results.get(0).getRecord().getEuclidean()); } diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index 0203f7ce..792e9669 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -33,6 +33,7 @@ import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -387,6 +388,8 @@ public void search(RecordCollectionOptions options, String embeddingName) { assertEquals(VectorSearchOptions.DEFAULT_RESULT_LIMIT, results.size()); // The third hotel should be the most similar assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + // Score should be different than zero + assertNotEquals(0.0, results.get(0).getScore()); assertNull(results.get(0).getRecord().getEuclidean()); } diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/RunAll.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/RunAll.java index 1a86b95d..7d704fe6 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/RunAll.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/RunAll.java @@ -17,7 +17,7 @@ import com.microsoft.semantickernel.samples.syntaxexamples.functions.Example59_OpenAIFunctionCalling; import com.microsoft.semantickernel.samples.syntaxexamples.functions.Example60_AdvancedMethodFunctions; import com.microsoft.semantickernel.samples.syntaxexamples.java.KernelFunctionYaml_Example; -import com.microsoft.semantickernel.samples.syntaxexamples.memory.AzureAISearchVectorStore; +import com.microsoft.semantickernel.samples.syntaxexamples.memory.VectorStoreWithAzureAISearch; import com.microsoft.semantickernel.samples.syntaxexamples.plugins.Example10_DescribeAllPluginsAndFunctions; import com.microsoft.semantickernel.samples.syntaxexamples.plugins.Example13_ConversationSummaryPlugin; import com.microsoft.semantickernel.samples.syntaxexamples.template.Example06_TemplateLanguage; @@ -38,7 +38,7 @@ public class RunAll { public static void main(String[] args) { List mains = Arrays.asList( - AzureAISearchVectorStore::main, + VectorStoreWithAzureAISearch::main, Example01_NativeFunctions::main, Example03_Arguments::main, Example05_InlineFunctionDefinition::main, diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java similarity index 77% rename from samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java rename to samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java index 70c7297d..a3bfbe61 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java @@ -5,7 +5,9 @@ import com.azure.ai.openai.OpenAIClientBuilder; import com.azure.core.credential.AzureKeyCredential; import com.azure.core.credential.KeyCredential; +import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.VolatileVectorStore; import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; @@ -20,7 +22,7 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -public class InMemory_DataStorage { +public class InMemoryVolatileVectorStore { private static final String CLIENT_KEY = System.getenv("CLIENT_KEY"); private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY"); @@ -34,14 +36,14 @@ public class InMemory_DataStorage { private static final int EMBEDDING_DIMENSIONS = 1536; static class GitHubFile { - + @JsonProperty("fileId") // Set a different name for the storage field if needed @VectorStoreRecordKeyAttribute() private final String id; @VectorStoreRecordDataAttribute() private final String description; @VectorStoreRecordDataAttribute private final String link; - @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = "Hnsw") + @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = "Hnsw", distanceFunction = "cosineDistance") private final List embedding; public GitHubFile( @@ -58,13 +60,18 @@ public GitHubFile( public String getId() { return id; } - public String getDescription() { return description; } + public String getLink() { + return link; + } + public List getEmbedding() { + return embedding; + } static String encodeId(String realId) { - return AzureAISearchVectorStore.GitHubFile.encodeId(realId); + return VectorStoreWithAzureAISearch.GitHubFile.encodeId(realId); } } @@ -93,10 +100,10 @@ public static void main(String[] args) { .withDimensions(EMBEDDING_DIMENSIONS) .build(); - inMemoryDataStorage(embeddingGeneration); + inMemoryStoreAndSearch(embeddingGeneration); } - public static void inMemoryDataStorage( + public static void inMemoryStoreAndSearch( OpenAITextEmbeddingGenerationService embeddingGeneration) { // Create a new Volatile vector store var volatileVectorStore = new VolatileVectorStore(); @@ -108,15 +115,32 @@ public static void inMemoryDataStorage( .build()); // Create collection if it does not exist and store data - List ids = collection + collection .createCollectionIfNotExistsAsync() .then(storeData(collection, embeddingGeneration, sampleData())) .block(); - // Retrieve all records from the collection - List data = collection.getBatchAsync(ids, null).block(); + // Search for results + // Volatile store executes an exhaustive search, for approximate search use Azure AI Search, Redis or JDBC with PostgreSQL + var results = search("How to get started", collection, embeddingGeneration).block(); + + if (results == null || results.isEmpty()) { + System.out.println("No search results found."); + return; + } + var searchResult = results.get(0); + System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", + searchResult.getScore(), searchResult.getRecord().link, + searchResult.getRecord().description); + } - data.forEach(gitHubFile -> System.out.println("Retrieved: " + gitHubFile.getDescription())); + private static Mono>> search( + String searchText, + VectorStoreRecordCollection recordCollection, + OpenAITextEmbeddingGenerationService embeddingGeneration) { + // Generate embeddings for the search text and search for the closest records + return embeddingGeneration.generateEmbeddingsAsync(Collections.singletonList(searchText)) + .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); } private static Mono> storeData( @@ -128,6 +152,7 @@ private static Mono> storeData( .flatMap(entry -> { System.out.println("Save '" + entry.getKey() + "' to memory."); + // Generate embeddings for the data and store it return embeddingGeneration .generateEmbeddingsAsync(Collections.singletonList(entry.getValue())) .flatMap(embeddings -> { diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java similarity index 92% rename from samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java rename to samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java index ab2b72aa..39813b2e 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearchVectorStore.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java @@ -12,10 +12,11 @@ import com.azure.search.documents.indexes.SearchIndexClientBuilder; import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; +import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStore; import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreOptions; -import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; @@ -30,7 +31,7 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -public class AzureAISearchVectorStore { +public class VectorStoreWithAzureAISearch { private static final String CLIENT_KEY = System.getenv("CLIENT_KEY"); private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY"); @@ -110,22 +111,21 @@ public static void main(String[] args) { .clientOptions(clientOptions()) .buildAsyncClient(); - dataStorageWithAzureAISearch(searchClient, embeddingGeneration); + storeAndSearch(searchClient, embeddingGeneration); } - public static void dataStorageWithAzureAISearch( + public static void storeAndSearch( SearchIndexAsyncClient searchClient, OpenAITextEmbeddingGenerationService embeddingGeneration) { - // Create a new Azure AI Search vector store - var azureAISearchVectorStore = com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStore - .builder() + // Build an Azure AI Search Vector Store + var azureAISearchVectorStore = AzureAISearchVectorStore.builder() .withSearchIndexAsyncClient(searchClient) .withOptions(new AzureAISearchVectorStoreOptions()) .build(); String collectionName = "skgithubfiles"; - var collection = (AzureAISearchVectorStoreRecordCollection) azureAISearchVectorStore + var collection = azureAISearchVectorStore .getCollection( collectionName, AzureAISearchVectorStoreRecordCollectionOptions.builder() @@ -154,15 +154,15 @@ public static void dataStorageWithAzureAISearch( private static Mono>> search( String searchText, - AzureAISearchVectorStoreRecordCollection recordCollection, + VectorStoreRecordCollection recordCollection, OpenAITextEmbeddingGenerationService embeddingGeneration) { - + // Generate embeddings for the search text and search for the closest records return embeddingGeneration.generateEmbeddingsAsync(Collections.singletonList(searchText)) .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); } private static Mono> storeData( - AzureAISearchVectorStoreRecordCollection recordCollection, + VectorStoreRecordCollection recordCollection, OpenAITextEmbeddingGenerationService embeddingGeneration, Map data) { @@ -170,6 +170,7 @@ private static Mono> storeData( .flatMap(entry -> { System.out.println("Save '" + entry.getKey() + "' to memory."); + // Generate embeddings for the data and store it return embeddingGeneration .generateEmbeddingsAsync(Collections.singletonList(entry.getValue())) .flatMap(embeddings -> { diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java similarity index 78% rename from samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java rename to samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java index 9e2e4ea4..078cd99b 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java @@ -5,11 +5,13 @@ import com.azure.ai.openai.OpenAIClientBuilder; import com.azure.core.credential.AzureKeyCredential; import com.azure.core.credential.KeyCredential; +import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStore; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; @@ -23,11 +25,11 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import javax.sql.DataSource; + import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -public class JDBC_DataStorage { +public class VectorStoreWithJDBC { private static final String CLIENT_KEY = System.getenv("CLIENT_KEY"); private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY"); @@ -38,18 +40,16 @@ public class JDBC_DataStorage { .getOrDefault("EMBEDDING_MODEL_ID", "text-embedding-3-large"); private static final int EMBEDDING_DIMENSIONS = 1536; - // Run a MySQL server with: - // docker run -d --name mysql-container -e MYSQL_ROOT_PASSWORD=root -e MYSQL_DATABASE=sk -p 3306:3306 mysql:latest static class GitHubFile { - + @JsonProperty("fileId") // Set a different name for the storage field if needed @VectorStoreRecordKeyAttribute() private final String id; @VectorStoreRecordDataAttribute() private final String description; @VectorStoreRecordDataAttribute private final String link; - @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = "Hnsw") + @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = "Hnsw", distanceFunction = "cosineDistance") private final List embedding; public GitHubFile() { @@ -70,10 +70,15 @@ public GitHubFile( public String getId() { return id; } - public String getDescription() { return description; } + public String getLink() { + return link; + } + public List getEmbedding() { + return embedding; + } static String encodeId(String realId) { byte[] bytes = Base64.getUrlEncoder().encode(realId.getBytes(StandardCharsets.UTF_8)); @@ -81,9 +86,12 @@ static String encodeId(String realId) { } } + // Run a MySQL server with: + // docker run -d --name mysql-container -e MYSQL_ROOT_PASSWORD=root -e MYSQL_DATABASE=sk -p 3306:3306 mysql:latest + public static void main(String[] args) throws SQLException { System.out.println("=============================================================="); - System.out.println("========== JDBC Vector Store Example =============="); + System.out.println("=============== JDBC Vector Store Example ===================="); System.out.println("=============================================================="); OpenAIAsyncClient client; @@ -100,30 +108,30 @@ public static void main(String[] args) throws SQLException { .buildAsyncClient(); } + // Create an OpenAI text embedding generation service var embeddingGeneration = OpenAITextEmbeddingGenerationService.builder() .withOpenAIAsyncClient(client) .withModelId(MODEL_ID) .withDimensions(EMBEDDING_DIMENSIONS) .build(); + storeAndSearch(embeddingGeneration); + } + + public static void storeAndSearch(OpenAITextEmbeddingGenerationService embeddingGeneration) { + // Configure the data source var dataSource = new MysqlDataSource(); dataSource.setUrl("jdbc:mysql://localhost:3306/sk"); dataSource.setPassword("root"); dataSource.setUser("root"); - dataStorageWithMySQL(dataSource, embeddingGeneration); - } - - public static void dataStorageWithMySQL( - DataSource dataSource, - OpenAITextEmbeddingGenerationService embeddingGeneration) { - // Build a query provider + // Other available query providers are PostgreSQLVectorStoreQueryProvider and SQLiteVectorStoreQueryProvider var queryProvider = MySQLVectorStoreQueryProvider.builder() .withDataSource(dataSource) .build(); - // Create a new vector store + // Build a vector store var jdbcVectorStore = JDBCVectorStore.builder() .withDataSource(dataSource) .withOptions(JDBCVectorStoreOptions.builder() @@ -131,6 +139,7 @@ public static void dataStorageWithMySQL( .build()) .build(); + // Set up the record collection to use String collectionName = "skgithubfiles"; var collection = jdbcVectorStore.getCollection(collectionName, JDBCVectorStoreRecordCollectionOptions.builder() @@ -138,14 +147,31 @@ public static void dataStorageWithMySQL( .build()); // Create collection if it does not exist and store data - List ids = collection + collection .createCollectionIfNotExistsAsync() .then(storeData(collection, embeddingGeneration, sampleData())) .block(); - List data = collection.getBatchAsync(ids, null).block(); + // Search for results + var results = search("How to get started", collection, embeddingGeneration).block(); + + if (results == null || results.isEmpty()) { + System.out.println("No search results found."); + return; + } + var searchResult = results.get(0); + System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", + searchResult.getScore(), searchResult.getRecord().link, + searchResult.getRecord().description); + } - data.forEach(gitHubFile -> System.out.println("Retrieved: " + gitHubFile.getDescription())); + private static Mono>> search( + String searchText, + VectorStoreRecordCollection recordCollection, + OpenAITextEmbeddingGenerationService embeddingGeneration) { + // Generate embeddings for the search text and search for the closest records + return embeddingGeneration.generateEmbeddingsAsync(Collections.singletonList(searchText)) + .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); } private static Mono> storeData( @@ -157,6 +183,7 @@ private static Mono> storeData( .flatMap(entry -> { System.out.println("Save '" + entry.getKey() + "' to memory."); + // Generate embeddings for the data and store it return embeddingGeneration .generateEmbeddingsAsync(Collections.singletonList(entry.getValue())) .flatMap(embeddings -> { diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java similarity index 75% rename from samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java rename to samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java index 90acae83..745e57a7 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java @@ -8,10 +8,14 @@ import com.azure.core.util.ClientOptions; import com.azure.core.util.MetricsOptions; import com.azure.core.util.TracingOptions; +import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; +import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.connectors.data.redis.RedisStorageType; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStore; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreOptions; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; @@ -27,7 +31,7 @@ import reactor.core.publisher.Mono; import redis.clients.jedis.JedisPooled; -public class Redis_DataStorage { +public class VectorStoreWithRedis { private static final String CLIENT_KEY = System.getenv("CLIENT_KEY"); private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY"); @@ -39,19 +43,16 @@ public class Redis_DataStorage { .getOrDefault("EMBEDDING_MODEL_ID", "text-embedding-3-large"); private static final int EMBEDDING_DIMENSIONS = 1536; - // Can start a test server with: - // docker run -d --name redis-stack -p 6379:6379 -p 8001:8001 redis/redis-stack:latest - private static final String REDIS_URL = "redis://127.0.0.1:6379"; public static class GitHubFile { - + @JsonProperty("fileId") // Set a different name for the storage field if needed @VectorStoreRecordKeyAttribute() private final String id; @VectorStoreRecordDataAttribute() private final String description; @VectorStoreRecordDataAttribute private final String link; - @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = "Hnsw") + @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = "Hnsw", distanceFunction = "cosineDistance") private final List embedding; public GitHubFile() { @@ -72,19 +73,24 @@ public GitHubFile( public String getId() { return id; } - public String getDescription() { return description; } + public String getLink() { return link; } + public List getEmbedding() { return embedding; } static String encodeId(String realId) { - return AzureAISearchVectorStore.GitHubFile.encodeId(realId); + return VectorStoreWithAzureAISearch.GitHubFile.encodeId(realId); } } + // Can start a test server with: + // docker run -d --name redis-stack -p 6379:6379 -p 8001:8001 redis/redis-stack:latest + private static final String REDIS_URL = "redis://127.0.0.1:6379"; + public static void main(String[] args) { System.out.println("=============================================================="); - System.out.println("========== Redis Vector Store Example =============="); + System.out.println("================ Redis Vector Store Example =================="); System.out.println("=============================================================="); OpenAIAsyncClient client; @@ -107,19 +113,22 @@ public static void main(String[] args) { .withDimensions(EMBEDDING_DIMENSIONS) .build(); - dataStorageWithRedis(embeddingGeneration); + storeAndSearch(embeddingGeneration); } - public static void dataStorageWithRedis( + public static void storeAndSearch( OpenAITextEmbeddingGenerationService embeddingGeneration) { - + // Configure redis client JedisPooled jedis = new JedisPooled(REDIS_URL); - RedisVectorStore vectorStore = RedisVectorStore.builder() + // Build a vector store + // Available storage types are JSON and HASHSET. Default is JSON. + var vectorStore = RedisVectorStore.builder() .withClient(jedis) - .withOptions(new RedisVectorStoreOptions()) + .withOptions(RedisVectorStoreOptions.builder().withStorageType(RedisStorageType.JSON).build()) .build(); + // Set up the record collection to use String collectionName = "skgithubfiles"; var collection = vectorStore.getCollection(collectionName, RedisJsonVectorStoreRecordCollectionOptions.builder() @@ -127,17 +136,32 @@ public static void dataStorageWithRedis( .build()); // Create collection if it does not exist and store data - List ids = collection + collection .createCollectionIfNotExistsAsync() .then(storeData(collection, embeddingGeneration, sampleData())) .block(); - List data = collection.getBatchAsync(ids, null).block(); + // Search for results + // Might need to wait for the data to be indexed + var results = search("How to get started", collection, embeddingGeneration).block(); - data.forEach(gitHubFile -> System.out.println("Retrieved: " + gitHubFile.getDescription())); - - // TODO: Implement search functionality using Redis. + if (results == null || results.isEmpty()) { + System.out.println("No search results found."); + return; + } + var searchResult = results.get(0); + System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", + searchResult.getScore(), searchResult.getRecord().link, + searchResult.getRecord().description); + } + private static Mono>> search( + String searchText, + VectorStoreRecordCollection recordCollection, + OpenAITextEmbeddingGenerationService embeddingGeneration) { + // Generate embeddings for the search text and search for the closest records + return embeddingGeneration.generateEmbeddingsAsync(Collections.singletonList(searchText)) + .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); } private static Mono> storeData( @@ -149,6 +173,7 @@ private static Mono> storeData( .flatMap(entry -> { System.out.println("Save '" + entry.getKey() + "' to memory."); + // Generate embeddings for the data and store it return embeddingGeneration .generateEmbeddingsAsync(Collections.singletonList(entry.getValue())) .flatMap(embeddings -> { @@ -179,11 +204,4 @@ private static Map sampleData() { "README: README associated with a sample chat summary react-based webapp" }, }).collect(Collectors.toMap(element -> element[0], element -> element[1])); } - - private static ClientOptions clientOptions() { - return new ClientOptions() - .setTracingOptions(new TracingOptions()) - .setMetricsOptions(new MetricsOptions()) - .setApplicationId("Semantic-Kernel"); - } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java index a62d495e..ce3413b7 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -207,7 +207,7 @@ public Mono> getBatchAsync(@Nonnull List keys, GetRecordOpt protected String getKeyFromRecord(Record data) { try { Field keyField = data.getClass() - .getDeclaredField(recordDefinition.getKeyField().getEffectiveStorageName()); + .getDeclaredField(recordDefinition.getKeyField().getName()); keyField.setAccessible(true); return (String) keyField.get(data); } catch (NoSuchFieldException | IllegalAccessException e) { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java index 80a39492..115fc43b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java @@ -397,8 +397,8 @@ public Mono>> searchAsync(List vector, Pair ftSearchParams = RedisVectorStoreCollectionSearchMapping .buildQuery(vector, options, recordDefinition, RedisStorageType.HASH_SET); - SearchResult searchResult = client.ftSearch(collectionName, - ftSearchParams.getLeft(), ftSearchParams.getRight()); + SearchResult searchResult = client.ftSearch(collectionName, ftSearchParams.getLeft(), + ftSearchParams.getRight()); return searchResult.getDocuments().stream() .map(doc -> { @@ -408,10 +408,13 @@ public Mono>> searchAsync(List vector, // Convert from Map to Map Map storage = new HashMap<>(); for (Map.Entry entry : doc.getProperties()) { + // Data and vector fields are returned as byte[] if (entry.getValue() instanceof byte[]) { storage.put(stringToBytes(entry.getKey()), (byte[]) entry.getValue()); - } else if (entry.getKey().equals( + } + // Score is returned as a string + else if (entry.getKey().equals( RedisVectorStoreCollectionSearchMapping.VECTOR_SCORE_FIELD)) { // Score is stored as a string in one of the fields score = Double.parseDouble((String) entry.getValue()); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java index 41d1b790..559bbefd 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java @@ -51,7 +51,6 @@ public static Pair buildQuery(List vector, if (storageType == RedisStorageType.HASH_SET) { // We also need to tell Redis to return the fields without decoding them // Vector fields specially need to be returned as raw bytes - searchParams.returnField(VECTOR_SCORE_FIELD, false); for (VectorStoreRecordDataField dataField : recordDefinition.getDataFields()) { searchParams.returnField(dataField.getEffectiveStorageName(), false); } @@ -60,6 +59,9 @@ public static Pair buildQuery(List vector, searchParams.returnField(v.getEffectiveStorageName(), false); } } + + // Also, return the score field, this can be decoded. + searchParams.returnField(VECTOR_SCORE_FIELD, true); } return Pair.of(knn, searchParams); From 3eea6cd783547b47b67d1fc362ea6ed645d105f3 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Wed, 25 Sep 2024 20:53:51 +0100 Subject: [PATCH 060/277] Add support for HSQLDB store --- .../OpenAITextEmbeddingGenerationService.java | 39 ++- api-test/integration-tests/pom.xml | 8 + .../tests/connectors/memory/jdbc/Hotel.java | 9 +- .../JDBCVectorStoreRecordCollectionTest.java | 199 +++++++++------ .../tests/connectors/memory/redis/Hotel.java | 25 +- .../memory/InMemoryVolatileVectorStore.java | 18 +- .../memory/VectorStoreWithAzureAISearch.java | 6 +- .../memory/VectorStoreWithJDBC.java | 19 +- .../memory/VectorStoreWithRedis.java | 30 ++- .../EmbeddingGenerationService.java | 19 +- .../handlebars/HandlebarsPromptTemplate.java | 67 +++-- .../HandlebarsPromptTemplateTest.java | 137 +++++++++++ .../HSQLDBVectorStoreQueryProvider.java | 228 ++++++++++++++++++ .../connectors/data/jdbc/JDBCVectorStore.java | 39 ++- .../jdbc/JDBCVectorStoreQueryProvider.java | 105 +++++--- .../jdbc/JDBCVectorStoreRecordCollection.java | 3 +- ...disVectorStoreCollectionSearchMapping.java | 9 +- .../VectorStoreRecordVectorAttribute.java | 6 +- .../VectorStoreRecordDefinition.java | 12 +- .../options/GetRecordOptions.java | 36 ++- .../microsoft/semantickernel/data/Hotel.java | 9 +- 21 files changed, 809 insertions(+), 214 deletions(-) create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/hsqldb/HSQLDBVectorStoreQueryProvider.java diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textembedding/OpenAITextEmbeddingGenerationService.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textembedding/OpenAITextEmbeddingGenerationService.java index cd2c7aa8..f8ccaeb0 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textembedding/OpenAITextEmbeddingGenerationService.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textembedding/OpenAITextEmbeddingGenerationService.java @@ -8,35 +8,37 @@ import com.microsoft.semantickernel.aiservices.openai.OpenAiService; import com.microsoft.semantickernel.exceptions.AIException; import com.microsoft.semantickernel.services.openai.OpenAiServiceBuilder; -import com.microsoft.semantickernel.services.textcompletion.TextGenerationService; import com.microsoft.semantickernel.services.textembedding.Embedding; import com.microsoft.semantickernel.services.textembedding.TextEmbeddingGenerationService; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import reactor.core.publisher.Mono; -import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.List; - /** * An OpenAI implementation of a {@link TextEmbeddingGenerationService}. - * */ public class OpenAITextEmbeddingGenerationService extends OpenAiService implements TextEmbeddingGenerationService { + private static final Logger LOGGER = LoggerFactory .getLogger(OpenAITextEmbeddingGenerationService.class); private static final int DEFAULT_DIMENSIONS = 1536; private final int dimensions; + public static final int EMBEDDING_DIMENSIONS_SMALL = 1536; + public static final int EMBEDDING_DIMENSIONS_LARGE = 3072; + /** * Creates a new {@link OpenAITextEmbeddingGenerationService}. * - * @param client OpenAI client + * @param client OpenAI client * @param deploymentName deployment name - * @param modelId OpenAI model id - * @param serviceId Service id + * @param modelId OpenAI model id + * @param serviceId Service id */ public OpenAITextEmbeddingGenerationService( OpenAIAsyncClient client, @@ -57,6 +59,24 @@ public static Builder builder() { return new Builder(); } + /** + * Generates embeddings for the given data. + * + * @param data The data to generate embeddings for. + * @return A Mono that completes with the embeddings. + */ + @Override + public Mono generateEmbeddingAsync(String data) { + return this.internalGenerateTextEmbeddingsAsync(Arrays.asList(data)) + .flatMap(embeddings -> { + if (embeddings.isEmpty()) { + return Mono.empty(); + } + + return Mono.just(embeddings.get(0)); + }); + } + /** * Generates embeddings for the given data. * @@ -88,6 +108,7 @@ protected Mono> internalGenerateTextEmbeddingsAsync(List */ public static class Builder extends OpenAiServiceBuilder { + private int dimensions = DEFAULT_DIMENSIONS; /** diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 84690d70..0eab06c3 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -122,6 +122,14 @@ jsonschema-module-jackson test + + + + org.hsqldb + hsqldb + 2.7.3 + test + diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java index d476b70a..2a1394f1 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java @@ -5,6 +5,7 @@ import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import java.util.List; @@ -24,19 +25,19 @@ public class Hotel { private final String description; @JsonProperty("summaryEmbedding1") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "euclidean") + @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) private final List euclidean; @JsonProperty("summaryEmbedding2") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "cosineDistance") + @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List cosineDistance; @JsonProperty("summaryEmbedding3") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "dotProduct") + @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.DOT_PRODUCT) private final List dotProduct; @JsonProperty("indexedSummaryEmbedding") - @VectorStoreRecordVectorAttribute(dimensions = 8, indexKind = "hnsw", distanceFunction = "euclidean") + @VectorStoreRecordVectorAttribute(dimensions = 8, indexKind = "hnsw", distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) private final List indexedEuclidean; @VectorStoreRecordDataAttribute private double rating; diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java index a4e5ba26..ffd6bba4 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -1,17 +1,33 @@ package com.microsoft.semantickernel.tests.connectors.memory.jdbc; -import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.microsoft.semantickernel.connectors.data.hsqldb.HSQLDBVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.connectors.data.jdbc.filter.SQLEqualToFilterClause; import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.connectors.data.jdbc.filter.SQLEqualToFilterClause; import com.microsoft.semantickernel.connectors.data.sqlite.SQLiteVectorStoreQueryProvider; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import com.mysql.cj.jdbc.MysqlDataSource; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.stream.Stream; +import javax.annotation.Nonnull; +import javax.sql.DataSource; +import org.hsqldb.jdbc.JDBCDataSourceFactory; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -25,17 +41,6 @@ import org.testcontainers.junit.jupiter.Testcontainers; import org.testcontainers.utility.DockerImageName; -import javax.annotation.Nonnull; -import javax.sql.DataSource; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; - @Testcontainers public class JDBCVectorStoreRecordCollectionTest { @@ -43,17 +48,21 @@ public class JDBCVectorStoreRecordCollectionTest { @Container private static final MySQLContainer MYSQL_CONTAINER = new MySQLContainer<>("mysql:5.7.34"); - private static final DockerImageName PGVECTOR = DockerImageName.parse("pgvector/pgvector:pg16").asCompatibleSubstituteFor("postgres"); + private static final DockerImageName PGVECTOR = DockerImageName.parse("pgvector/pgvector:pg16") + .asCompatibleSubstituteFor("postgres"); @Container - private static final PostgreSQLContainer POSTGRESQL_CONTAINER = new PostgreSQLContainer<>(PGVECTOR); + private static final PostgreSQLContainer POSTGRESQL_CONTAINER = new PostgreSQLContainer<>( + PGVECTOR); public enum QueryProvider { MySQL, PostgreSQL, - SQLite + SQLite, + HSQLDB } - private JDBCVectorStoreRecordCollection buildRecordCollection(QueryProvider provider, @Nonnull String collectionName) { + private JDBCVectorStoreRecordCollection buildRecordCollection(QueryProvider provider, + @Nonnull String collectionName) { SQLVectorStoreQueryProvider queryProvider; DataSource dataSource; @@ -65,8 +74,8 @@ private JDBCVectorStoreRecordCollection buildRecordCollection(QueryProvid mysqlDataSource.setPassword(MYSQL_CONTAINER.getPassword()); dataSource = mysqlDataSource; queryProvider = MySQLVectorStoreQueryProvider.builder() - .withDataSource(dataSource) - .build(); + .withDataSource(dataSource) + .build(); break; case PostgreSQL: PGSimpleDataSource pgSimpleDataSource = new PGSimpleDataSource(); @@ -75,29 +84,52 @@ private JDBCVectorStoreRecordCollection buildRecordCollection(QueryProvid pgSimpleDataSource.setPassword(POSTGRESQL_CONTAINER.getPassword()); dataSource = pgSimpleDataSource; queryProvider = PostgreSQLVectorStoreQueryProvider.builder() - .withDataSource(dataSource) - .build(); + .withDataSource(dataSource) + .build(); break; case SQLite: SQLiteDataSource sqliteDataSource = new SQLiteDataSource(); sqliteDataSource.setUrl("jdbc:sqlite:file:testdb"); dataSource = sqliteDataSource; queryProvider = SQLiteVectorStoreQueryProvider.builder() - .withDataSource(sqliteDataSource) - .build(); + .withDataSource(sqliteDataSource) + .build(); + break; + case HSQLDB: + try { + Path file = Files.createTempFile("testdb", ".db"); + file.toFile().deleteOnExit(); + + Properties properties = new Properties(); + properties.putAll( + Map.of( + "url", "jdbc:hsqldb:file:" + file.toFile().getAbsolutePath() + + ";sql.syntax_mys=true", + "user", "SA", + "password", "" + ) + ); + + dataSource = JDBCDataSourceFactory.createDataSource(properties); + } catch (Exception e) { + throw new RuntimeException(e); + } + + queryProvider = HSQLDBVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build(); break; default: throw new IllegalArgumentException("Unknown query provider: " + provider); } - - JDBCVectorStoreRecordCollection recordCollection = new JDBCVectorStoreRecordCollection<>( - dataSource, - collectionName, - JDBCVectorStoreRecordCollectionOptions.builder() - .withRecordClass(Hotel.class) - .withQueryProvider(queryProvider) - .build()); + JDBCVectorStoreRecordCollection recordCollection = new JDBCVectorStoreRecordCollection<>( + dataSource, + collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .withQueryProvider(queryProvider) + .build()); recordCollection.prepareAsync().block(); recordCollection.createCollectionIfNotExistsAsync().block(); @@ -112,32 +144,41 @@ public void buildRecordCollection(QueryProvider provider) { private List getHotels() { return Arrays.asList( - new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f),null, null, null, 4.0), - new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f),null, null, null, 4.0), - new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f),null, null, null, 5.0), - new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f),null, null, null, 4.0), - new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f),null, null, null, 4.0) + new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", + Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), null, null, null, + 4.0), + new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", + Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), null, null, null, + 4.0), + new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", + Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), null, null, null, + 5.0), + new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", + Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), null, null, null, + 4.0), + new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", + Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), null, null, null, + 4.0) ); } /** - * Search embeddings similar to the third hotel embeddings. - * In order of similarity: - * 1. Hotel 3 - * 2. Hotel 1 - * 3. Hotel 4 + * Search embeddings similar to the third hotel embeddings. In order of similarity: 1. Hotel 3 + * 2. Hotel 1 3. Hotel 4 */ - private static final List SEARCH_EMBEDDINGS = Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.2f); + private static final List SEARCH_EMBEDDINGS = Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, + -0.8f, 1.1f, -2.2f, 8.2f); @ParameterizedTest @EnumSource(QueryProvider.class) public void upsertAndGetRecordAsync(QueryProvider provider) { String collectionName = "upsertAndGetRecordAsync"; - JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, + collectionName); List hotels = getHotels(); for (Hotel hotel : hotels) { - recordCollection.upsertAsync(hotel, null).block(); + recordCollection.upsertAsync(hotel, null).block(); } // Upsert the first time @@ -168,11 +209,12 @@ public void upsertAndGetRecordAsync(QueryProvider provider) { @EnumSource(QueryProvider.class) public void getBatchAsync(QueryProvider provider) { String collectionName = "getBatchAsync"; - JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, + collectionName); List hotels = getHotels(); for (Hotel hotel : hotels) { - recordCollection.upsertAsync(hotel, null).block(); + recordCollection.upsertAsync(hotel, null).block(); } List keys = new ArrayList<>(); @@ -189,10 +231,11 @@ public void getBatchAsync(QueryProvider provider) { @EnumSource(QueryProvider.class) public void upsertBatchAndGetBatchAsync(QueryProvider provider) { String collectionName = "upsertBatchAndGetBatchAsync"; - JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, + collectionName); List hotels = getHotels(); - recordCollection.upsertBatchAsync(hotels, null).block(); + recordCollection.upsertBatchAsync(hotels, null).block(); List keys = new ArrayList<>(); for (Hotel hotel : hotels) { @@ -208,12 +251,13 @@ public void upsertBatchAndGetBatchAsync(QueryProvider provider) { @EnumSource(QueryProvider.class) public void insertAndReplaceAsync(QueryProvider provider) { String collectionName = "insertAndReplaceAsync"; - JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, + collectionName); List hotels = getHotels(); - recordCollection.upsertBatchAsync(hotels, null).block(); - recordCollection.upsertBatchAsync(hotels, null).block(); - recordCollection.upsertBatchAsync(hotels, null).block(); + recordCollection.upsertBatchAsync(hotels, null).block(); + recordCollection.upsertBatchAsync(hotels, null).block(); + recordCollection.upsertBatchAsync(hotels, null).block(); List keys = new ArrayList<>(); for (Hotel hotel : hotels) { @@ -229,13 +273,14 @@ public void insertAndReplaceAsync(QueryProvider provider) { @EnumSource(QueryProvider.class) public void deleteRecordAsync(QueryProvider provider) { String collectionName = "deleteRecordAsync"; - JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, + collectionName); List hotels = getHotels(); - recordCollection.upsertBatchAsync(hotels, null).block(); + recordCollection.upsertBatchAsync(hotels, null).block(); for (Hotel hotel : hotels) { - recordCollection.deleteAsync(hotel.getId(), null).block(); + recordCollection.deleteAsync(hotel.getId(), null).block(); Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block(); assertNull(retrievedHotel); } @@ -245,17 +290,18 @@ public void deleteRecordAsync(QueryProvider provider) { @EnumSource(QueryProvider.class) public void deleteBatchAsync(QueryProvider provider) { String collectionName = "deleteBatchAsync"; - JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, + collectionName); List hotels = getHotels(); - recordCollection.upsertBatchAsync(hotels, null).block(); + recordCollection.upsertBatchAsync(hotels, null).block(); List keys = new ArrayList<>(); for (Hotel hotel : hotels) { keys.add(hotel.getId()); } - recordCollection.deleteBatchAsync(keys, null).block(); + recordCollection.deleteBatchAsync(keys, null).block(); for (String key : keys) { Hotel retrievedHotel = recordCollection.getAsync(key, null).block(); @@ -267,10 +313,11 @@ public void deleteBatchAsync(QueryProvider provider) { @EnumSource(QueryProvider.class) public void getWithNoVectors(QueryProvider provider) { String collectionName = "getWithNoVectors"; - JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, + collectionName); List hotels = getHotels(); - recordCollection.upsertBatchAsync(hotels, null).block(); + recordCollection.upsertBatchAsync(hotels, null).block(); GetRecordOptions options = GetRecordOptions.builder() .includeVectors(false) @@ -299,10 +346,11 @@ public void getWithNoVectors(QueryProvider provider) { @EnumSource(QueryProvider.class) public void getBatchWithNoVectors(QueryProvider provider) { String collectionName = "getBatchWithNoVectors"; - JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, + collectionName); List hotels = getHotels(); - recordCollection.upsertBatchAsync(hotels, null).block(); + recordCollection.upsertBatchAsync(hotels, null).block(); GetRecordOptions options = GetRecordOptions.builder() .includeVectors(false) @@ -349,7 +397,8 @@ private static Stream provideSearchParameters() { @MethodSource("provideSearchParameters") public void exactSearch(QueryProvider provider, String embeddingName) { String collectionName = "search" + embeddingName; - JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, + collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -360,7 +409,8 @@ public void exactSearch(QueryProvider provider, String embeddingName) { .build(); // Embeddings similar to the third hotel - List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, options).block(); + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, + options).block(); assertNotNull(results); assertEquals(3, results.size()); // The third hotel should be the most similar @@ -384,7 +434,8 @@ public void exactSearch(QueryProvider provider, String embeddingName) { @EnumSource(QueryProvider.class) public void approximateSearch(QueryProvider provider) { String collectionName = "searchWithIndex"; - JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, + collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -395,7 +446,8 @@ public void approximateSearch(QueryProvider provider) { .build(); // Embeddings similar to the third hotel - List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, options).block(); + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, + options).block(); assertNotNull(results); assertEquals(5, results.size()); // The third hotel should be the most similar @@ -406,7 +458,8 @@ public void approximateSearch(QueryProvider provider) { @MethodSource("provideSearchParameters") public void searchWithFilter(QueryProvider provider, String embeddingName) { String collectionName = "searchWithFilter"; - JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, + collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); @@ -415,11 +468,13 @@ public void searchWithFilter(QueryProvider provider, String embeddingName) { .withVectorFieldName(embeddingName) .withLimit(3) .withVectorSearchFilter( - VectorSearchFilter.builder().withEqualToFilterClause(new SQLEqualToFilterClause("rating", 4.0)).build()) + VectorSearchFilter.builder() + .withEqualToFilterClause(new SQLEqualToFilterClause("rating", 4.0)).build()) .build(); // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned - List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, options).block(); + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, + options).block(); assertNotNull(results); assertEquals(3, results.size()); // The first hotel should be the most similar @@ -430,12 +485,14 @@ public void searchWithFilter(QueryProvider provider, String embeddingName) { @Test public void postgresSearchIncludeAndNotIncludeVectors() { String collectionName = "searchIncludeAndNotIncludeVectors"; - JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(QueryProvider.PostgreSQL, collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection( + QueryProvider.PostgreSQL, collectionName); List hotels = getHotels(); recordCollection.upsertBatchAsync(hotels, null).block(); - List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, null).block(); + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, + null).block(); assertNotNull(results); assertEquals(3, results.size()); // The third hotel should be the most similar diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java index 04c2f138..185d134d 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java @@ -5,10 +5,11 @@ import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; - +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import java.util.List; public class Hotel { + @VectorStoreRecordKeyAttribute private final String id; @@ -23,15 +24,15 @@ public class Hotel { private final String description; @JsonProperty("summaryEmbedding1") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "euclidean") + @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) private final List euclidean; @JsonProperty("summaryEmbedding2") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "cosineDistance") + @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List cosineDistance; @JsonProperty("summaryEmbedding3") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "dotProduct") + @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.DOT_PRODUCT) private final List dotProduct; @VectorStoreRecordDataAttribute private double rating; @@ -42,14 +43,14 @@ public Hotel() { @JsonCreator public Hotel( - @JsonProperty("id") String id, - @JsonProperty("name") String name, - @JsonProperty("code") int code, - @JsonProperty("summary") String description, - @JsonProperty("summaryEmbedding1") List euclidean, - @JsonProperty("summaryEmbedding2") List cosineDistance, - @JsonProperty("summaryEmbedding3") List dotProduct, - @JsonProperty("rating") double rating) { + @JsonProperty("id") String id, + @JsonProperty("name") String name, + @JsonProperty("code") int code, + @JsonProperty("summary") String description, + @JsonProperty("summaryEmbedding1") List euclidean, + @JsonProperty("summaryEmbedding2") List cosineDistance, + @JsonProperty("summaryEmbedding3") List dotProduct, + @JsonProperty("rating") double rating) { this.id = id; this.name = name; this.code = code; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java index a3bfbe61..a7b0d663 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -43,7 +44,7 @@ static class GitHubFile { private final String description; @VectorStoreRecordDataAttribute private final String link; - @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = "Hnsw", distanceFunction = "cosineDistance") + @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = "Hnsw", distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List embedding; public GitHubFile( @@ -60,12 +61,15 @@ public GitHubFile( public String getId() { return id; } + public String getDescription() { return description; } + public String getLink() { return link; } + public List getEmbedding() { return embedding; } @@ -130,17 +134,17 @@ public static void inMemoryStoreAndSearch( } var searchResult = results.get(0); System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", - searchResult.getScore(), searchResult.getRecord().link, - searchResult.getRecord().description); + searchResult.getScore(), searchResult.getRecord().link, + searchResult.getRecord().description); } private static Mono>> search( - String searchText, - VectorStoreRecordCollection recordCollection, - OpenAITextEmbeddingGenerationService embeddingGeneration) { + String searchText, + VectorStoreRecordCollection recordCollection, + OpenAITextEmbeddingGenerationService embeddingGeneration) { // Generate embeddings for the search text and search for the closest records return embeddingGeneration.generateEmbeddingsAsync(Collections.singletonList(searchText)) - .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); + .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); } private static Mono> storeData( diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java index 39813b2e..0a8e5e7c 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java @@ -20,6 +20,7 @@ import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Base64; @@ -27,7 +28,6 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; - import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -49,14 +49,16 @@ public class VectorStoreWithAzureAISearch { private static final int EMBEDDING_DIMENSIONS = 1536; static class GitHubFile { + @JsonProperty("fileId") // Set a different name for the storage field if needed @VectorStoreRecordKeyAttribute() private final String id; @VectorStoreRecordDataAttribute() + @VectorStoreRecordVectorAttribute(distanceFunction = DistanceFunction.COSINE_DISTANCE, dimensions = EMBEDDING_DIMENSIONS) private final String description; @VectorStoreRecordDataAttribute private final String link; - @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = "Hnsw") + @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = "Hnsw", distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List embedding; public GitHubFile() { diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java index 078cd99b..161fc044 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java @@ -16,6 +16,7 @@ import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.mysql.cj.jdbc.MysqlDataSource; import java.nio.charset.StandardCharsets; import java.sql.SQLException; @@ -40,7 +41,6 @@ public class VectorStoreWithJDBC { .getOrDefault("EMBEDDING_MODEL_ID", "text-embedding-3-large"); private static final int EMBEDDING_DIMENSIONS = 1536; - static class GitHubFile { @JsonProperty("fileId") // Set a different name for the storage field if needed @VectorStoreRecordKeyAttribute() @@ -49,7 +49,7 @@ static class GitHubFile { private final String description; @VectorStoreRecordDataAttribute private final String link; - @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = "Hnsw", distanceFunction = "cosineDistance") + @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = "Hnsw", distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List embedding; public GitHubFile() { @@ -70,12 +70,15 @@ public GitHubFile( public String getId() { return id; } + public String getDescription() { return description; } + public String getLink() { return link; } + public List getEmbedding() { return embedding; } @@ -161,17 +164,17 @@ public static void storeAndSearch(OpenAITextEmbeddingGenerationService embedding } var searchResult = results.get(0); System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", - searchResult.getScore(), searchResult.getRecord().link, - searchResult.getRecord().description); + searchResult.getScore(), searchResult.getRecord().link, + searchResult.getRecord().description); } private static Mono>> search( - String searchText, - VectorStoreRecordCollection recordCollection, - OpenAITextEmbeddingGenerationService embeddingGeneration) { + String searchText, + VectorStoreRecordCollection recordCollection, + OpenAITextEmbeddingGenerationService embeddingGeneration) { // Generate embeddings for the search text and search for the closest records return embeddingGeneration.generateEmbeddingsAsync(Collections.singletonList(searchText)) - .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); + .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); } private static Mono> storeData( diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java index 745e57a7..be5af4b7 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.stream.Collectors; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import redis.clients.jedis.JedisPooled; @@ -43,7 +44,6 @@ public class VectorStoreWithRedis { .getOrDefault("EMBEDDING_MODEL_ID", "text-embedding-3-large"); private static final int EMBEDDING_DIMENSIONS = 1536; - public static class GitHubFile { @JsonProperty("fileId") // Set a different name for the storage field if needed @VectorStoreRecordKeyAttribute() @@ -52,7 +52,7 @@ public static class GitHubFile { private final String description; @VectorStoreRecordDataAttribute private final String link; - @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = "Hnsw", distanceFunction = "cosineDistance") + @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = "Hnsw", distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List embedding; public GitHubFile() { @@ -73,11 +73,18 @@ public GitHubFile( public String getId() { return id; } + public String getDescription() { return description; } - public String getLink() { return link; } - public List getEmbedding() { return embedding; } + + public String getLink() { + return link; + } + + public List getEmbedding() { + return embedding; + } static String encodeId(String realId) { return VectorStoreWithAzureAISearch.GitHubFile.encodeId(realId); @@ -125,7 +132,8 @@ public static void storeAndSearch( // Available storage types are JSON and HASHSET. Default is JSON. var vectorStore = RedisVectorStore.builder() .withClient(jedis) - .withOptions(RedisVectorStoreOptions.builder().withStorageType(RedisStorageType.JSON).build()) + .withOptions( + RedisVectorStoreOptions.builder().withStorageType(RedisStorageType.JSON).build()) .build(); // Set up the record collection to use @@ -151,17 +159,17 @@ public static void storeAndSearch( } var searchResult = results.get(0); System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", - searchResult.getScore(), searchResult.getRecord().link, - searchResult.getRecord().description); + searchResult.getScore(), searchResult.getRecord().link, + searchResult.getRecord().description); } private static Mono>> search( - String searchText, - VectorStoreRecordCollection recordCollection, - OpenAITextEmbeddingGenerationService embeddingGeneration) { + String searchText, + VectorStoreRecordCollection recordCollection, + OpenAITextEmbeddingGenerationService embeddingGeneration) { // Generate embeddings for the search text and search for the closest records return embeddingGeneration.generateEmbeddingsAsync(Collections.singletonList(searchText)) - .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); + .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); } private static Mono> storeData( diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java index 86d4d27d..79ce998e 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java @@ -1,14 +1,15 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.services.textembedding; -import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.services.AIService; -import reactor.core.publisher.Mono; - import java.util.List; +import reactor.core.publisher.Mono; -/** Interface for text embedding generation services */ +/** + * Interface for text embedding generation services + */ public interface EmbeddingGenerationService extends AIService { + /** * Generates a list of embeddings associated to the data * @@ -16,4 +17,14 @@ public interface EmbeddingGenerationService extends AIService { * @return List of embeddings of each data point */ Mono> generateEmbeddingsAsync(List data); + + /** + * Generates an embedding associated to the data + * + * @param data Text to generate embedding for + * @return Embedding of the data + */ + + Mono generateEmbeddingAsync(TValue data); + } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplate.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplate.java index ec701ce3..aed37718 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplate.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplate.java @@ -29,7 +29,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.Locale; import java.util.Map.Entry; import java.util.Optional; @@ -37,6 +36,7 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import org.apache.commons.text.StringEscapeUtils; import reactor.core.publisher.Mono; /** @@ -89,8 +89,7 @@ public Object resolve(Object context, String name) { if ("role".equalsIgnoreCase(name)) { return ((ChatMessageContent) context).getAuthorRole().name(); } else if ("content".equalsIgnoreCase(name)) { - return ContextVariableTypeConverter - .escapeXmlString(((ChatMessageContent) context).getContent()); + return ((ChatMessageContent) context).getContent(); } } return UNRESOLVED; @@ -131,17 +130,24 @@ private static class ContextVariableResolver implements ValueResolver { @Override public Object resolve(Object context, String name) { Object value = null; + ContextVariable variable = null; if (context instanceof KernelFunctionArguments) { - ContextVariable variable = ((KernelFunctionArguments) context).get(name); - value = variable != null ? variable.getValue() : UNRESOLVED; - } - if (context instanceof ContextVariable) { - value = ((ContextVariable) context).getValue(); + variable = ((KernelFunctionArguments) context).get(name); + } else if (context instanceof ContextVariable) { + variable = ((ContextVariable) context); } - if (value == null) { + + if (variable == null || variable.getValue() == null) { return UNRESOLVED; - } else { + } + + value = variable.getValue(); + + if (value instanceof Iterable) { return value; + } else { + // It is likely this will come escaped, but will be re escaped by the handlebars engine + return promptString(variable); } } @@ -149,11 +155,23 @@ public Object resolve(Object context, String name) { public Object resolve(Object context) { if (context instanceof ContextVariable) { Object result = ((ContextVariable) context).getValue(); - return result != null ? result : UNRESOLVED; + + if (result == null) { + return UNRESOLVED; + } else if (result instanceof Iterable) { + return result; + } else { + return promptString(((ContextVariable) context)); + } } return UNRESOLVED; } + private String promptString(ContextVariable context) { + // This will come escaped, but will be re escaped by the handlebars engine + return StringEscapeUtils.unescapeXml(context.toPromptString()); + } + @Override public Set> propertySet(Object context) { if (context instanceof KernelFunctionArguments) { @@ -192,22 +210,19 @@ public HandleBarsPromptTemplateHandler( } private Helper handleEach(InvocationContext invocationContext) { - return (context, options) -> { - if (context instanceof ContextVariable) { - return ((ContextVariable) context) + return (variable, options) -> { + if (variable instanceof ContextVariable) { + return ((ContextVariable) variable) .toPromptString(invocationContext.getContextVariableTypes()); } - if (context instanceof Iterable) { + if (variable instanceof Iterable) { StringBuilder sb = new StringBuilder(); - Iterator iterator = ((Iterable) context).iterator(); - while (iterator.hasNext()) { - Object element = iterator.next(); + + for (Object element : (Iterable) variable) { if (element instanceof KernelPlugin) { KernelPlugin plugin = (KernelPlugin) element; - Iterator> functions = plugin.iterator(); - while (functions.hasNext()) { - KernelFunction function = functions.next(); + for (KernelFunction function : plugin) { sb.append(options.fn(function)); } } else { @@ -218,10 +233,11 @@ private Helper handleEach(InvocationContext invocationContext) { } ContextVariableType type = invocationContext.getContextVariableTypes() - .getVariableTypeForClass(context.getClass()); + .getVariableTypeForClass(variable.getClass()); if (type != null) { - return type.getConverter() - .toPromptString(invocationContext.getContextVariableTypes(), context); + return type + .getConverter() + .toPromptString(invocationContext.getContextVariableTypes(), variable); } return null; }; @@ -248,7 +264,8 @@ private CharSequence handleMessage(Object context, Options options) return new Handlebars.SafeString( String.format( "%s", - role.toLowerCase(Locale.ROOT), content)); + role.toLowerCase(Locale.ROOT), + content)); } return null; } diff --git a/semantickernel-api/src/test/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplateTest.java b/semantickernel-api/src/test/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplateTest.java index beedb9b3..cf20ca51 100644 --- a/semantickernel-api/src/test/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplateTest.java +++ b/semantickernel-api/src/test/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplateTest.java @@ -122,4 +122,141 @@ void testRenderAsync() { .collect(joining(" ")); assertEquals(expResult, normalizedResult); } + + public static class Foo { + + @JsonProperty("val") + private final String val; + + public Foo(String val) { + this.val = val; + } + + public String getVal() { + return val; + } + } + + @Test + public void testSerializesObject() { + PromptTemplateConfig promptTemplate = PromptTemplateConfig.builder() + .withTemplate("{{input}}") + .withTemplateFormat("handlebars") + .build(); + + HandlebarsPromptTemplate instance = new HandlebarsPromptTemplate(promptTemplate); + + KernelFunctionArguments arguments = KernelFunctionArguments.builder() + .withVariable("input", new Foo("bar"), + ContextVariableJacksonConverter.create(Foo.class)) + .build(); + + // Return from renderAsync is normalized to remove empty lines and leading/trailing whitespace + String expResult = StringEscapeUtils.escapeXml11("{ \"val\" : \"bar\"}"); + + String result = instance.renderAsync(Kernel.builder().build(), arguments, null) + .block(); + Assertions.assertEquals(expResult, result.replaceAll("\\n", "")); + } + + @Test + public void testMessageContent() { + PromptTemplateConfig promptTemplate = PromptTemplateConfig.builder() + .withTemplate( + "{{#each input}}\n" + + "{{content}}\n" + + "{{/each}}") + .withTemplateFormat("handlebars") + .build(); + + HandlebarsPromptTemplate instance = new HandlebarsPromptTemplate(promptTemplate); + + KernelFunctionArguments arguments = KernelFunctionArguments.builder() + .withVariable("input", new ChatHistory() + .addAssistantMessage("foo") + .addUserMessage("bar\"<>&")) + .build(); + + // Return from renderAsync is normalized to remove empty lines and leading/trailing whitespace + String expResult = "foobar"<>&"; + + String result = instance.renderAsync(Kernel.builder().build(), arguments, null) + .block(); + Assertions.assertEquals(expResult, result.replaceAll("\\n", "")); + } + + @Test + public void testMessageHandler() { + PromptTemplateConfig promptTemplate = PromptTemplateConfig.builder() + .withTemplate("{{#message role=\"user\"}}\n" + + "{{input}}\n" + + "{{/message}}") + .withTemplateFormat("handlebars") + .build(); + + HandlebarsPromptTemplate instance = new HandlebarsPromptTemplate(promptTemplate); + + KernelFunctionArguments arguments = KernelFunctionArguments.builder() + .withVariable("input", "bar\"<>&") + .build(); + + // Return from renderAsync is normalized to remove empty lines and leading/trailing whitespace + String expResult = "bar"<>&"; + + String result = instance.renderAsync(Kernel.builder().build(), arguments, null) + .block(); + Assertions.assertEquals(expResult, result.replaceAll("\\n", "")); + } + + @Test + public void iterableWithContextVariable() { + PromptTemplateConfig promptTemplate = PromptTemplateConfig.builder() + .withTemplate( + "{{#each input}}" + + "{{this}}" + + "{{/each}}") + .withTemplateFormat("handlebars") + .build(); + + HandlebarsPromptTemplate instance = new HandlebarsPromptTemplate(promptTemplate); + + KernelFunctionArguments arguments = KernelFunctionArguments.builder() + .withVariable("input", Arrays.asList(ContextVariable.of("foo\"<>&"))) + .build(); + + // Return from renderAsync is normalized to remove empty lines and leading/trailing whitespace + String expResult = "foo"<>&"; + + String result = instance.renderAsync(Kernel.builder().build(), arguments, null) + .block(); + Assertions.assertEquals(expResult, result.replaceAll("\\n", "")); + } + + @Test + public void withCustomConverter() { + PromptTemplateConfig promptTemplate = PromptTemplateConfig.builder() + .withTemplate("{{#each input}}{{this}}{{/each}}") + .withTemplateFormat("handlebars") + .build(); + + HandlebarsPromptTemplate instance = new HandlebarsPromptTemplate(promptTemplate); + + ContextVariableTypeConverter converter = ContextVariableTypeConverter.builder( + Foo.class) + .toPromptString(Foo::getVal) + .build(); + KernelFunctionArguments arguments = KernelFunctionArguments.builder() + .withVariable("input", ContextVariable.of(new Foo("bar\"<>&"), converter)) + .build(); + + // Return from renderAsync is normalized to remove empty lines and leading/trailing whitespace + String expResult = "bar"<>&"; + + String result = instance.renderAsync(Kernel.builder().build(), arguments, + InvocationContext.builder() + .withContextVariableConverter(converter) + .build()) + .block(); + Assertions.assertEquals(expResult, result.replaceAll("\\n", "")); + } } \ No newline at end of file diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/hsqldb/HSQLDBVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/hsqldb/HSQLDBVectorStoreQueryProvider.java new file mode 100644 index 00000000..e8a0549d --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/hsqldb/HSQLDBVectorStoreQueryProvider.java @@ -0,0 +1,228 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.hsqldb; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; +import com.microsoft.semantickernel.exceptions.SKException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.lang.reflect.Field; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.time.OffsetDateTime; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import javax.sql.DataSource; + +public class HSQLDBVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider { + + @SuppressFBWarnings("EI_EXPOSE_REP2") + private HSQLDBVectorStoreQueryProvider( + DataSource dataSource, + String collectionsTable, + String prefixForCollectionTables, + int defaultVarCharLength) { + super( + dataSource, + collectionsTable, + prefixForCollectionTables, + buildSupportedKeyTypes(defaultVarCharLength), + buildSupportedDataTypes(defaultVarCharLength), + buildSupportedVectorTypes(defaultVarCharLength)); + } + + private static Map, String> buildSupportedVectorTypes(int defaultVarCharLength) { + HashMap, String> supportedVectorTypes = new HashMap<>(); + supportedVectorTypes.put(String.class, "VARCHAR(" + defaultVarCharLength + ")"); + supportedVectorTypes.put(List.class, "VARCHAR(" + defaultVarCharLength + ")"); + supportedVectorTypes.put(Collection.class, "VARCHAR(" + defaultVarCharLength + ")"); + return supportedVectorTypes; + } + + private static Map, String> buildSupportedDataTypes(int defaultVarCharLength) { + HashMap, String> supportedDataTypes = new HashMap<>(); + supportedDataTypes.put(String.class, "VARCHAR(" + defaultVarCharLength + ")"); + supportedDataTypes.put(Integer.class, "INTEGER"); + supportedDataTypes.put(int.class, "INTEGER"); + supportedDataTypes.put(Long.class, "BIGINT"); + supportedDataTypes.put(long.class, "BIGINT"); + supportedDataTypes.put(Float.class, "REAL"); + supportedDataTypes.put(float.class, "REAL"); + supportedDataTypes.put(Double.class, "DOUBLE"); + supportedDataTypes.put(double.class, "DOUBLE"); + supportedDataTypes.put(Boolean.class, "BOOLEAN"); + supportedDataTypes.put(boolean.class, "BOOLEAN"); + supportedDataTypes.put(OffsetDateTime.class, "TIMESTAMPTZ"); + return supportedDataTypes; + } + + private static HashMap, String> buildSupportedKeyTypes(int defaultVarCharLength) { + HashMap, String> supportedKeyTypes = new HashMap<>(); + supportedKeyTypes.put(String.class, "VARCHAR(" + defaultVarCharLength + ")"); + return supportedKeyTypes; + } + + private void setStatementValues(PreparedStatement statement, Object record, + List fields) { + for (int i = 0; i < fields.size(); ++i) { + VectorStoreRecordField field = fields.get(i); + try { + Field recordField = record.getClass().getDeclaredField(field.getName()); + recordField.setAccessible(true); + Object value = recordField.get(record); + + if (field instanceof VectorStoreRecordKeyField) { + statement.setObject(i + 1, (String) value); + } else if (field instanceof VectorStoreRecordVectorField) { + Class vectorType = record.getClass().getDeclaredField(field.getName()) + .getType(); + + // If the vector field is other than String, serialize it to JSON + if (vectorType.equals(String.class)) { + statement.setObject(i + 1, value); + } else { + // Serialize the vector to JSON + statement.setObject(i + 1, new ObjectMapper().writeValueAsString(value)); + } + } else { + statement.setObject(i + 1, value); + } + } catch (NoSuchFieldException | IllegalAccessException | SQLException e) { + throw new SKException("Failed to set statement values", e); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + } + + /** + * Upserts records into the collection. + * + * @param collectionName the collection name + * @param records the records to upsert + * @param recordDefinition the record definition + * @param options the upsert options + * @throws SKException if the upsert fails + */ + @Override + @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") + // SQL query is generated dynamically with valid identifiers + + public void upsertRecords(String collectionName, List records, + VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { + validateSQLidentifier(getCollectionTableName(collectionName)); + + List fields = recordDefinition.getAllFields(); + + String keyName = recordDefinition.getKeyField().getStorageName(); + + String updater = fields + .stream() + .map(VectorStoreRecordField::getStorageName) + .map(it -> "t." + it + "=vals." + it) + .collect(Collectors.joining(",")); + + String setter = fields + .stream() + .map(VectorStoreRecordField::getStorageName) + .map(it -> "vals." + it) + .collect(Collectors.joining(",")); + + String query = formatQuery( + "MERGE INTO %s AS t USING (VALUES (%s)) AS vals(%s) " + + "ON t.%s=vals.%s WHEN MATCHED THEN UPDATE SET %s " + + "WHEN NOT MATCHED THEN INSERT VALUES %s", + getCollectionTableName(collectionName), + getWildcardString(fields.size()), + getQueryColumnsFromFields(fields), + keyName, + keyName, + updater, + setter); + + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(query)) { + for (Object record : records) { + setStatementValues(statement, record, recordDefinition.getAllFields()); + statement.addBatch(); + } + + statement.executeBatch(); + } catch (SQLException e) { + throw new SKException("Failed to upsert records", e); + } + } + + /** + * Creates a new builder. + * + * @return the builder + */ + public static Builder builder() { + return new Builder(); + } + + public static class Builder + extends JDBCVectorStoreQueryProvider.Builder { + + private DataSource dataSource; + private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; + private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; + private int defaultVarCharLength = 255; + + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withDataSource(DataSource dataSource) { + this.dataSource = dataSource; + return this; + } + + /** + * Sets the collections table name. + * + * @param collectionsTable the collections table name + * @return the builder + */ + public Builder withCollectionsTable(String collectionsTable) { + this.collectionsTable = validateSQLidentifier(collectionsTable); + return this; + } + + /** + * Sets the prefix for collection tables. + * + * @param prefixForCollectionTables the prefix for collection tables + * @return the builder + */ + public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { + this.prefixForCollectionTables = validateSQLidentifier(prefixForCollectionTables); + return this; + } + + public Builder setDefaultVarCharLength(int defaultVarCharLength) { + this.defaultVarCharLength = defaultVarCharLength; + return this; + } + + public HSQLDBVectorStoreQueryProvider build() { + if (dataSource == null) { + throw new SKException("DataSource is required"); + } + + return new HSQLDBVectorStoreQueryProvider( + dataSource, + collectionsTable, + prefixForCollectionTables, + defaultVarCharLength); + } + + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java index 81387d65..4e349eb1 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java @@ -3,27 +3,28 @@ import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import reactor.core.publisher.Mono; -import reactor.core.scheduler.Schedulers; - +import java.util.List; import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.sql.DataSource; -import java.util.List; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; /** * A JDBC vector store. */ public class JDBCVectorStore implements SQLVectorStore { + private final DataSource dataSource; private final JDBCVectorStoreOptions options; private final SQLVectorStoreQueryProvider queryProvider; /** - * Creates a new instance of the {@link JDBCVectorStore}. - * If using this constructor, call {@link #prepareAsync()} before using the vector store. + * Creates a new instance of the {@link JDBCVectorStore}. If using this constructor, call + * {@link #prepareAsync()} before using the vector store. * * @param dataSource the connection * @param options the options @@ -55,8 +56,8 @@ public static Builder builder() { /** * Gets a collection from the vector store. * - * @param collectionName The name of the collection. - * @param options The options for the collection. + * @param collectionName The name of the collection. + * @param options The options for the collection. * @return The collection. */ @Override @@ -95,6 +96,27 @@ public VectorStoreRecordCollection getCollection( .build()); } + /** + * Gets a collection from the vector store. + * + * @param collectionName The name of the collection. + * @param recordClass The class type of the record. + * @param recordDefinition The record definition. + * @return The collection. + */ + public VectorStoreRecordCollection getCollection( + @Nonnull String collectionName, + @Nonnull Class recordClass, + @Nullable VectorStoreRecordDefinition recordDefinition) { + return getCollection( + collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(recordClass) + .withRecordDefinition(recordDefinition) + .withQueryProvider(this.queryProvider) + .build()); + } + /** * Gets the names of all collections in the vector store. * @@ -119,6 +141,7 @@ public Mono prepareAsync() { * Builder for creating a {@link JDBCVectorStore}. */ public static class Builder { + private DataSource dataSource; private JDBCVectorStoreOptions options; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index bb8e8d2a..9b2fa1b9 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -1,27 +1,19 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; import com.microsoft.semantickernel.data.vectorsearch.VectorOperations; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; -import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; -import com.microsoft.semantickernel.exceptions.SKException; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Nonnull; -import javax.sql.DataSource; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -30,22 +22,26 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.annotation.Nonnull; +import javax.sql.DataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class JDBCVectorStoreQueryProvider implements SQLVectorStoreQueryProvider { + private static final Logger LOGGER = LoggerFactory .getLogger(JDBCVectorStoreQueryProvider.class); private final Map, String> supportedKeyTypes; private final Map, String> supportedDataTypes; private final Map, String> supportedVectorTypes; - private final DataSource dataSource; + protected final DataSource dataSource; private final String collectionsTable; private final String prefixForCollectionTables; @@ -81,8 +77,24 @@ protected JDBCVectorStoreQueryProvider( supportedVectorTypes.put(Collection.class, "TEXT"); } + public JDBCVectorStoreQueryProvider( + @SuppressFBWarnings("EI_EXPOSE_REP2") @Nonnull DataSource dataSource, + @Nonnull String collectionsTable, + @Nonnull String prefixForCollectionTables, + @Nonnull HashMap, String> supportedKeyTypes, + @Nonnull Map, String> supportedDataTypes, + @Nonnull Map, String> supportedVectorTypes) { + this.dataSource = dataSource; + this.collectionsTable = collectionsTable; + this.prefixForCollectionTables = prefixForCollectionTables; + this.supportedKeyTypes = new HashMap<>(supportedKeyTypes); + this.supportedDataTypes = new HashMap<>(supportedDataTypes); + this.supportedVectorTypes = new HashMap<>(supportedVectorTypes); + } + /** * Creates a new builder. + * * @return the builder */ public static Builder builder() { @@ -91,6 +103,7 @@ public static Builder builder() { /** * Formats a wildcard string for a query. + * * @param wildcards the number of wildcards * @return the formatted wildcard string */ @@ -102,6 +115,7 @@ protected String getWildcardString(int wildcards) { /** * Gets the key column name from a key field. + * * @param keyField the key field * @return the key column name */ @@ -111,6 +125,7 @@ protected String getKeyColumnName(VectorStoreRecordField keyField) { /** * Formats the query columns from a record definition. + * * @param fields the fields to get the columns from * @return the formatted query columns */ @@ -123,8 +138,9 @@ protected String getQueryColumnsFromFields(List fields) /** * Formats the column names and types for a table. + * * @param fields the fields - * @param types the types + * @param types the types * @return the formatted column names and types */ protected String getColumnNamesAndTypes(List fields, @@ -172,8 +188,7 @@ public Map, String> getSupportedVectorTypes() { } /** - * Prepares the vector store. - * Executes any necessary setup steps for the vector store. + * Prepares the vector store. Executes any necessary setup steps for the vector store. * * @throws SKException if an error occurs while preparing the vector store */ @@ -236,12 +251,13 @@ public boolean collectionExists(String collectionName) { /** * Creates a collection. * - * @param collectionName the collection name + * @param collectionName the collection name * @param recordDefinition the record definition * @throws SKException if an error occurs while creating the collection */ @Override - @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") // SQL query is generated dynamically with valid identifiers + @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") + // SQL query is generated dynamically with valid identifiers public void createCollection(String collectionName, VectorStoreRecordDefinition recordDefinition) { @@ -342,13 +358,13 @@ public List getCollectionNames() { /** * Gets a list of records from the store. * - * @param collectionName the collection name - * @param keys the keys + * @param collectionName the collection name + * @param keys the keys * @param recordDefinition the record definition - * @param mapper the mapper - * @param options the options + * @param mapper the mapper + * @param options the options + * @param the record type * @return the records - * @param the record type * @throws SKException if an error occurs while getting the records */ @Override @@ -363,7 +379,18 @@ public List getRecords(String collectionName, List keys fields = recordDefinition.getNonVectorFields(); } - String query = formatQuery("SELECT %s FROM %s WHERE %s IN (%s)", + String query; + + if (options != null && options.isWildcardKeyMatching()) { + if (keys.size() > 1) { + throw new SKException("If using wildcard key matching, only one key is allowed"); + } + query = "SELECT %s FROM %s WHERE %s LIKE (%s)"; + } else { + query = "SELECT %s FROM %s WHERE %s IN (%s)"; + } + + query = formatQuery(query, getQueryColumnsFromFields(fields), getCollectionTableName(collectionName), getKeyColumnName(recordDefinition.getKeyField()), @@ -398,10 +425,10 @@ public void upsertRecords(String collectionName, List records, /** * Deletes records. * - * @param collectionName the collection name - * @param keys the keys + * @param collectionName the collection name + * @param keys the keys * @param recordDefinition the record definition - * @param options the options + * @param options the options * @throws SKException if an error occurs while deleting the records */ @Override @@ -426,7 +453,9 @@ public void deleteRecords(String collectionName, List keys, protected List getRecordsWithFilter(String collectionName, VectorStoreRecordDefinition recordDefinition, - VectorStoreRecordMapper mapper, GetRecordOptions options, String filter, + VectorStoreRecordMapper mapper, + GetRecordOptions options, + String filter, List parameters) { List fields; if (options.isIncludeVectors()) { @@ -462,17 +491,17 @@ protected List getRecordsWithFilter(String collectionName, } /** - * Vector search. - * Executes a vector search query and returns the results. - * The results are mapped to the specified record type using the provided mapper. - * The query is executed against the specified collection. + * Vector search. Executes a vector search query and returns the results. The results are mapped + * to the specified record type using the provided mapper. The query is executed against the + * specified collection. * - * @param the record type - * @param collectionName the collection name + * @param the record type + * @param collectionName the collection name * @param vector the vector to search with * @param options the search options * @param recordDefinition the record definition - * @param mapper the mapper, responsible for mapping the result set to the record type. + * @param mapper the mapper, responsible for mapping the result set to the record + * type. * @return the search results */ @Override @@ -529,7 +558,7 @@ public static String validateSQLidentifier(String identifier) { * Formats a query. * * @param query the query - * @param args the arguments + * @param args the arguments * @return the formatted query */ public String formatQuery(String query, String... args) { @@ -541,12 +570,14 @@ public String formatQuery(String query, String... args) { */ public static class Builder implements SQLVectorStoreQueryProvider.Builder { + private DataSource dataSource; private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; /** * Sets the data source. + * * @param dataSource the data source * @return the builder */ @@ -558,6 +589,7 @@ public Builder withDataSource(DataSource dataSource) { /** * Sets the collections table name. + * * @param collectionsTable the collections table name * @return the builder */ @@ -568,6 +600,7 @@ public Builder withCollectionsTable(String collectionsTable) { /** * Sets the prefix for collection tables. + * * @param prefixForCollectionTables the prefix for collection tables * @return the builder */ diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java index ce3413b7..f43fca6e 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -28,8 +28,7 @@ import reactor.core.scheduler.Schedulers; public class JDBCVectorStoreRecordCollection - implements SQLVectorStoreRecordCollection, - VectorizedSearch { + implements SQLVectorStoreRecordCollection { private final String collectionName; private final VectorStoreRecordDefinition recordDefinition; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java index 559bbefd..f9ea409b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java @@ -8,17 +8,16 @@ import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import com.microsoft.semantickernel.exceptions.SKException; -import org.apache.commons.lang3.tuple.Pair; -import redis.clients.jedis.args.SortingOrder; -import redis.clients.jedis.search.FTSearchParams; -import redis.clients.jedis.search.Query; - import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.List; import java.util.stream.Collectors; +import org.apache.commons.lang3.tuple.Pair; +import redis.clients.jedis.args.SortingOrder; +import redis.clients.jedis.search.FTSearchParams; public class RedisVectorStoreCollectionSearchMapping { + static final String VECTOR_SCORE_FIELD = "vector_score"; public static Pair buildQuery(List vector, diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java index efe56285..8bd3cdd2 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.vectorstorage.attributes; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import javax.annotation.Nullable; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -31,5 +33,7 @@ /** * Distance function to be used for to compute the distance between vectors. */ - String distanceFunction() default ""; + @Nullable + DistanceFunction distanceFunction(); + } \ No newline at end of file diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java index 98378254..a0b0a817 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java @@ -6,7 +6,6 @@ import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; import com.microsoft.semantickernel.exceptions.SKException; - import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Collections; @@ -21,6 +20,7 @@ * Represents a definition of a vector store record. */ public class VectorStoreRecordDefinition { + private final VectorStoreRecordKeyField keyField; private final List dataFields; private final List vectorFields; @@ -40,6 +40,7 @@ public List getDataFields() { /** * Gets the vector fields in the record definition. + * * @return List of VectorStoreRecordVectorField */ public List getVectorFields() { @@ -48,6 +49,7 @@ public List getVectorFields() { /** * Gets all fields in the record definition. + * * @return List of VectorStoreRecordField */ public List getAllFields() { @@ -56,6 +58,7 @@ public List getAllFields() { /** * Gets the non-vector fields in the record definition. + * * @return List of VectorStoreRecordField */ public List getNonVectorFields() { @@ -64,6 +67,7 @@ public List getNonVectorFields() { /** * Checks if the record definition contains a field with the specified name. + * * @param fieldName The name of the field to check. * @return boolean */ @@ -73,6 +77,7 @@ public boolean containsField(String fieldName) { /** * Gets the field with the specified name. + * * @param fieldName The name of the field to get. * @return VectorStoreRecordField */ @@ -114,6 +119,7 @@ private static VectorStoreRecordDefinition checkFields( /** * Create a VectorStoreRecordDefinition from a collection of fields. + * * @param fields The fields to create the definition from. * @return VectorStoreRecordDefinition */ @@ -138,6 +144,7 @@ public static VectorStoreRecordDefinition fromFields(List recordClass) .withFieldType(field.getType()) .withDimensions(vectorAttribute.dimensions()) .withIndexKind(IndexKind.fromString(vectorAttribute.indexKind())) - .withDistanceFunction( - DistanceFunction.fromString(vectorAttribute.distanceFunction())) + .withDistanceFunction(vectorAttribute.distanceFunction()) .build()); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java index 0025394a..444d4533 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java @@ -7,10 +7,26 @@ * Options for getting a record. */ public class GetRecordOptions { + private final boolean includeVectors; - public GetRecordOptions(boolean includeVectors) { + private final boolean wildcardKeyMatching; + + public GetRecordOptions( + boolean includeVectors) { + this.includeVectors = includeVectors; + this.wildcardKeyMatching = false; + } + + public GetRecordOptions( + boolean includeVectors, + boolean wildcardKeyMatching) { this.includeVectors = includeVectors; + this.wildcardKeyMatching = wildcardKeyMatching; + } + + public boolean isWildcardKeyMatching() { + return wildcardKeyMatching; } /** @@ -23,7 +39,9 @@ public static Builder builder() { } public static class Builder implements SemanticKernelBuilder { + private boolean includeVectors; + private boolean wildcardKeyMatching = false; /** * Sets whether to include vectors. @@ -36,6 +54,20 @@ public Builder includeVectors(boolean includeVectors) { return this; } + /** + * Sets whether to use wildcard key matching. Default is false. Wildcard key matching allows + * for matching multiple ids, for instance using "LIKE 'a%'" on a SQL query. + *

+ * NOTE: Currently this is only supported by the SQL connectors. + * + * @param wildcardKeyMatching whether to use wildcard key matching + * @return GetRecordOptions.Builder + */ + public Builder setWildcardKeyMatching(boolean wildcardKeyMatching) { + this.wildcardKeyMatching = wildcardKeyMatching; + return this; + } + /** * Builds the options. * @@ -43,7 +75,7 @@ public Builder includeVectors(boolean includeVectors) { */ @Override public GetRecordOptions build() { - return new GetRecordOptions(includeVectors); + return new GetRecordOptions(includeVectors, wildcardKeyMatching); } } diff --git a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java index 5255407d..7d5b62e5 100644 --- a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java +++ b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java @@ -6,6 +6,7 @@ import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import java.util.List; @@ -24,19 +25,19 @@ public class Hotel { private final String description; @JsonProperty("summaryEmbedding1") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "euclidean") + @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) private final List euclidean; @JsonProperty("summaryEmbedding2") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "cosineDistance") + @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List cosineDistance; @JsonProperty("summaryEmbedding3") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "cosineSimilarity") + @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.COSINE_SIMILARITY) private final List cosineSimilarity; @JsonProperty("summaryEmbedding4") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "dotProduct") + @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.DOT_PRODUCT) private final List dotProduct; @VectorStoreRecordDataAttribute private double rating; From ac87802ed1aa0615aaf0c5988ce72fd31bed6279 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Wed, 25 Sep 2024 15:09:35 -0700 Subject: [PATCH 061/277] Update redis tests failure logs --- ...edisHashSetVectorStoreRecordCollectionTest.java | 14 ++++++++------ .../RedisJsonVectorStoreRecordCollectionTest.java | 14 ++++++++------ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index 975e7679..8e07fdd2 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -369,6 +369,8 @@ private static Stream provideSearchParameters() { ); } + private final String indexingFailureMessage = "If you are running in a slow machine, data might not be indexed yet. Adjust setup delay if needed"; + @ParameterizedTest @MethodSource("provideSearchParameters") public void search(RecordCollectionOptions options, String embeddingName) { @@ -385,9 +387,9 @@ public void search(RecordCollectionOptions options, String embeddingName) { // Embeddings similar to the third hotel List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); assertNotNull(results); - assertEquals(VectorSearchOptions.DEFAULT_RESULT_LIMIT, results.size()); + assertEquals(VectorSearchOptions.DEFAULT_RESULT_LIMIT, results.size(), indexingFailureMessage); // The third hotel should be the most similar - assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId(), indexingFailureMessage); // Score should be different than zero assertNotEquals(0.0, results.get(0).getScore()); assertNull(results.get(0).getRecord().getEuclidean()); @@ -410,9 +412,9 @@ public void searchWithVectors(RecordCollectionOptions options, String embeddingN // Embeddings similar to the third hotel List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); assertNotNull(results); - assertEquals(VectorSearchOptions.DEFAULT_RESULT_LIMIT, results.size()); + assertEquals(VectorSearchOptions.DEFAULT_RESULT_LIMIT, results.size(), indexingFailureMessage); // The third hotel should be the most similar - assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId(), indexingFailureMessage); assertNotNull(results.get(0).getRecord().getEuclidean()); } @@ -434,8 +436,8 @@ public void searchWithOffSet(RecordCollectionOptions options, String embeddingNa // Embeddings similar to the third hotel List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); assertNotNull(results); - assertEquals(4, results.size()); + assertEquals(4, results.size(), indexingFailureMessage); // The first hotel should be the most similar - assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); + assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId(), indexingFailureMessage); } } diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index 792e9669..df41e84e 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -369,6 +369,8 @@ private static Stream provideSearchParameters() { ); } + private final String indexingFailureMessage = "If you are running in a slow machine, data might not be indexed yet. Adjust setup delay if needed"; + @ParameterizedTest @MethodSource("provideSearchParameters") public void search(RecordCollectionOptions options, String embeddingName) { @@ -385,9 +387,9 @@ public void search(RecordCollectionOptions options, String embeddingName) { // Embeddings similar to the third hotel List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); assertNotNull(results); - assertEquals(VectorSearchOptions.DEFAULT_RESULT_LIMIT, results.size()); + assertEquals(VectorSearchOptions.DEFAULT_RESULT_LIMIT, results.size(), indexingFailureMessage); // The third hotel should be the most similar - assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId(), indexingFailureMessage); // Score should be different than zero assertNotEquals(0.0, results.get(0).getScore()); assertNull(results.get(0).getRecord().getEuclidean()); @@ -410,9 +412,9 @@ public void searchWithVectors(RecordCollectionOptions options, String embeddingN // Embeddings similar to the third hotel List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); assertNotNull(results); - assertEquals(VectorSearchOptions.DEFAULT_RESULT_LIMIT, results.size()); + assertEquals(VectorSearchOptions.DEFAULT_RESULT_LIMIT, results.size(), indexingFailureMessage); // The third hotel should be the most similar - assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId(), indexingFailureMessage); assertNotNull(results.get(0).getRecord().getEuclidean()); } @@ -434,8 +436,8 @@ public void searchWithOffSet(RecordCollectionOptions options, String embeddingNa // Embeddings similar to the third hotel List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); assertNotNull(results); - assertEquals(4, results.size()); + assertEquals(4, results.size(), indexingFailureMessage); // The first hotel should be the most similar - assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); + assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId(), indexingFailureMessage); } } From 41ee0b8aea08571b618cb8407d42946f01bfddf0 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Thu, 26 Sep 2024 11:57:38 +0100 Subject: [PATCH 062/277] add hsqldb to test --- .../memory/jdbc/JDBCVectorStoreTest.java | 67 ++++++++++++++----- 1 file changed, 50 insertions(+), 17 deletions(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java index 33928262..a774ce52 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java @@ -1,5 +1,6 @@ package com.microsoft.semantickernel.tests.connectors.memory.jdbc; +import com.microsoft.semantickernel.connectors.data.hsqldb.HSQLDBVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStore; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions; import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; @@ -8,6 +9,7 @@ import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.sqlite.SQLiteVectorStoreQueryProvider; import com.mysql.cj.jdbc.MysqlDataSource; +import org.hsqldb.jdbc.JDBCDataSourceFactory; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.postgresql.ds.PGSimpleDataSource; @@ -19,22 +21,30 @@ import org.testcontainers.utility.DockerImageName; import javax.sql.DataSource; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.List; +import java.util.Map; +import java.util.Properties; import com.microsoft.semantickernel.tests.connectors.memory.jdbc.JDBCVectorStoreRecordCollectionTest.QueryProvider; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @Testcontainers public class JDBCVectorStoreTest { + @Container private static final MySQLContainer MYSQL_CONTAINER = new MySQLContainer<>("mysql:5.7.34"); - private static final DockerImageName PGVECTOR = DockerImageName.parse("pgvector/pgvector:pg16").asCompatibleSubstituteFor("postgres"); + private static final DockerImageName PGVECTOR = DockerImageName.parse("pgvector/pgvector:pg16") + .asCompatibleSubstituteFor("postgres"); @Container - private static final PostgreSQLContainer POSTGRESQL_CONTAINER = new PostgreSQLContainer<>(PGVECTOR); + private static final PostgreSQLContainer POSTGRESQL_CONTAINER = new PostgreSQLContainer<>( + PGVECTOR); private JDBCVectorStore buildVectorStore(QueryProvider provider) { SQLVectorStoreQueryProvider queryProvider; @@ -48,8 +58,8 @@ private JDBCVectorStore buildVectorStore(QueryProvider provider) { mysqlDataSource.setPassword(MYSQL_CONTAINER.getPassword()); dataSource = mysqlDataSource; queryProvider = MySQLVectorStoreQueryProvider.builder() - .withDataSource(dataSource) - .build(); + .withDataSource(dataSource) + .build(); break; case PostgreSQL: PGSimpleDataSource pgSimpleDataSource = new PGSimpleDataSource(); @@ -58,30 +68,53 @@ private JDBCVectorStore buildVectorStore(QueryProvider provider) { pgSimpleDataSource.setPassword(POSTGRESQL_CONTAINER.getPassword()); dataSource = pgSimpleDataSource; queryProvider = PostgreSQLVectorStoreQueryProvider.builder() - .withDataSource(dataSource) - .build(); + .withDataSource(dataSource) + .build(); break; case SQLite: SQLiteDataSource sqliteDataSource = new SQLiteDataSource(); sqliteDataSource.setUrl("jdbc:sqlite:file:test"); dataSource = sqliteDataSource; queryProvider = SQLiteVectorStoreQueryProvider.builder() - .withDataSource(sqliteDataSource) - .build(); + .withDataSource(sqliteDataSource) + .build(); + break; + case HSQLDB: + try { + Path file = Files.createTempFile("testdb", ".db"); + file.toFile().deleteOnExit(); + + Properties properties = new Properties(); + properties.putAll( + Map.of( + "url", "jdbc:hsqldb:file:" + file.toFile().getAbsolutePath() + + ";sql.syntax_mys=true", + "user", "SA", + "password", "" + ) + ); + + dataSource = JDBCDataSourceFactory.createDataSource(properties); + } catch (Exception e) { + throw new RuntimeException(e); + } + + queryProvider = HSQLDBVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build(); break; default: throw new IllegalArgumentException("Unknown query provider: " + provider); } - JDBCVectorStore vectorStore = JDBCVectorStore.builder() - .withDataSource(dataSource) - .withOptions( - JDBCVectorStoreOptions.builder() - .withQueryProvider(queryProvider) - .build() - ) - .build(); + .withDataSource(dataSource) + .withOptions( + JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build() + ) + .build(); vectorStore.prepareAsync().block(); return vectorStore; @@ -99,7 +132,7 @@ public void getCollectionNamesAsync(QueryProvider provider) { for (String collectionName : collectionNames) { vectorStore.getCollection(collectionName, - JDBCVectorStoreRecordCollectionOptions.builder() + JDBCVectorStoreRecordCollectionOptions.builder() .withRecordClass(Hotel.class) .build()).createCollectionAsync().block(); } From 96e5a3ee3cc36012e86f726dbe50759a57b3ccb9 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Thu, 26 Sep 2024 12:25:15 +0100 Subject: [PATCH 063/277] changelog --- CHANGELOG.md | 1 + README.md | 2 +- .../semantickernel/samples/syntaxexamples/RunAll.java | 6 +++++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34931b26..32939807 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Upgraded to openai sdk 1.0.0-beta.11 - Added convenience method `FunctionInvocation.withResultTypeAutoConversion` which sets the return type and registers a type converter based on Jackson for the return type. +- Added localization support for error/debug messages ### Bug Fixes diff --git a/README.md b/README.md index 2f00d959..27585762 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ in a different direction, but also to consider the impact on the larger ecosyste To learn more and get started: - Read the [documentation](https://learn.microsoft.com/en-us/semantic-kernel/overview/?tabs=Java&pivots=programming-language-java) -- Learn how to [contribute](https://learn.microsoft.com/en-us/semantic-kernel/get-started/contributing?tabs=Java&pivots=programming-language-java) to the project +- Learn how to [contribute](https://learn.microsoft.com/en-us/semantic-kernel/support/contributing?tabs=Java&pivots=programming-language-java) to the project - Join the [Discord community](https://aka.ms/SKDiscord) - Attend [regular office hours and SK community events](COMMUNITY.md) - Follow the team on our [blog](https://aka.ms/sk/blog) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/RunAll.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/RunAll.java index 7d704fe6..f634a331 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/RunAll.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/RunAll.java @@ -6,6 +6,8 @@ import com.microsoft.semantickernel.samples.syntaxexamples.chatcompletion.Example33_Chat; import com.microsoft.semantickernel.samples.syntaxexamples.chatcompletion.Example44_MultiChatCompletion; import com.microsoft.semantickernel.samples.syntaxexamples.chatcompletion.Example63_ChatCompletionPrompts; +import com.microsoft.semantickernel.samples.syntaxexamples.chatcompletion.responseschema.Example_ChatWithResponseFormat; +import com.microsoft.semantickernel.samples.syntaxexamples.chatcompletion.responseschema.Example_ChatWithResponseFormatToolCall; import com.microsoft.semantickernel.samples.syntaxexamples.configuration.Example08_RetryHandler; import com.microsoft.semantickernel.samples.syntaxexamples.configuration.Example41_HttpClientUsage; import com.microsoft.semantickernel.samples.syntaxexamples.configuration.Example58_ConfigureExecutionSettings; @@ -66,7 +68,9 @@ public static void main(String[] args) { Example63_ChatCompletionPrompts::main, Example64_MultiplePromptTemplates::main, Example69_MutableKernelPlugin::main, - KernelFunctionYaml_Example::main); + KernelFunctionYaml_Example::main, + Example_ChatWithResponseFormat::main, + Example_ChatWithResponseFormatToolCall::main); Scanner scanner = new Scanner(System.in); mains.forEach(mainMethod -> { From 9ed8312fcb154c4415d9cbb4cc988d90ae1fc325 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 26 Sep 2024 08:27:06 -0700 Subject: [PATCH 064/277] Update change log --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32939807..223a91d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ - Added convenience method `FunctionInvocation.withResultTypeAutoConversion` which sets the return type and registers a type converter based on Jackson for the return type. - Added localization support for error/debug messages +- Add vector search to experimental vector stores. + - Approximate vector search for Azure AI Search, Redis and JDBC with Postgres. + - Exhaustive vector search for VolatileVectorStore and default JDBC query provider, MySQL and SQLite. ### Bug Fixes From 1e530ad7ecf3281ff114803d96b29b12cc58091b Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 26 Sep 2024 08:34:59 -0700 Subject: [PATCH 065/277] Add HSQLDB to CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 223a91d2..8859c23f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ - Added localization support for error/debug messages - Add vector search to experimental vector stores. - Approximate vector search for Azure AI Search, Redis and JDBC with Postgres. - - Exhaustive vector search for VolatileVectorStore and default JDBC query provider, MySQL and SQLite. + - Exhaustive vector search for VolatileVectorStore and default JDBC query provider, MySQL, SQLite and HSQLDB. ### Bug Fixes From 5a8de543b8c2b85d8c663380d73d9818cfcd9f8e Mon Sep 17 00:00:00 2001 From: GitHub Date: Thu, 26 Sep 2024 17:32:47 +0100 Subject: [PATCH 066/277] [maven-release-plugin] prepare release java-1.3.0 --- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 5 ++--- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- pom.xml | 9 ++++----- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-bom/pom.xml | 4 ++-- semantickernel-experimental/pom.xml | 2 +- 20 files changed, 25 insertions(+), 27 deletions(-) diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index 7dd0a007..e0d89a04 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.0-SNAPSHOT + 1.3.0 ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index fd1f418b..a32264a8 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.0-SNAPSHOT + 1.3.0 ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index 302a354e..6ebd035d 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -1,13 +1,12 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.3.0-SNAPSHOT + 1.3.0 ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 0eab06c3..5de41c79 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.3.0-SNAPSHOT + 1.3.0 ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index 027022de..82b01532 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.0-SNAPSHOT + 1.3.0 ../pom.xml diff --git a/pom.xml b/pom.xml index ebff91a9..0cb6edf2 100644 --- a/pom.xml +++ b/pom.xml @@ -1,13 +1,12 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.3.0-SNAPSHOT + 1.3.0 pom https://www.github.com/microsoft/semantic-kernel @@ -687,7 +686,7 @@ // Copyright (c) Microsoft. All rights reserved. - + @@ -835,6 +834,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - HEAD + java-1.3.0 diff --git a/samples/pom.xml b/samples/pom.xml index 92f61803..81dd5673 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.0-SNAPSHOT + 1.3.0 ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index 6a3f835c..1c84931d 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.3.0-SNAPSHOT + 1.3.0 ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index f46399ab..23dff050 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.3.0-SNAPSHOT + 1.3.0 ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index d9ab68bc..3e1c65dd 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.3.0-SNAPSHOT + 1.3.0 ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index 71a74025..dd90e0e8 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.3.0-SNAPSHOT + 1.3.0 ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index 8917499e..b480860c 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.3.0-SNAPSHOT + 1.3.0 ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index 33e90396..4ca08d65 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.3.0-SNAPSHOT + 1.3.0 ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 53d501db..9a04cde0 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.3.0-SNAPSHOT + 1.3.0 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index c94ccdca..611515be 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.3.0-SNAPSHOT + 1.3.0 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index 17b7143f..66c98e1d 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.3.0-SNAPSHOT + 1.3.0 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index 1c13db81..9c56e5a5 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.3.0-SNAPSHOT + 1.3.0 ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index d0dafe5d..605ad8a7 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.0-SNAPSHOT + 1.3.0 ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 61d6baf2..625b8a0e 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.3.0-SNAPSHOT + 1.3.0 pom Semantic Kernel Java BOM @@ -271,6 +271,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - HEAD + java-1.3.0 diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 1c1f5b92..8207b635 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.0-SNAPSHOT + 1.3.0 semantickernel-experimental From 8bee95b4f0e2aaf8ff558aee81177ed8436b85c8 Mon Sep 17 00:00:00 2001 From: GitHub Date: Thu, 26 Sep 2024 17:32:47 +0100 Subject: [PATCH 067/277] [maven-release-plugin] prepare for next development iteration --- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- pom.xml | 4 ++-- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-bom/pom.xml | 4 ++-- semantickernel-experimental/pom.xml | 2 +- 20 files changed, 22 insertions(+), 22 deletions(-) diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index e0d89a04..67c44fd2 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.0 + 1.3.1-SNAPSHOT ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index a32264a8..e8a3f6a9 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.0 + 1.3.1-SNAPSHOT ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index 6ebd035d..56f93045 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.0 + 1.3.1-SNAPSHOT ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 5de41c79..ff618703 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.3.0 + 1.3.1-SNAPSHOT ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index 82b01532..e6924679 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.0 + 1.3.1-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 0cb6edf2..c571f636 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.0 + 1.3.1-SNAPSHOT pom https://www.github.com/microsoft/semantic-kernel @@ -834,6 +834,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.3.0 + HEAD diff --git a/samples/pom.xml b/samples/pom.xml index 81dd5673..e3772428 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.0 + 1.3.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index 1c84931d..eebdadd3 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.3.0 + 1.3.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 23dff050..43266da4 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.3.0 + 1.3.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index 3e1c65dd..de435a33 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.3.0 + 1.3.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index dd90e0e8..2e40f729 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.3.0 + 1.3.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index b480860c..f6b51a70 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.3.0 + 1.3.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index 4ca08d65..1bee5197 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.3.0 + 1.3.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 9a04cde0..f042a3ef 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.3.0 + 1.3.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index 611515be..a7bd5f6e 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.3.0 + 1.3.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index 66c98e1d..0c1d8dbf 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.3.0 + 1.3.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index 9c56e5a5..c9b95005 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.3.0 + 1.3.1-SNAPSHOT ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index 605ad8a7..6f63e152 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.0 + 1.3.1-SNAPSHOT ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 625b8a0e..dd284fe5 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.3.0 + 1.3.1-SNAPSHOT pom Semantic Kernel Java BOM @@ -271,6 +271,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.3.0 + HEAD diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 8207b635..c7b890ad 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.0 + 1.3.1-SNAPSHOT semantickernel-experimental From a2bfa8b0ab055b6e254a188c64e3bc39481c0250 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 1 Oct 2024 22:47:58 -0700 Subject: [PATCH 068/277] Adjust IndexKind and DistanceFunction defaults --- .../openai/chatcompletion/JsonSchemaTest.java | 2 +- .../tests/connectors/memory/jdbc/Hotel.java | 3 +- .../tests/connectors/memory/redis/Hotel.java | 8 ++-- .../memory/InMemoryVolatileVectorStore.java | 3 +- .../memory/VectorStoreWithAzureAISearch.java | 4 +- .../memory/VectorStoreWithJDBC.java | 2 +- .../memory/VectorStoreWithRedis.java | 3 +- .../HandlebarsPromptTemplateTest.java | 2 +- ...rchVectorStoreCollectionCreateMapping.java | 6 ++- .../jdbc/JDBCVectorStoreQueryProvider.java | 12 ++++-- .../PostgreSQLVectorDistanceFunction.java | 8 ++-- .../postgres/PostgreSQLVectorIndexKind.java | 11 +++-- .../PostgreSQLVectorStoreQueryProvider.java | 17 ++++---- ...disVectorStoreCollectionCreateMapping.java | 8 ++-- .../VolatileVectorStoreRecordCollection.java | 7 +-- .../VectorStoreRecordVectorAttribute.java | 7 ++- .../definition/DistanceFunction.java | 26 +++-------- .../vectorstorage/definition/IndexKind.java | 43 ++++++++++--------- .../VectorStoreRecordDefinition.java | 2 +- .../VectorStoreRecordVectorField.java | 15 +++---- 20 files changed, 93 insertions(+), 96 deletions(-) diff --git a/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/JsonSchemaTest.java b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/JsonSchemaTest.java index fad9ab80..d0cdc0d9 100644 --- a/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/JsonSchemaTest.java +++ b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/JsonSchemaTest.java @@ -18,7 +18,7 @@ public void jacksonGenerationTest() throws JsonProcessingException { Assertions.assertEquals("foo", format.getJsonSchema().getName()); Assertions.assertTrue(format.getJsonSchema().getSchema() - .replaceAll("\n", "") + .replaceAll("\\r\\n|\\r|\\n", "") .replaceAll(" +", "") .contains( "\"type\":\"object\",\"properties\":{\"bar\":{}}")); diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java index 2a1394f1..c2b93c2d 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java @@ -6,6 +6,7 @@ import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; import java.util.List; @@ -37,7 +38,7 @@ public class Hotel { private final List dotProduct; @JsonProperty("indexedSummaryEmbedding") - @VectorStoreRecordVectorAttribute(dimensions = 8, indexKind = "hnsw", distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) + @VectorStoreRecordVectorAttribute(dimensions = 8, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) private final List indexedEuclidean; @VectorStoreRecordDataAttribute private double rating; diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java index 185d134d..bf46f77e 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java @@ -6,6 +6,8 @@ import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; + import java.util.List; public class Hotel { @@ -24,15 +26,15 @@ public class Hotel { private final String description; @JsonProperty("summaryEmbedding1") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) + @VectorStoreRecordVectorAttribute(dimensions = 8, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) private final List euclidean; @JsonProperty("summaryEmbedding2") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE) + @VectorStoreRecordVectorAttribute(dimensions = 8, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List cosineDistance; @JsonProperty("summaryEmbedding3") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.DOT_PRODUCT) + @VectorStoreRecordVectorAttribute(dimensions = 8, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.DOT_PRODUCT) private final List dotProduct; @VectorStoreRecordDataAttribute private double rating; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java index a7b0d663..f5d9d7f1 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java @@ -20,6 +20,7 @@ import java.util.Map; import java.util.stream.Collectors; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -44,7 +45,7 @@ static class GitHubFile { private final String description; @VectorStoreRecordDataAttribute private final String link; - @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = "Hnsw", distanceFunction = DistanceFunction.COSINE_DISTANCE) + @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List embedding; public GitHubFile( diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java index 0a8e5e7c..9de59ed7 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java @@ -28,6 +28,8 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; + +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -58,7 +60,7 @@ static class GitHubFile { private final String description; @VectorStoreRecordDataAttribute private final String link; - @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = "Hnsw", distanceFunction = DistanceFunction.COSINE_DISTANCE) + @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List embedding; public GitHubFile() { diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java index 161fc044..6cd4953f 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java @@ -49,7 +49,7 @@ static class GitHubFile { private final String description; @VectorStoreRecordDataAttribute private final String link; - @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = "Hnsw", distanceFunction = DistanceFunction.COSINE_DISTANCE) + @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List embedding; public GitHubFile() { diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java index be5af4b7..aee00a5b 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java @@ -28,6 +28,7 @@ import java.util.stream.Collectors; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import redis.clients.jedis.JedisPooled; @@ -52,7 +53,7 @@ public static class GitHubFile { private final String description; @VectorStoreRecordDataAttribute private final String link; - @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = "Hnsw", distanceFunction = DistanceFunction.COSINE_DISTANCE) + @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List embedding; public GitHubFile() { diff --git a/semantickernel-api/src/test/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplateTest.java b/semantickernel-api/src/test/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplateTest.java index cf20ca51..228d64dc 100644 --- a/semantickernel-api/src/test/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplateTest.java +++ b/semantickernel-api/src/test/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplateTest.java @@ -156,7 +156,7 @@ public void testSerializesObject() { String result = instance.renderAsync(Kernel.builder().build(), arguments, null) .block(); - Assertions.assertEquals(expResult, result.replaceAll("\\n", "")); + Assertions.assertEquals(expResult, result.replaceAll("\\r\\n|\\r|\\n", "")); } @Test diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java index 2c733b33..7afe3c35 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java @@ -10,6 +10,8 @@ import com.azure.search.documents.indexes.models.VectorSearchAlgorithmConfiguration; import com.azure.search.documents.indexes.models.VectorSearchAlgorithmMetric; import com.azure.search.documents.indexes.models.VectorSearchProfile; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; @@ -31,7 +33,7 @@ private static String getAlgorithmConfigName(VectorStoreRecordVectorField vector private static VectorSearchAlgorithmMetric getAlgorithmMetric( @Nonnull VectorStoreRecordVectorField vectorField) { - if (vectorField.getDistanceFunction() == null) { + if (vectorField.getDistanceFunction() == DistanceFunction.UNDEFINED) { return VectorSearchAlgorithmMetric.COSINE; } @@ -50,7 +52,7 @@ private static VectorSearchAlgorithmMetric getAlgorithmMetric( private static VectorSearchAlgorithmConfiguration getAlgorithmConfig( @Nonnull VectorStoreRecordVectorField vectorField) { - if (vectorField.getIndexKind() == null) { + if (vectorField.getIndexKind() == IndexKind.UNDEFINED) { return new HnswAlgorithmConfiguration(getAlgorithmConfigName(vectorField)) .setParameters(new HnswParameters().setMetric(getAlgorithmMetric(vectorField))); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index 9b2fa1b9..0c07588c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -5,6 +5,7 @@ import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; @@ -263,7 +264,9 @@ public void createCollection(String collectionName, // No approximate search is supported in JDBCVectorStoreQueryProvider if (recordDefinition.getVectorFields().stream() - .anyMatch(field -> field.getIndexKind() != null)) { + .anyMatch( + field -> field.getIndexKind() != null && field.getIndexKind() != IndexKind.FLAT + && field.getIndexKind() != IndexKind.UNDEFINED)) { LOGGER .warn(String.format("Indexes are not supported in %s. Ignoring indexKind property.", this.getClass().getName())); @@ -532,9 +535,10 @@ public List> search(String collectionName, List records = getRecordsWithFilter(collectionName, recordDefinition, mapper, new GetRecordOptions(true), filter, parameters); - DistanceFunction distanceFunction = vectorField.getDistanceFunction() == null - ? DistanceFunction.EUCLIDEAN_DISTANCE - : vectorField.getDistanceFunction(); + DistanceFunction distanceFunction = vectorField + .getDistanceFunction() == DistanceFunction.UNDEFINED + ? DistanceFunction.EUCLIDEAN_DISTANCE + : vectorField.getDistanceFunction(); return VectorOperations.exactSimilaritySearch(records, vector, vectorField, distanceFunction, options); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java index ae2a6bff..001a5499 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java @@ -5,7 +5,7 @@ public enum PostgreSQLVectorDistanceFunction { L2("vector_l2_ops", "<->"), COSINE("vector_cosine_ops", "<=>"), INNER_PRODUCT("vector_ip_ops", - "<#>"); + "<#>"), UNDEFINED(null, null); private final String value; private final String operator; @@ -24,10 +24,6 @@ public String getOperator() { } public static PostgreSQLVectorDistanceFunction fromDistanceFunction(DistanceFunction function) { - if (function == null) { - return null; - } - switch (function) { case EUCLIDEAN_DISTANCE: return L2; @@ -35,6 +31,8 @@ public static PostgreSQLVectorDistanceFunction fromDistanceFunction(DistanceFunc return COSINE; case DOT_PRODUCT: return INNER_PRODUCT; + case UNDEFINED: + return UNDEFINED; default: throw new IllegalArgumentException("Unsupported distance function: " + function); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java index 939caaed..1a04fb18 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java @@ -4,7 +4,7 @@ import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; public enum PostgreSQLVectorIndexKind { - HNSW("hnsw"), IVFFLAT("ivfflat"); + HNSW("hnsw"), IVFFLAT("ivfflat"), UNDEFINED(null); private final String value; @@ -17,15 +17,14 @@ public String getValue() { } public static PostgreSQLVectorIndexKind fromIndexKind(IndexKind indexKind) { - if (indexKind == null) { - return null; - } - switch (indexKind) { case HNSW: return HNSW; - case FLAT: + case IVFFLAT: return IVFFLAT; + case FLAT: + case UNDEFINED: + return UNDEFINED; default: throw new IllegalArgumentException("Unsupported index kind: " + indexKind); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index df4f0127..15f7971b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -164,12 +164,12 @@ private String createIndexForVectorField(String collectionName, PostgreSQLVectorDistanceFunction distanceFunction = PostgreSQLVectorDistanceFunction .fromDistanceFunction(vectorField.getDistanceFunction()); - // If indexKind is not specified, no index is created - // and pgvector performs exact nearest neighbor search. - if (indexKind == null) { + // If there is no approximate search index associated to the vector field, + // there is no need to create an index and pgvector performs exact nearest neighbor search. + if (indexKind == PostgreSQLVectorIndexKind.UNDEFINED) { return null; } - if (distanceFunction == null) { + if (distanceFunction == PostgreSQLVectorDistanceFunction.UNDEFINED) { throw new SKException( "Distance function is required for vector field: " + vectorField.getName()); } @@ -358,10 +358,11 @@ public List> search(String collectionName, PostgreSQLVectorDistanceFunction distanceFunction = PostgreSQLVectorDistanceFunction .fromDistanceFunction(vectorField.getDistanceFunction()); - // If indexKind is not specified, there is no index associated to the vector field - // and pgvector performs exact nearest neighbor search. - // If indexKind is specified, a distance function is required. - if (indexKind != null && distanceFunction == null) { + // If there is no approximate search index associated to the vector field, + // there is no index defined in the database and pgvector performs exact nearest neighbor search. + // If indexKind is defined, distance function is required. + if (indexKind != PostgreSQLVectorIndexKind.UNDEFINED + && distanceFunction == PostgreSQLVectorDistanceFunction.UNDEFINED) { throw new SKException( "Distance function is required for vector field: " + vectorField.getName()); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java index 2613be90..a9d7a329 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; @@ -32,8 +34,8 @@ public class RedisVectorStoreCollectionCreateMapping { private static String getAlgorithmMetric( VectorStoreRecordVectorField vectorField) { - if (vectorField.getDistanceFunction() == null) { - return RedisVectorDistanceMetric.EUCLIDEAN; + if (vectorField.getDistanceFunction() == DistanceFunction.UNDEFINED) { + return RedisVectorDistanceMetric.COSINE; } switch (vectorField.getDistanceFunction()) { @@ -51,7 +53,7 @@ private static String getAlgorithmMetric( private static Schema.VectorField.VectorAlgo getAlgorithmConfig( VectorStoreRecordVectorField vectorField) { - if (vectorField.getIndexKind() == null) { + if (vectorField.getIndexKind() == IndexKind.UNDEFINED) { return Schema.VectorField.VectorAlgo.HNSW; } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java index ec82700d..3b40c16b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java @@ -276,9 +276,10 @@ public Mono>> searchAsync(List vector, : (VectorStoreRecordVectorField) recordDefinition .getField(effectiveOptions.getVectorFieldName()); - DistanceFunction distanceFunction = vectorField.getDistanceFunction() == null - ? DistanceFunction.EUCLIDEAN_DISTANCE - : vectorField.getDistanceFunction(); + DistanceFunction distanceFunction = vectorField + .getDistanceFunction() == DistanceFunction.UNDEFINED + ? DistanceFunction.EUCLIDEAN_DISTANCE + : vectorField.getDistanceFunction(); List records = VolatileVectorStoreCollectionSearchMapping.filterRecords( new ArrayList<>(getCollection().values()), effectiveOptions.getVectorSearchFilter(), diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java index 8bd3cdd2..a72c2c69 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java @@ -2,6 +2,8 @@ package com.microsoft.semantickernel.data.vectorstorage.attributes; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; + import javax.annotation.Nullable; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; @@ -28,12 +30,13 @@ /** * Type of index to be used for the vector. */ - String indexKind() default ""; + @Nullable + IndexKind indexKind() default IndexKind.UNDEFINED; /** * Distance function to be used for to compute the distance between vectors. */ @Nullable - DistanceFunction distanceFunction(); + DistanceFunction distanceFunction() default DistanceFunction.UNDEFINED; } \ No newline at end of file diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java index aacdbba6..8b2577a7 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java @@ -17,7 +17,11 @@ public enum DistanceFunction { /** * Euclidean distance function. Also known as L2 distance. */ - EUCLIDEAN_DISTANCE("euclidean"); + EUCLIDEAN_DISTANCE("euclidean"), + /** + * No distance function specified. It will default to the database's default distance function. + */ + UNDEFINED(null); private final String value; @@ -28,24 +32,4 @@ public enum DistanceFunction { public String getValue() { return value; } - - /** - * Converts a string to a DistanceFunction. - * If the string is null or empty, the method returns DistanceFunction.COSINE_SIMILARITY. - * - * @param text the string to convert - * @return the DistanceFunction - */ - public static DistanceFunction fromString(String text) { - if (text == null || text.isEmpty()) { - return null; - } - - for (DistanceFunction b : DistanceFunction.values()) { - if (b.value.equalsIgnoreCase(text)) { - return b; - } - } - throw new IllegalArgumentException("No distance function with value " + text + " found"); - } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java index 08d330b0..cda1598b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java @@ -2,7 +2,28 @@ package com.microsoft.semantickernel.data.vectorstorage.definition; public enum IndexKind { - HNSW("Hnsw"), FLAT("Flat"); + /** + * Hierarchical Navigable Small World, which performs an approximate nearest neighbour (ANN) search. + */ + HNSW("Hnsw"), + + /** + * Flat index, which performs an exact nearest neighbour search. + * Also referred to as exhaustive k nearest neighbor in some databases. + * High recall accuracy, but slower and more expensive than HNSW. + * Better with smaller datasets. + */ + FLAT("Flat"), + + /** + * Inverted file index, which performs an approximate nearest neighbour (ANN) search. + */ + IVFFLAT("IVFFlat"), + + /** + * No index specified. It will default to the database's default index. + */ + UNDEFINED(null); private final String value; @@ -13,24 +34,4 @@ public enum IndexKind { public String getValue() { return value; } - - /** - * Converts a string to an IndexKind. - * If the string is null or empty, the method returns IndexKind.FLAT. - * - * @param text the string to convert - * @return the IndexKind - */ - public static IndexKind fromString(String text) { - if (text == null || text.isEmpty()) { - return null; - } - - for (IndexKind b : IndexKind.values()) { - if (b.value.equalsIgnoreCase(text)) { - return b; - } - } - throw new IllegalArgumentException("No index kind with value " + text + " found"); - } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java index a0b0a817..eeedfe73 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java @@ -203,7 +203,7 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) .withStorageName(storageName) .withFieldType(field.getType()) .withDimensions(vectorAttribute.dimensions()) - .withIndexKind(IndexKind.fromString(vectorAttribute.indexKind())) + .withIndexKind(vectorAttribute.indexKind()) .withDistanceFunction(vectorAttribute.distanceFunction()) .build()); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java index 47f47fd4..e5b94898 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java @@ -9,9 +9,7 @@ */ public class VectorStoreRecordVectorField extends VectorStoreRecordField { private final int dimensions; - @Nullable private final IndexKind indexKind; - @Nullable private final DistanceFunction distanceFunction; public static Builder builder() { @@ -37,8 +35,9 @@ public VectorStoreRecordVectorField( @Nullable DistanceFunction distanceFunction) { super(name, storageName, fieldType); this.dimensions = dimensions; - this.indexKind = indexKind; - this.distanceFunction = distanceFunction; + this.indexKind = indexKind == null ? IndexKind.UNDEFINED : indexKind; + this.distanceFunction = distanceFunction == null ? DistanceFunction.UNDEFINED + : distanceFunction; } /** @@ -55,7 +54,6 @@ public int getDimensions() { * * @return the index kind */ - @Nullable public IndexKind getIndexKind() { return indexKind; } @@ -65,7 +63,6 @@ public IndexKind getIndexKind() { * * @return the distance function */ - @Nullable public DistanceFunction getDistanceFunction() { return distanceFunction; } @@ -73,10 +70,8 @@ public DistanceFunction getDistanceFunction() { public static class Builder extends VectorStoreRecordField.Builder { private int dimensions; - @Nullable - private IndexKind indexKind; - @Nullable - private DistanceFunction distanceFunction; + private IndexKind indexKind = IndexKind.UNDEFINED; + private DistanceFunction distanceFunction = DistanceFunction.UNDEFINED; /** * Sets the number of dimensions in the vector. From df5bf066f0cc7192deb17a4aa773b321028657da Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 1 Oct 2024 22:58:41 -0700 Subject: [PATCH 069/277] Fix SQLite tests file creation --- .../JDBCVectorStoreRecordCollectionTest.java | 21 +++++++++++++++---- .../memory/jdbc/JDBCVectorStoreTest.java | 11 ++++++---- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java index ffd6bba4..ec63d9ec 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -17,6 +17,8 @@ import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import com.mysql.cj.jdbc.MysqlDataSource; + +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -61,6 +63,16 @@ public enum QueryProvider { HSQLDB } + static Path createTempDbFile(String prefix) { + try { + Path file = Files.createTempFile(prefix, ".db"); + file.toFile().deleteOnExit(); + return file; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + private JDBCVectorStoreRecordCollection buildRecordCollection(QueryProvider provider, @Nonnull String collectionName) { SQLVectorStoreQueryProvider queryProvider; @@ -88,17 +100,18 @@ private JDBCVectorStoreRecordCollection buildRecordCollection(QueryProvid .build(); break; case SQLite: + Path sqliteDb = createTempDbFile("sqliteDb"); SQLiteDataSource sqliteDataSource = new SQLiteDataSource(); - sqliteDataSource.setUrl("jdbc:sqlite:file:testdb"); + sqliteDataSource.setUrl("jdbc:sqlite:file:" + sqliteDb.toFile().getAbsolutePath()); dataSource = sqliteDataSource; + queryProvider = SQLiteVectorStoreQueryProvider.builder() - .withDataSource(sqliteDataSource) + .withDataSource(dataSource) .build(); break; case HSQLDB: try { - Path file = Files.createTempFile("testdb", ".db"); - file.toFile().deleteOnExit(); + Path file = createTempDbFile("testHSQLDB"); Properties properties = new Properties(); properties.putAll( diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java index a774ce52..f3a8744d 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java @@ -21,6 +21,7 @@ import org.testcontainers.utility.DockerImageName; import javax.sql.DataSource; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; @@ -30,6 +31,7 @@ import com.microsoft.semantickernel.tests.connectors.memory.jdbc.JDBCVectorStoreRecordCollectionTest.QueryProvider; +import static com.microsoft.semantickernel.tests.connectors.memory.jdbc.JDBCVectorStoreRecordCollectionTest.createTempDbFile; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -72,17 +74,18 @@ private JDBCVectorStore buildVectorStore(QueryProvider provider) { .build(); break; case SQLite: + Path sqliteDb = createTempDbFile("testSQLite"); SQLiteDataSource sqliteDataSource = new SQLiteDataSource(); - sqliteDataSource.setUrl("jdbc:sqlite:file:test"); + sqliteDataSource.setUrl("jdbc:sqlite:file:" + sqliteDb.toFile().getAbsolutePath()); dataSource = sqliteDataSource; + queryProvider = SQLiteVectorStoreQueryProvider.builder() - .withDataSource(sqliteDataSource) + .withDataSource(dataSource) .build(); break; case HSQLDB: try { - Path file = Files.createTempFile("testdb", ".db"); - file.toFile().deleteOnExit(); + Path file = createTempDbFile("testHSQLDB"); Properties properties = new Properties(); properties.putAll( From e8daf1e9afb6e9882f63892b975113139e9d68bb Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 1 Oct 2024 23:01:02 -0700 Subject: [PATCH 070/277] Update --- .../semantickernel/tests/connectors/memory/redis/Hotel.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java index bf46f77e..81b962bb 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java @@ -30,7 +30,7 @@ public class Hotel { private final List euclidean; @JsonProperty("summaryEmbedding2") - @VectorStoreRecordVectorAttribute(dimensions = 8, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE) + @VectorStoreRecordVectorAttribute(dimensions = 8) private final List cosineDistance; @JsonProperty("summaryEmbedding3") From b9d935a0d4e35cb2bee0a1ded0a322d95f258d93 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:48:19 +0100 Subject: [PATCH 071/277] Add plugin for document text splitting --- .../semantickernel-syntax-examples/pom.xml | 13 +- .../rag/DocumentSplittingExample.java | 81 ++++ samples/semantickernel-sample-plugins/pom.xml | 4 +- .../pom.xml | 65 +++ .../semantic/kernel/rag/splitting/Chunk.java | 16 + .../rag/splitting/ChunkEndCondition.java | 22 ++ .../rag/splitting/ChunkPostProcessor.java | 9 + .../kernel/rag/splitting/Document.java | 11 + .../rag/splitting/OverlapCondition.java | 19 + .../kernel/rag/splitting/Splitter.java | 372 ++++++++++++++++++ .../kernel/rag/splitting/TextSplitter.java | 29 ++ .../rag/splitting/TrivialChunkFilter.java | 17 + .../rag/splitting/document/TextDocument.java | 22 ++ .../overlap/CountOverlapCondition.java | 39 ++ .../splitting/overlap/NoOverlapCondition.java | 18 + .../overlap/PercentageOverlapCondition.java | 46 +++ .../postprocessors/NoOpPostProcessor.java | 20 + .../postprocessors/RemoveWhitespace.java | 21 + .../splitconditions/CountSplitCondition.java | 32 ++ .../splitconditions/NewLineSplitter.java | 17 + .../splitconditions/ParagraphSplitter.java | 24 ++ .../splitconditions/RegexSplitter.java | 97 +++++ .../splitconditions/SentenceSplitter.java | 17 + .../splitconditions/SplitPoints.java | 35 ++ .../splitconditions/WhiteSpaceFilter.java | 21 + .../splitconditions/WordSplitter.java | 17 + .../src/test/java/DocumentSplitTest.java | 172 ++++++++ 27 files changed, 1254 insertions(+), 2 deletions(-) create mode 100644 samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/rag/DocumentSplittingExample.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/Chunk.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/ChunkEndCondition.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/ChunkPostProcessor.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/Document.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/OverlapCondition.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/Splitter.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/TextSplitter.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/TrivialChunkFilter.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/document/TextDocument.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/CountOverlapCondition.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/NoOverlapCondition.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/PercentageOverlapCondition.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/postprocessors/NoOpPostProcessor.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/postprocessors/RemoveWhitespace.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/CountSplitCondition.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/NewLineSplitter.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/ParagraphSplitter.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/RegexSplitter.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/SentenceSplitter.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/SplitPoints.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/WhiteSpaceFilter.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/WordSplitter.java create mode 100644 samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/test/java/DocumentSplitTest.java diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 43266da4..42258751 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -1,5 +1,6 @@ - + 4.0.0 com.microsoft.semantic-kernel @@ -81,6 +82,16 @@ com.microsoft.semantic-kernel semantickernel-aiservices-google + + com.microsoft.semantic-kernel + semantickernel-text-splitter-plugin + ${project.version} + + + org.apache.pdfbox + pdfbox + 3.0.3 + com.google.cloud diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/rag/DocumentSplittingExample.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/rag/DocumentSplittingExample.java new file mode 100644 index 00000000..b081514b --- /dev/null +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/rag/DocumentSplittingExample.java @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.syntaxexamples.rag; + +import com.microsoft.semantic.kernel.rag.splitting.Chunk; +import com.microsoft.semantic.kernel.rag.splitting.Document; +import com.microsoft.semantic.kernel.rag.splitting.Splitter; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; +import java.util.List; +import org.apache.pdfbox.io.RandomAccessReadBuffer; +import org.apache.pdfbox.pdfparser.PDFParser; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.text.PDFTextStripper; +import reactor.core.publisher.Flux; + +public class DocumentSplittingExample { + + private static String BENEFITS_DOC = "https://raw.githubusercontent.com/Azure-Samples/azure-search-openai-demo-java/refs/heads/main/data/Benefit_Options.pdf"; + + private static class PDFDocument implements Document { + + private final byte[] pdf; + + private PDFDocument(byte[] pdf) { + this.pdf = pdf; + } + + @Override + public Flux getContent() { + try { + PDFParser parser = new PDFParser( + RandomAccessReadBuffer.createBufferFromStream(new ByteArrayInputStream(pdf))); + PDDocument document = parser.parse(); + String text = new PDFTextStripper().getText(document); + + return Flux.just(text); + } catch (IOException e) { + return Flux.error(e); + } + } + } + + public static void main(String[] args) throws IOException, InterruptedException { + byte[] pdfBytes = getPdfDoc(); + PDFDocument pdfDoc = new PDFDocument(pdfBytes); + + Splitter splitter = Splitter + .builder() + .maxParagraphsPerChunk(4) + .overlapNPercent(30.0f) + .trimWhitespace() + .build(); + + List chunks = splitter + .splitDocument(pdfDoc) + .collectList() + .block(); + + chunks + .forEach(chunk -> { + System.out.println("========="); + System.out.println(chunk.getContents()); + }); + } + + private static byte[] getPdfDoc() throws IOException, InterruptedException { + HttpResponse doc = HttpClient.newHttpClient() + .send(HttpRequest.newBuilder() + .GET() + .uri(URI.create(BENEFITS_DOC)) + .build(), + BodyHandlers.ofByteArray()); + return doc.body(); + } + +} diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index a7bd5f6e..26df728c 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -1,5 +1,6 @@ - + 4.0.0 com.microsoft.semantic-kernel @@ -15,5 +16,6 @@ semantickernel-openapi-plugin semantickernel-presidio-plugin + semantickernel-text-splitter-plugin diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml new file mode 100644 index 00000000..e0157c82 --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml @@ -0,0 +1,65 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-sample-plugins + 1.3.1-SNAPSHOT + ../pom.xml + + + semantickernel-text-splitter-plugin + semantickernel-text-splitter-plugin + jar + + + + + com.microsoft.semantic-kernel + semantickernel-bom + ${project.version} + pom + import + + + + + + + com.microsoft.semantic-kernel + semantickernel-api + + + + org.apache.logging.log4j + log4j-api + runtime + + + org.apache.logging.log4j + log4j-core + runtime + + + org.apache.logging.log4j + log4j-slf4j2-impl + runtime + + + com.fasterxml.jackson.core + jackson-databind + compile + + + com.fasterxml.jackson.core + jackson-core + compile + + + org.junit.jupiter + junit-jupiter-api + test + + + diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/Chunk.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/Chunk.java new file mode 100644 index 00000000..f815656e --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/Chunk.java @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting; + +public class Chunk { + + private final String chunk; + + public Chunk(String chunk) { + this.chunk = chunk; + } + + public String getContents() { + return chunk; + } + +} diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/ChunkEndCondition.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/ChunkEndCondition.java new file mode 100644 index 00000000..bcbfd0f1 --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/ChunkEndCondition.java @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting; + +/** + * Defines the condition that should be met for a chunk to be considered full. + */ +public interface ChunkEndCondition { + + /** + * Accepts a string and returns the number of character that should be considered as the end of + * the FIRST chunk within the string. This method will be subsiquently called until all pages + * are found. + *

+ * Return -1 if the value does not contain enough characters to be considered as a full chunk. + * + * @param value the value to be checked + * @return the index of the character that should be considered as the end of the first chunk in + * the string + */ + public int getEndOfNextChunk(String value); + +} diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/ChunkPostProcessor.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/ChunkPostProcessor.java new file mode 100644 index 00000000..4a1d19f8 --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/ChunkPostProcessor.java @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting; + +/** + * A post processor that processes a chunk after it has been split. + */ +public interface ChunkPostProcessor { + Chunk process(Chunk chunk); +} diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/Document.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/Document.java new file mode 100644 index 00000000..730d2727 --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/Document.java @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting; + +import reactor.core.publisher.Flux; + +/** + * A document to be read and split into chunks. + */ +public interface Document { + Flux getContent(); +} diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/OverlapCondition.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/OverlapCondition.java new file mode 100644 index 00000000..fc816807 --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/OverlapCondition.java @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting; + +/** + * Defines how much overlap is allowed between two pages. + */ +public interface OverlapCondition { + + /** + * Returns the index of the first character that should be considered as the beginning of the + * overlap. + * + * @param chunk the chunk to be checked + * @return the index of the first character that should be considered as the beginning of the + * overlap + */ + public int getOverlapIndex(String chunk); + +} diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/Splitter.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/Splitter.java new file mode 100644 index 00000000..f2623357 --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/Splitter.java @@ -0,0 +1,372 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting; + +import com.microsoft.semantic.kernel.rag.splitting.overlap.CountOverlapCondition; +import com.microsoft.semantic.kernel.rag.splitting.overlap.NoOverlapCondition; +import com.microsoft.semantic.kernel.rag.splitting.overlap.PercentageOverlapCondition; +import com.microsoft.semantic.kernel.rag.splitting.postprocessors.RemoveWhitespace; +import com.microsoft.semantic.kernel.rag.splitting.splitconditions.CountSplitCondition; +import com.microsoft.semantic.kernel.rag.splitting.splitconditions.NewLineSplitter; +import com.microsoft.semantic.kernel.rag.splitting.splitconditions.ParagraphSplitter; +import com.microsoft.semantic.kernel.rag.splitting.splitconditions.SentenceSplitter; +import com.microsoft.semantic.kernel.rag.splitting.splitconditions.WhiteSpaceFilter; +import com.microsoft.semantic.kernel.rag.splitting.splitconditions.WordSplitter; +import com.microsoft.semantickernel.exceptions.SKException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import reactor.core.publisher.Flux; + +/** + * Splits a document into chunks based on supplied chunking strategy. + *

+ * The chunking strategies and conditions are somewhat soft limits. In scenarios where trivial + * chunks or tokens would be formed by the chunking strategy (for instance a chunk of only a few + * words would be formed), these will be merged into the previous chunk. As such it is possible that + * the chunks returned may be larger than the specified chunking strategy would imply, after other + * trivial chunks have been merged into them. + */ +public class Splitter { + + private static final Logger LOGGER = LoggerFactory.getLogger(Splitter.class.getName()); + + private final List chunkEndConditions; + private final OverlapCondition overlapCondition; + private final TrivialChunkFilter trivialChunkFilter; + private final ChunkPostProcessor chunkPostProcessor; + + public Splitter( + List chunkEndConditions, + OverlapCondition overlapCondition, + TrivialChunkFilter trivialChunkFilter, + ChunkPostProcessor chunkPostProcessor) { + this.chunkEndConditions = Collections.unmodifiableList(chunkEndConditions); + this.overlapCondition = overlapCondition; + this.trivialChunkFilter = trivialChunkFilter; + this.chunkPostProcessor = chunkPostProcessor; + } + + /** + * Splits a document into chunks. + * + * @param document the document to split + * @return chunks + */ + public Flux splitDocument(Document document) { + return splitDocument(document, chunkEndConditions, overlapCondition, trivialChunkFilter, + chunkPostProcessor); + } + + /** + * Splits a document into chunks. + * + * @param document the document to split + * @param chunkEndConditions the conditions that determine the end of a chunk + * @param overlapCondition the condition that determines the overlap between chunks + * @param trivialChunkFilter the filter that determines if a chunk is trivial + * @param chunkPostProcessor the post processor to apply to the chunks + * @return a flux of chunks + */ + public static Flux splitDocument( + Document document, + List chunkEndConditions, + OverlapCondition overlapCondition, + TrivialChunkFilter trivialChunkFilter, + ChunkPostProcessor chunkPostProcessor) { + + return document + .getContent() + //TODO: Make the chunking work on true streaming data + .reduce("", (a, b) -> a + b) + .flatMapMany(doc -> { + List chunks = chunkDocument( + chunkEndConditions, + overlapCondition, + trivialChunkFilter, + chunkPostProcessor, + doc); + + return Flux.fromIterable(chunks); + }); + } + + private static List chunkDocument(List chunkEndConditions, + OverlapCondition overlapCondition, TrivialChunkFilter trivialChunkFilter, + ChunkPostProcessor chunkPostProcessor, String doc) { + List chunks = new ArrayList<>(); + + int previousChunkEndIndex = -1; + + while (doc != null && doc.length() > 0) { + + String finalDoc = doc; + + Optional index = chunkEndConditions + .stream() + .map(condition -> condition.getEndOfNextChunk(finalDoc)) + .filter(i -> i != -1) + .min(Integer::compareTo); + + if (index.isPresent()) { + String chunkText = doc.substring(0, index.get()); + + if (chunkText.length() <= previousChunkEndIndex) { + LOGGER.warn( + "This entier chunk consists of overlapped data, this will result in infinite loop. Skipping this chunk."); + + // previous chunk should already contain this text..skip it + doc = doc.substring(previousChunkEndIndex, doc.length()); + + previousChunkEndIndex = 0; + continue; + } + + int overlapIndex = overlapCondition.getOverlapIndex(chunkText); + previousChunkEndIndex = chunkText.length() - overlapIndex; + doc = doc.substring(overlapIndex, doc.length()); + + chunks.add(new Chunk(chunkText)); + } else { + chunks.add(new Chunk(doc)); + break; + } + } + + chunks = mergeTrivialChunks(chunks, trivialChunkFilter); + chunks = tidyChunks(chunks, chunkPostProcessor); + return chunks; + } + + /** + * Tidies up the chunks by applying the post processor. + * + * @param chunks the list of chunks to tidy + * @param chunkPostProcessor the post processor to apply + * @return the list of tidied chunks + */ + private static List tidyChunks(List chunks, + ChunkPostProcessor chunkPostProcessor) { + return chunks + .stream() + .map(chunkPostProcessor::process) + .toList(); + } + + /** + * Merges chunks considered trivial into the previous chunk. + * + * @param chunks the list of chunks to merge + * @param trivialChunkFilter the filter to determine if a chunk is trivial + * @return the list of chunks with trivial chunks merged + */ + private static List mergeTrivialChunks(List chunks, + TrivialChunkFilter trivialChunkFilter) { + + List result = new ArrayList<>(); + Chunk current = null; + for (int i = 0; i < chunks.size(); i++) { + if (current == null) { + current = chunks.get(i); + } else { + current = new Chunk( + current.getContents() + chunks.get(i).getContents()); + } + + if (!trivialChunkFilter.isTrivialChunk(current.getContents())) { + result.add(current); + current = null; + } + } + + if (current != null) { + Chunk last = result.remove(result.size() - 1); + result.add(new Chunk(last.getContents() + current.getContents())); + } + + return result; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + + private List chunkEndConditions = new ArrayList<>(); + private OverlapCondition overlapCondition = new NoOverlapCondition(); + private TrivialChunkFilter trivialChunkFilter = new WhiteSpaceFilter(10); + private ChunkPostProcessor chunkPostProcessor; + + /** + * Splits the document into chunks based on the number of sentences. + * + * @param maxSentencesPerChunk the number of sentences per chunk + * @return the builder + */ + public Builder maxSentencesPerChunk(int maxSentencesPerChunk) { + return addChunkEndCondition( + new CountSplitCondition(maxSentencesPerChunk, new SentenceSplitter())); + } + + /** + * Splits the document into chunks based on the number of words. + * + * @param maxWordsPerChunk the number of words per chunk + * @return the builder + */ + public Builder maxWordsPerChunk(int maxWordsPerChunk) { + return addChunkEndCondition( + new CountSplitCondition(maxWordsPerChunk, new WordSplitter())); + } + + /** + * Splits the document into chunks based on the number of lines. + * + * @param maxLinesPerChunk the number of lines per chunk + * @return the builder + */ + public Builder maxLinesPerChunk(int maxLinesPerChunk) { + return addChunkEndCondition( + new CountSplitCondition(maxLinesPerChunk, new NewLineSplitter())); + } + + /** + * Splits the document into chunks based on the number of paragraphs. + *

+ * NOTE: The ParagraphSplitter is not perfect, see {@link ParagraphSplitter} for more + * information. + * + * @param maxParagraphsPerChunk the number of paragraphs per chunk + * @return the builder + */ + public Builder maxParagraphsPerChunk(int maxParagraphsPerChunk) { + return addChunkEndCondition( + new CountSplitCondition(maxParagraphsPerChunk, new ParagraphSplitter())); + } + + /** + * Overlaps chunks by the given number of lines. + * + * @param overlap the number of characters to overlap + * @return the builder + */ + public Builder overlapNLines(int overlap) { + return setOverlapCondition(new CountOverlapCondition(overlap, new NewLineSplitter())); + } + + /** + * Overlaps chunks by the given number of sentences. + * + * @param overlap the number of sentences to overlap + * @return the builder + */ + public Builder overlapNSentences(int overlap) { + return setOverlapCondition(new CountOverlapCondition(overlap, new SentenceSplitter())); + } + + /** + * Overlaps chunks by the given number of words. + * + * @param overlap the number of words to overlap + * @return the builder + */ + public Builder overlapNWords(int overlap) { + return setOverlapCondition(new CountOverlapCondition(overlap, new WordSplitter())); + } + + /** + * Overlaps chunks by the given percentage. Percentage is calculated based on the number of + * characters in the chunk. Will split at the beginning of the word that gives the required + * percentage. + * + * @param overlap the percentage overlap + * @return the builder + */ + public Builder overlapNPercent(float overlap) { + return setOverlapCondition(new PercentageOverlapCondition(overlap, new WordSplitter())); + } + + /** + * Merges chunks that are less than the given character count. + * + * @param length the length of the chunk + * @return the builder + */ + public Builder mergeChunksLessThanCharCount(int length) { + return setTrivialSplitFilter(new WhiteSpaceFilter(length)); + } + + /** + * Trims whitespace from all chunks. + * + * @return the builder + */ + public Builder trimWhitespace() { + return setChunkPostProcessor(new RemoveWhitespace()); + } + + /** + * Adds a chunk post processor to the builder. This is used to process the chunk after it + * has been split. For example, to remove unwanted whitespace. + * + * @param chunkPostProcessor the post processor to add + * @return the builder + */ + public Builder setChunkPostProcessor(ChunkPostProcessor chunkPostProcessor) { + this.chunkPostProcessor = chunkPostProcessor; + return this; + } + + /** + * Adds a page end condition to the builder. These are applied as OR conditions, i.e the + * page will be the size of the SMALLEST condition. + * + * @param chunkEndCondition + * @return + */ + public Builder addChunkEndCondition(ChunkEndCondition chunkEndCondition) { + chunkEndConditions.add(chunkEndCondition); + return this; + } + + /** + * Adds an overlap condition to the builder. This condition is used to determine the overlap + * between chunks. + * + * @param overlapCondition + * @return the builder + */ + public Builder setOverlapCondition(OverlapCondition overlapCondition) { + this.overlapCondition = overlapCondition; + return this; + } + + /** + * Adds a trivial split filter to the builder. Pages that match this filter will be merged + * into the previous chunk. + * + * @param trivialChunkFilter + * @return the builder + */ + public Builder setTrivialSplitFilter(TrivialChunkFilter trivialChunkFilter) { + this.trivialChunkFilter = trivialChunkFilter; + return this; + } + + public Splitter build() { + if (chunkEndConditions.size() == 0) { + throw new SKException( + "At least one chunk end condition must be provided"); + } + return new Splitter( + chunkEndConditions, + overlapCondition, + trivialChunkFilter, + chunkPostProcessor); + } + + } +} diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/TextSplitter.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/TextSplitter.java new file mode 100644 index 00000000..fca62683 --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/TextSplitter.java @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting; + +import com.microsoft.semantic.kernel.rag.splitting.splitconditions.SplitPoints; + +import java.util.List; + +/** + * Interface for splitting text into chunks. + */ +public interface TextSplitter { + + /** + * Get all the split points for the given document. + * + * @param doc the document to split + * @return the split points + */ + List getSplitPoints(String doc); + + /** + * Get the first n split points for the given document. + * + * @param doc the document to split + * @param n the number of split points to get + * @return the split points + */ + List getNSplitPoints(String doc, int n); +} \ No newline at end of file diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/TrivialChunkFilter.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/TrivialChunkFilter.java new file mode 100644 index 00000000..206a6da0 --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/TrivialChunkFilter.java @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting; + +/** + * A filter that determines if a chunk is trivial and should be merged with the previous chunk. + */ +public interface TrivialChunkFilter { + + /** + * Returns true if the chunk is trivial and should be merged with the previous chunk. + * + * @param doc the chunk to be checked + * @return true if the chunk is trivial + */ + public boolean isTrivialChunk(String doc); + +} \ No newline at end of file diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/document/TextDocument.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/document/TextDocument.java new file mode 100644 index 00000000..046d411d --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/document/TextDocument.java @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting.document; + +import com.microsoft.semantic.kernel.rag.splitting.Document; +import reactor.core.publisher.Flux; + +/** + * A document that contains a plain text string. + */ +public class TextDocument implements Document { + + private final String document; + + public TextDocument(String document) { + this.document = document; + } + + @Override + public Flux getContent() { + return Flux.just(document); + } +} diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/CountOverlapCondition.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/CountOverlapCondition.java new file mode 100644 index 00000000..d95ea30e --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/CountOverlapCondition.java @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting.overlap; + +import com.microsoft.semantic.kernel.rag.splitting.OverlapCondition; +import com.microsoft.semantic.kernel.rag.splitting.TextSplitter; +import com.microsoft.semantic.kernel.rag.splitting.splitconditions.SplitPoints; +import java.util.List; + +/** + * Overlap condition based on counting the number of "splits" i.e if splitting by words, would allow + * you to define an overlap as n words, or if by sentences, then n sentences. + */ +public class CountOverlapCondition implements OverlapCondition { + + // The number of splits to count to consider the condition met + private final int count; + + // The type of splitter to use to get the split points. + private final TextSplitter splitter; + + public CountOverlapCondition(int count, TextSplitter splitter) { + this.count = count; + this.splitter = splitter; + } + + @Override + public int getOverlapIndex(String chunk) { + List splitPoints = splitter.getSplitPoints(chunk); + + if (splitPoints.size() == 0) { + return 0; + } + + int i = Math.max(splitPoints.size() - count, 0); + i = Math.min(splitPoints.size() - 1, i); + + return splitPoints.get(i).getStart(); + } +} diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/NoOverlapCondition.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/NoOverlapCondition.java new file mode 100644 index 00000000..23595bcd --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/NoOverlapCondition.java @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting.overlap; + +import com.microsoft.semantic.kernel.rag.splitting.OverlapCondition; + +/** + * An overlap condition that does not overlap. + */ +public class NoOverlapCondition implements OverlapCondition { + + public NoOverlapCondition() { + } + + @Override + public int getOverlapIndex(String chunk) { + return chunk.length(); + } +} diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/PercentageOverlapCondition.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/PercentageOverlapCondition.java new file mode 100644 index 00000000..e4f804c3 --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/PercentageOverlapCondition.java @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting.overlap; + +import com.microsoft.semantic.kernel.rag.splitting.OverlapCondition; +import com.microsoft.semantic.kernel.rag.splitting.TextSplitter; +import com.microsoft.semantic.kernel.rag.splitting.splitconditions.SplitPoints; +import java.util.List; +import org.slf4j.Logger; + +/** + * Overlap condition based on percentage of the characters in the chunk. It will return the full + * split that gives atleast the percentage of the characters in the chunk, i.e if you are splitting + * based on sentence it will return the full sentence. + */ +public class PercentageOverlapCondition implements OverlapCondition { + + private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger( + PercentageOverlapCondition.class); + + private final float percentage; + private final TextSplitter splitter; + + public PercentageOverlapCondition(float percentage, TextSplitter splitter) { + if (percentage < 0 || percentage > 100) { + LOGGER.warn("Percentage must be between 0 and 100, clamping value to this range 100"); + percentage = Math.min(100, Math.max(0, percentage)); + } + this.percentage = percentage; + this.splitter = splitter; + } + + @Override + public int getOverlapIndex(String chunk) { + List splitPoints = splitter.getSplitPoints(chunk); + + float index = chunk.length() * (100.0f - percentage) / 100.0f; + + for (SplitPoints splitPoint : splitPoints) { + if (splitPoint.getEnd() > index) { + return splitPoint.getStart(); + } + } + + return 0; + } +} diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/postprocessors/NoOpPostProcessor.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/postprocessors/NoOpPostProcessor.java new file mode 100644 index 00000000..c1697e1a --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/postprocessors/NoOpPostProcessor.java @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting.postprocessors; + +import com.microsoft.semantic.kernel.rag.splitting.Chunk; +import com.microsoft.semantic.kernel.rag.splitting.ChunkPostProcessor; + +/** + * A post processor that does nothing. + */ +public class NoOpPostProcessor implements ChunkPostProcessor { + + public NoOpPostProcessor() { + } + + @Override + public Chunk process(Chunk chunk) { + return chunk; + } + +} diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/postprocessors/RemoveWhitespace.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/postprocessors/RemoveWhitespace.java new file mode 100644 index 00000000..497fa664 --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/postprocessors/RemoveWhitespace.java @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting.postprocessors; + +import com.microsoft.semantic.kernel.rag.splitting.Chunk; +import com.microsoft.semantic.kernel.rag.splitting.ChunkPostProcessor; + +/** + * A post processor that removes leading and trailing whitespace from a chunk. + */ +public class RemoveWhitespace implements ChunkPostProcessor { + + @Override + public Chunk process(Chunk chunk) { + return new Chunk(chunk.getContents() + .replaceAll("^\\s+", "") + .replaceAll("^[\n\r]+", "") + .replaceAll("\\s+$", "") + .replaceAll("[\n\r]+$", "")); + } + +} diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/CountSplitCondition.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/CountSplitCondition.java new file mode 100644 index 00000000..bea0c6a4 --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/CountSplitCondition.java @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting.splitconditions; + +import com.microsoft.semantic.kernel.rag.splitting.ChunkEndCondition; +import com.microsoft.semantic.kernel.rag.splitting.TextSplitter; +import java.util.List; + +/** + * Overlap condition based on counting the number of "splits" i.e if splitting by words, would allow + * you to define a chunk as n words, or if by sentences, then n sentences. + */ +public class CountSplitCondition implements ChunkEndCondition { + + private final int count; + private final TextSplitter splitter; + + public CountSplitCondition(int count, TextSplitter splitter) { + this.count = count; + this.splitter = splitter; + } + + @Override + public int getEndOfNextChunk(String doc) { + List splitPoints = splitter.getNSplitPoints(doc, count); + + if (splitPoints.size() < count) { + return splitPoints.get(splitPoints.size() - 1).getEnd(); + } + + return splitPoints.get(count - 1).getEnd(); + } +} diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/NewLineSplitter.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/NewLineSplitter.java new file mode 100644 index 00000000..f4f1a3d9 --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/NewLineSplitter.java @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting.splitconditions; + +import java.util.regex.Pattern; + +/** + * A splitter that splits text based on new lines. + */ +public class NewLineSplitter extends RegexSplitter { + + private static final Pattern NEW_LINE_SPLIT_REGEX = Pattern + .compile("[(\\r\\n)\\n\\r]+", Pattern.MULTILINE); + + public NewLineSplitter() { + super(NEW_LINE_SPLIT_REGEX); + } +} diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/ParagraphSplitter.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/ParagraphSplitter.java new file mode 100644 index 00000000..958079f8 --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/ParagraphSplitter.java @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting.splitconditions; + +import java.util.regex.Pattern; + +/** + * A splitter that splits text into paragraphs. + *

+ * This is a very simple and inaccurate splitter that splits text into paragraphs based on either: + *

+ *

    + *
  • Two or more consecutive newlines.
    • + *
  • A line ending in a end sentence character (i.e a period)
    • + *
+ */ +public class ParagraphSplitter extends RegexSplitter { + + private static final Pattern PARAGRAPH_SPLIT_REGEX = Pattern + .compile("[(\\r\\n)\\n\\r]{2,}|([\\.\\．!\\?。]+\\s*[(\\r\\n)\\n\\r])", Pattern.MULTILINE); + + public ParagraphSplitter() { + super(PARAGRAPH_SPLIT_REGEX); + } +} diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/RegexSplitter.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/RegexSplitter.java new file mode 100644 index 00000000..8876323b --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/RegexSplitter.java @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting.splitconditions; + +import com.microsoft.semantic.kernel.rag.splitting.TextSplitter; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.MatchResult; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * A text splitter that uses a regex to find the deliminators + */ +public abstract class RegexSplitter implements TextSplitter { + + public static final int DEFAULT_TRIVIAL_SPLIT_LENGTH = 10; + + private final Pattern pattern; + /** + * Splits below this length are considered trivial and will be merged + */ + private final int trivialSplitLength; + + /** + * Splitter that uses the given regex pattern to split the text + * + * @param pattern the regex pattern to split the text + */ + public RegexSplitter(Pattern pattern) { + this(pattern, DEFAULT_TRIVIAL_SPLIT_LENGTH); + } + + /** + * Splitter that uses the given regex pattern to split the text + * + * @param pattern the regex pattern to split the text + * @param trivialSplitLength the length of a split below which it will be considered trivial and + * will be merged + */ + public RegexSplitter(Pattern pattern, int trivialSplitLength) { + this.pattern = pattern; + this.trivialSplitLength = trivialSplitLength; + } + + @Override + public List getSplitPoints(String doc) { + return getNSplitPoints(doc, Integer.MAX_VALUE); + } + + @Override + public List getNSplitPoints(String doc, int n) { + Matcher matcher = pattern.matcher(doc); + + List points = matcher.results() + .collect(Collectors.toList()); + + List result = new ArrayList<>(); + + int previousEnd = 0; + for (MatchResult point : points) { + if (isTrivialSplit( + /* start= */ previousEnd, + /* end= */ point.start(), + doc, + trivialSplitLength)) { + continue; + } + result.add(new SplitPoints(previousEnd, point.end())); + previousEnd = point.end(); + if (result.size() >= n) { + break; + } + } + + if (result.size() < n && !isTrivialSplit(previousEnd, doc.length(), doc, 1)) { + result.add(new SplitPoints(previousEnd, doc.length())); + } + + if (result.isEmpty()) { + return List.of(new SplitPoints(0, doc.length())); + } + + return result; + } + + private boolean isTrivialSplit(int start, int end, String doc, int trivialSplitLength) { + String split = doc.substring(start, end); + + // Remove all split characters and whitespace + split = pattern.matcher(split).replaceAll(""); + split = split.replaceAll("\\s+", ""); + + return split.length() <= trivialSplitLength; + + } +} diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/SentenceSplitter.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/SentenceSplitter.java new file mode 100644 index 00000000..bfe5e7e5 --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/SentenceSplitter.java @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting.splitconditions; + +import java.util.regex.Pattern; + +/** + * A splitter that splits text into sentences. + */ +public class SentenceSplitter extends RegexSplitter { + + private static final Pattern SENTENCE_SPLIT_REGEX = Pattern + .compile("[\\.\\．!\\?。]+", Pattern.MULTILINE); + + public SentenceSplitter() { + super(SENTENCE_SPLIT_REGEX); + } +} diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/SplitPoints.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/SplitPoints.java new file mode 100644 index 00000000..5aa61d59 --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/SplitPoints.java @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting.splitconditions; + +/** + * A class that represents the start and end points of a split. I.e if splitting by word, these + * would be the indeces of the first and last char in the word within the chunk. + */ +public class SplitPoints { + + private final int start; + private final int end; + + public SplitPoints(int start, int end) { + this.start = start; + this.end = end; + } + + /** + * The index of the first character in the split. + * + * @return the index of the first character in the split + */ + public int getStart() { + return start; + } + + /** + * The index of the last character in the split. + * + * @return the index of the last character in the split + */ + public int getEnd() { + return end; + } +} \ No newline at end of file diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/WhiteSpaceFilter.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/WhiteSpaceFilter.java new file mode 100644 index 00000000..8dabbf40 --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/WhiteSpaceFilter.java @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting.splitconditions; + +import com.microsoft.semantic.kernel.rag.splitting.TrivialChunkFilter; + +/** + * A filter that merges chunks that have less than N non-whitespace characters. + */ +public class WhiteSpaceFilter implements TrivialChunkFilter { + + private final int trivialCharacterCount; + + public WhiteSpaceFilter(int trivialCharacterCount) { + this.trivialCharacterCount = trivialCharacterCount; + } + + @Override + public boolean isTrivialChunk(String doc) { + return doc.replaceAll("\\s+", "").length() < trivialCharacterCount; + } +} diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/WordSplitter.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/WordSplitter.java new file mode 100644 index 00000000..8a0e6852 --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/WordSplitter.java @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantic.kernel.rag.splitting.splitconditions; + +import java.util.regex.Pattern; + +/** + * A splitter that splits text into words. + */ +public class WordSplitter extends RegexSplitter { + + private static final Pattern WORD_SPLIT_REGEX = Pattern + .compile("[,、;: ()\\[\\]{}\t\n]+", Pattern.MULTILINE); + + public WordSplitter() { + super(WORD_SPLIT_REGEX, 1); + } +} diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/test/java/DocumentSplitTest.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/test/java/DocumentSplitTest.java new file mode 100644 index 00000000..edd05491 --- /dev/null +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/test/java/DocumentSplitTest.java @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft. All rights reserved. +import com.microsoft.semantic.kernel.rag.splitting.Chunk; +import com.microsoft.semantic.kernel.rag.splitting.Splitter; +import com.microsoft.semantic.kernel.rag.splitting.document.TextDocument; +import com.microsoft.semantic.kernel.rag.splitting.overlap.CountOverlapCondition; +import com.microsoft.semantic.kernel.rag.splitting.overlap.PercentageOverlapCondition; +import com.microsoft.semantic.kernel.rag.splitting.splitconditions.CountSplitCondition; +import com.microsoft.semantic.kernel.rag.splitting.splitconditions.NewLineSplitter; +import com.microsoft.semantic.kernel.rag.splitting.splitconditions.ParagraphSplitter; +import com.microsoft.semantic.kernel.rag.splitting.splitconditions.SentenceSplitter; +import com.microsoft.semantic.kernel.rag.splitting.splitconditions.WordSplitter; +import java.util.List; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class DocumentSplitTest { + + private final String NUMBERS = """ + 01 02 03 04 05 06 07 08 09 10. + 11 12 13 14 15 16 17 18 19 20. + 21 22 23 24 25 26 27 28 29 30. + 31 32 33 34 35 36 37 38 39 40. + 41 42 43 44 45 46 47 48 49 50. + 51 52 53 54 55 56 57 58 59 60. + 61 62 63 64 65 66 67 68 69 70. + 71 72 73 74 75 76 77 78 79 80. + 81 82 83 84 85 86 87 88 89 90. + 91 92 93 94 95 96 97 98 99 100. + """.stripIndent(); + + private final String PARAGRAPHS = """ + 01 02 03 04 05 06 07 08 09 10. 11 12 13 14 15 + 16 17 18 19 20. 21 22 23 24 + 25 26 27 28 29 30. + + 31 32 33 34 35 36 37 38 39 40. 41 42 43 44 + 45 46 47 48 49 50. + + 51 52 53 54 55 56 57 58 59 60. 61 + 62 63 64 65 66 67 68 69 70. 71 72 73 + 74 75 76 77 78 79 80. + + + 81 82 83 84 85 86 87 88 89 90. + + + 91 92 93 94 95 96 97 98 99 100. + """.stripIndent(); + + @Test + public void testWordSplit() { + List chunks = Splitter.builder() + .addChunkEndCondition(new CountSplitCondition(9, new WordSplitter())) + .setOverlapCondition(new PercentageOverlapCondition(20.0f, new WordSplitter())) + .trimWhitespace() + .build() + .splitDocument(new TextDocument(NUMBERS)) + .collectList() + .block(); + + Assertions.assertEquals(14, chunks.size()); + + Assertions.assertEquals(""" + 01 02 03 04 05 06 07 08 09""" + .stripIndent(), chunks.get(0).getContents()); + + Assertions.assertEquals(""" + 08 09 10. + 11 12 13 14 15 16""" + .stripIndent(), chunks.get(1).getContents()); + + Assertions.assertEquals(""" + 92 93 94 95 96 97 98 99 100.""" + .stripIndent(), chunks.get(13).getContents()); + } + + @Test + public void testSentenceSplit() { + List chunks = Splitter.builder() + .addChunkEndCondition(new CountSplitCondition(4, new SentenceSplitter())) + .setOverlapCondition(new CountOverlapCondition(2, new SentenceSplitter())) + .trimWhitespace() + .build() + .splitDocument(new TextDocument(NUMBERS)) + .collectList() + .block(); + + Assertions.assertEquals(4, chunks.size()); + + Assertions.assertEquals(""" + 01 02 03 04 05 06 07 08 09 10. + 11 12 13 14 15 16 17 18 19 20. + 21 22 23 24 25 26 27 28 29 30. + 31 32 33 34 35 36 37 38 39 40.""" + .stripIndent(), chunks.get(0).getContents()); + Assertions.assertEquals(""" + 21 22 23 24 25 26 27 28 29 30. + 31 32 33 34 35 36 37 38 39 40. + 41 42 43 44 45 46 47 48 49 50. + 51 52 53 54 55 56 57 58 59 60.""" + .stripIndent(), chunks.get(1).getContents()); + Assertions.assertEquals(""" + 61 62 63 64 65 66 67 68 69 70. + 71 72 73 74 75 76 77 78 79 80. + 81 82 83 84 85 86 87 88 89 90. + 91 92 93 94 95 96 97 98 99 100.""" + .stripIndent(), chunks.get(3).getContents()); + } + + @Test + public void testParagraphSplitter() { + + List chunks = Splitter.builder() + .addChunkEndCondition(new CountSplitCondition(2, new ParagraphSplitter())) + .setOverlapCondition(new CountOverlapCondition(2, new WordSplitter())) + .trimWhitespace() + .build() + .splitDocument(new TextDocument(PARAGRAPHS)) + .collectList() + .block(); + + Assertions.assertEquals( + """ + 01 02 03 04 05 06 07 08 09 10. 11 12 13 14 15 + 16 17 18 19 20. 21 22 23 24 + 25 26 27 28 29 30. + + 31 32 33 34 35 36 37 38 39 40. 41 42 43 44 + 45 46 47 48 49 50.""".stripIndent(), + chunks.get(0).getContents()); + + Assertions.assertEquals( + """ + 89 90. + + + 91 92 93 94 95 96 97 98 99 100.""".stripIndent(), + chunks.get(2).getContents()); + } + + @Test + public void testNewLineSplitter() { + + List chunks = Splitter.builder() + .addChunkEndCondition(new CountSplitCondition(2, new NewLineSplitter())) + .trimWhitespace() + .build() + .splitDocument(new TextDocument(PARAGRAPHS)) + .collectList() + .block(); + + Assertions.assertEquals( + """ + 01 02 03 04 05 06 07 08 09 10. 11 12 13 14 15 + 16 17 18 19 20. 21 22 23 24""".stripIndent(), + chunks.get(0).getContents()); + Assertions.assertEquals( + """ + 25 26 27 28 29 30. + + 31 32 33 34 35 36 37 38 39 40. 41 42 43 44""".stripIndent(), + chunks.get(1).getContents()); + + Assertions.assertEquals( + """ + 81 82 83 84 85 86 87 88 89 90. + + + 91 92 93 94 95 96 97 98 99 100.""".stripIndent(), + chunks.get(4).getContents()); + } +} From 1c19458e1c22e2028ecb4db8d2e614238b377e53 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Mon, 7 Oct 2024 13:28:01 +0100 Subject: [PATCH 072/277] fix typo --- .../semantic/kernel/rag/splitting/ChunkEndCondition.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/ChunkEndCondition.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/ChunkEndCondition.java index bcbfd0f1..536b2352 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/ChunkEndCondition.java +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/ChunkEndCondition.java @@ -8,7 +8,7 @@ public interface ChunkEndCondition { /** * Accepts a string and returns the number of character that should be considered as the end of - * the FIRST chunk within the string. This method will be subsiquently called until all pages + * the FIRST chunk within the string. This method will be subsequently called until all pages * are found. *

* Return -1 if the value does not contain enough characters to be considered as a full chunk. From 4342c71429013c5eb10fd1affc663848240579e8 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Mon, 7 Oct 2024 13:33:59 +0100 Subject: [PATCH 073/277] fix typo --- .../kernel/rag/splitting/splitconditions/SplitPoints.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/SplitPoints.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/SplitPoints.java index 5aa61d59..3090a86c 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/SplitPoints.java +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/SplitPoints.java @@ -3,7 +3,7 @@ /** * A class that represents the start and end points of a split. I.e if splitting by word, these - * would be the indeces of the first and last char in the word within the chunk. + * would be the indices of the first and last char in the word within the chunk. */ public class SplitPoints { From b7563617dee3b5bb8cb87a31e2fd4425079db45a Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Mon, 7 Oct 2024 18:10:17 +0100 Subject: [PATCH 074/277] Add custom splitter example --- .../rag/DocumentSplittingExample.java | 78 +++++++++++++++++++ .../samples/syntaxexamples/rag/example.md | 22 ++++++ .../kernel/rag/splitting/Splitter.java | 7 +- .../kernel/rag/splitting/TextSplitter.java | 9 ++- .../overlap/CountOverlapCondition.java | 4 +- .../splitting/overlap/NoOverlapCondition.java | 4 + .../overlap/PercentageOverlapCondition.java | 6 +- .../splitconditions/CountSplitCondition.java | 8 +- .../splitconditions/RegexSplitter.java | 15 ++-- .../{SplitPoints.java => SplitPoint.java} | 4 +- 10 files changed, 132 insertions(+), 25 deletions(-) create mode 100644 samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/resources/com/microsoft/semantickernel/samples/syntaxexamples/rag/example.md rename samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/{SplitPoints.java => SplitPoint.java} (91%) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/rag/DocumentSplittingExample.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/rag/DocumentSplittingExample.java index b081514b..549d4f4d 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/rag/DocumentSplittingExample.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/rag/DocumentSplittingExample.java @@ -4,6 +4,12 @@ import com.microsoft.semantic.kernel.rag.splitting.Chunk; import com.microsoft.semantic.kernel.rag.splitting.Document; import com.microsoft.semantic.kernel.rag.splitting.Splitter; +import com.microsoft.semantic.kernel.rag.splitting.TextSplitter; +import com.microsoft.semantic.kernel.rag.splitting.document.TextDocument; +import com.microsoft.semantic.kernel.rag.splitting.overlap.NoOverlapCondition; +import com.microsoft.semantic.kernel.rag.splitting.splitconditions.CountSplitCondition; +import com.microsoft.semantic.kernel.rag.splitting.splitconditions.SplitPoint; +import com.microsoft.semantickernel.implementation.EmbeddedResourceLoader; import java.io.ByteArrayInputStream; import java.io.IOException; import java.net.URI; @@ -12,11 +18,14 @@ import java.net.http.HttpResponse; import java.net.http.HttpResponse.BodyHandlers; import java.util.List; +import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.apache.pdfbox.io.RandomAccessReadBuffer; import org.apache.pdfbox.pdfparser.PDFParser; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.text.PDFTextStripper; import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; public class DocumentSplittingExample { @@ -46,6 +55,11 @@ public Flux getContent() { } public static void main(String[] args) throws IOException, InterruptedException { + useCustomChunker(); + useInbuiltChunker(); + } + + private static void useInbuiltChunker() throws IOException, InterruptedException { byte[] pdfBytes = getPdfDoc(); PDFDocument pdfDoc = new PDFDocument(pdfBytes); @@ -68,6 +82,70 @@ public static void main(String[] args) throws IOException, InterruptedException }); } + public static void useCustomChunker() throws IOException, InterruptedException { + + String example = EmbeddedResourceLoader.readFile("example.md", + DocumentSplittingExample.class); + + // Define how we are splitting tokens, in this case we are splitting on headers of an md file + // i.e followed by one or more # characters + TextSplitter textSplitter = (doc, numTokens) -> { + // Split on headers + Pattern pattern = Pattern.compile("(\\r?\\n|\\r)\s*#+", Pattern.MULTILINE); + + Flux splitPoints = Flux.fromStream(pattern.matcher(doc).results()) + .map(window -> window.start()); + + return createWindows(doc, splitPoints); + }; + + // Split into single sections + CountSplitCondition condition = new CountSplitCondition(1, textSplitter); + + Splitter splitter = Splitter + .builder() + .addChunkEndCondition(condition) + // No overlap + .setOverlapCondition(NoOverlapCondition.build()) + // Tidy up the text + .trimWhitespace() + .build(); + + String chunks = splitter + .splitDocument(new TextDocument(example)) + .collectList() + .map(it -> it.stream() + .map(chunk -> chunk.getContents()) + .collect(Collectors.joining("\n============\n"))) + .block(); + + System.out.println(chunks); + } + + /* + * Transforms: [ 2, 10, 20, 100 ] -> [ (0, 2), (2, 10), (10, 20), (20, 100), (100, ) + * ] + */ + private static List createWindows(String doc, Flux splitPoints) { + return Flux.concat( + Flux.just(0), + splitPoints, + Flux.just(doc.length())) + .window(2, 1) + .concatMap(window -> { + return window.collectList() + .flatMap(list -> { + if (list.size() <= 1) { + return Mono.empty(); + } + return Mono.just( + new SplitPoint(list.get(0), list.get(1))); + }); + }) + .collectList() + .block(); + } + private static byte[] getPdfDoc() throws IOException, InterruptedException { HttpResponse doc = HttpClient.newHttpClient() .send(HttpRequest.newBuilder() diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/resources/com/microsoft/semantickernel/samples/syntaxexamples/rag/example.md b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/resources/com/microsoft/semantickernel/samples/syntaxexamples/rag/example.md new file mode 100644 index 00000000..72591343 --- /dev/null +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/resources/com/microsoft/semantickernel/samples/syntaxexamples/rag/example.md @@ -0,0 +1,22 @@ +## Section 1 + +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna +aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis +aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint +occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. + +## Section 2 + +Another section. + +### Subsection 1 + +1, 2, 3, 4, 5, 6, 7, 8, 9, 10. + +# Section 3 + +This is the last section. + +``` +some code +``` \ No newline at end of file diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/Splitter.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/Splitter.java index f2623357..5e888c52 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/Splitter.java +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/Splitter.java @@ -118,15 +118,16 @@ private static List chunkDocument(List chunkEndConditi "This entier chunk consists of overlapped data, this will result in infinite loop. Skipping this chunk."); // previous chunk should already contain this text..skip it - doc = doc.substring(previousChunkEndIndex, doc.length()); + doc = doc.substring(Math.min(previousChunkEndIndex, doc.length()), + doc.length()); - previousChunkEndIndex = 0; + previousChunkEndIndex = -1; continue; } int overlapIndex = overlapCondition.getOverlapIndex(chunkText); previousChunkEndIndex = chunkText.length() - overlapIndex; - doc = doc.substring(overlapIndex, doc.length()); + doc = doc.substring(Math.min(overlapIndex, doc.length()), doc.length()); chunks.add(new Chunk(chunkText)); } else { diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/TextSplitter.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/TextSplitter.java index fca62683..e579d3c7 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/TextSplitter.java +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/TextSplitter.java @@ -1,8 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantic.kernel.rag.splitting; -import com.microsoft.semantic.kernel.rag.splitting.splitconditions.SplitPoints; - +import com.microsoft.semantic.kernel.rag.splitting.splitconditions.SplitPoint; import java.util.List; /** @@ -16,7 +15,9 @@ public interface TextSplitter { * @param doc the document to split * @return the split points */ - List getSplitPoints(String doc); + default List getSplitPoints(String doc) { + return getNSplitPoints(doc, Integer.MAX_VALUE); + } /** * Get the first n split points for the given document. @@ -25,5 +26,5 @@ public interface TextSplitter { * @param n the number of split points to get * @return the split points */ - List getNSplitPoints(String doc, int n); + List getNSplitPoints(String doc, int n); } \ No newline at end of file diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/CountOverlapCondition.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/CountOverlapCondition.java index d95ea30e..d3740653 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/CountOverlapCondition.java +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/CountOverlapCondition.java @@ -3,7 +3,7 @@ import com.microsoft.semantic.kernel.rag.splitting.OverlapCondition; import com.microsoft.semantic.kernel.rag.splitting.TextSplitter; -import com.microsoft.semantic.kernel.rag.splitting.splitconditions.SplitPoints; +import com.microsoft.semantic.kernel.rag.splitting.splitconditions.SplitPoint; import java.util.List; /** @@ -25,7 +25,7 @@ public CountOverlapCondition(int count, TextSplitter splitter) { @Override public int getOverlapIndex(String chunk) { - List splitPoints = splitter.getSplitPoints(chunk); + List splitPoints = splitter.getSplitPoints(chunk); if (splitPoints.size() == 0) { return 0; diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/NoOverlapCondition.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/NoOverlapCondition.java index 23595bcd..696a864c 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/NoOverlapCondition.java +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/NoOverlapCondition.java @@ -11,6 +11,10 @@ public class NoOverlapCondition implements OverlapCondition { public NoOverlapCondition() { } + public static OverlapCondition build() { + return new NoOverlapCondition(); + } + @Override public int getOverlapIndex(String chunk) { return chunk.length(); diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/PercentageOverlapCondition.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/PercentageOverlapCondition.java index e4f804c3..79749708 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/PercentageOverlapCondition.java +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/overlap/PercentageOverlapCondition.java @@ -3,7 +3,7 @@ import com.microsoft.semantic.kernel.rag.splitting.OverlapCondition; import com.microsoft.semantic.kernel.rag.splitting.TextSplitter; -import com.microsoft.semantic.kernel.rag.splitting.splitconditions.SplitPoints; +import com.microsoft.semantic.kernel.rag.splitting.splitconditions.SplitPoint; import java.util.List; import org.slf4j.Logger; @@ -31,11 +31,11 @@ public PercentageOverlapCondition(float percentage, TextSplitter splitter) { @Override public int getOverlapIndex(String chunk) { - List splitPoints = splitter.getSplitPoints(chunk); + List splitPoints = splitter.getSplitPoints(chunk); float index = chunk.length() * (100.0f - percentage) / 100.0f; - for (SplitPoints splitPoint : splitPoints) { + for (SplitPoint splitPoint : splitPoints) { if (splitPoint.getEnd() > index) { return splitPoint.getStart(); } diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/CountSplitCondition.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/CountSplitCondition.java index bea0c6a4..4734cadc 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/CountSplitCondition.java +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/CountSplitCondition.java @@ -21,7 +21,13 @@ public CountSplitCondition(int count, TextSplitter splitter) { @Override public int getEndOfNextChunk(String doc) { - List splitPoints = splitter.getNSplitPoints(doc, count); + List splitPoints = splitter.getNSplitPoints(doc, count) + .stream() + .filter(it -> it != null) + .filter(it -> it.getEnd() != 0) + .filter(it -> it.getEnd() != it.getStart()) + .filter(it -> it.getStart() != doc.length()) + .toList(); if (splitPoints.size() < count) { return splitPoints.get(splitPoints.size() - 1).getEnd(); diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/RegexSplitter.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/RegexSplitter.java index 8876323b..5828e4a4 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/RegexSplitter.java +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/RegexSplitter.java @@ -44,18 +44,13 @@ public RegexSplitter(Pattern pattern, int trivialSplitLength) { } @Override - public List getSplitPoints(String doc) { - return getNSplitPoints(doc, Integer.MAX_VALUE); - } - - @Override - public List getNSplitPoints(String doc, int n) { + public List getNSplitPoints(String doc, int n) { Matcher matcher = pattern.matcher(doc); List points = matcher.results() .collect(Collectors.toList()); - List result = new ArrayList<>(); + List result = new ArrayList<>(); int previousEnd = 0; for (MatchResult point : points) { @@ -66,7 +61,7 @@ public List getNSplitPoints(String doc, int n) { trivialSplitLength)) { continue; } - result.add(new SplitPoints(previousEnd, point.end())); + result.add(new SplitPoint(previousEnd, point.end())); previousEnd = point.end(); if (result.size() >= n) { break; @@ -74,11 +69,11 @@ public List getNSplitPoints(String doc, int n) { } if (result.size() < n && !isTrivialSplit(previousEnd, doc.length(), doc, 1)) { - result.add(new SplitPoints(previousEnd, doc.length())); + result.add(new SplitPoint(previousEnd, doc.length())); } if (result.isEmpty()) { - return List.of(new SplitPoints(0, doc.length())); + return List.of(new SplitPoint(0, doc.length())); } return result; diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/SplitPoints.java b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/SplitPoint.java similarity index 91% rename from samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/SplitPoints.java rename to samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/SplitPoint.java index 3090a86c..8f621adb 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/SplitPoints.java +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/src/main/java/com/microsoft/semantic/kernel/rag/splitting/splitconditions/SplitPoint.java @@ -5,12 +5,12 @@ * A class that represents the start and end points of a split. I.e if splitting by word, these * would be the indices of the first and last char in the word within the chunk. */ -public class SplitPoints { +public class SplitPoint { private final int start; private final int end; - public SplitPoints(int start, int end) { + public SplitPoint(int start, int end) { this.start = start; this.end = end; } From 1625467dd1ea323897f0872fb3b2de9a2c5abc35 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Wed, 9 Oct 2024 01:52:57 -0700 Subject: [PATCH 075/277] Add tag list support to JDBC vector stores --- .../tests/connectors/memory/jdbc/Hotel.java | 38 ++++-- .../JDBCVectorStoreRecordCollectionTest.java | 65 +++++++--- .../tests/connectors/memory/redis/Hotel.java | 23 ++-- ...ashSetVectorStoreRecordCollectionTest.java | 28 ++++ ...isJsonVectorStoreRecordCollectionTest.java | 28 ++++ .../memory/InMemoryVolatileVectorStore.java | 14 +- .../memory/VectorStoreWithAzureAISearch.java | 19 ++- .../memory/VectorStoreWithJDBC.java | 16 +-- .../memory/VectorStoreWithRedis.java | 18 +-- .../semantickernel-learn-resources/pom.xml | 10 ++ .../AzureAISearchVectorStore.java | 7 +- ...rchVectorStoreCollectionSearchMapping.java | 103 +++++++++++---- ...reAISearchVectorStoreRecordCollection.java | 7 +- ...zureAISearchAnyTagEqualToFilterClause.java | 27 ---- .../AzureAISearchEqualToFilterClause.java | 55 -------- .../HSQLDBVectorStoreQueryProvider.java | 69 ++++++---- .../jdbc/JDBCVectorStoreQueryProvider.java | 103 +++++++++++++-- .../jdbc/JDBCVectorStoreRecordMapper.java | 14 +- .../SQLVectorStoreFilterQueryProvider.java | 18 +++ .../jdbc/SQLVectorStoreQueryProvider.java | 1 + ...torStoreRecordCollectionSearchMapping.java | 53 -------- .../jdbc/filter/SQLEqualToFilterClause.java | 51 -------- .../mysql/MySQLVectorStoreQueryProvider.java | 9 +- .../PostgreSQLVectorStoreQueryProvider.java | 121 ++++++++++-------- .../PostgreSQLVectorStoreRecordMapper.java | 14 +- ...disHashSetVectorStoreRecordCollection.java | 1 + .../RedisJsonVectorStoreRecordCollection.java | 2 +- ...disVectorStoreCollectionSearchMapping.java | 80 ++++++++++-- .../filter/RedisEqualToFilterClause.java | 42 ------ .../SQLiteVectorStoreQueryProvider.java | 7 + .../filter/AnyTagEqualToFilterClause.java | 2 +- .../data/filter/EqualToFilterClause.java | 12 -- .../data/filter/FilterClause.java | 7 - .../data/filter/FilterMapping.java | 32 +++++ .../data/vectorsearch/VectorSearchFilter.java | 15 ++- ...ribute.java => VectorStoreRecordData.java} | 2 +- ...tribute.java => VectorStoreRecordKey.java} | 2 +- ...bute.java => VectorStoreRecordVector.java} | 2 +- .../VectorStoreRecordDefinition.java | 24 ++-- .../microsoft/semantickernel/data/Hotel.java | 24 ++-- ...latileVectorStoreRecordCollectionTest.java | 6 +- 41 files changed, 660 insertions(+), 511 deletions(-) delete mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchAnyTagEqualToFilterClause.java delete mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchEqualToFilterClause.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreFilterQueryProvider.java delete mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollectionSearchMapping.java delete mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/filter/SQLEqualToFilterClause.java delete mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/filter/RedisEqualToFilterClause.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/{VectorStoreRecordDataAttribute.java => VectorStoreRecordData.java} (94%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/{VectorStoreRecordKeyAttribute.java => VectorStoreRecordKey.java} (90%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/{VectorStoreRecordVectorAttribute.java => VectorStoreRecordVector.java} (95%) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java index c2b93c2d..a7a11901 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java @@ -2,9 +2,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVector; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; @@ -12,39 +12,43 @@ public class Hotel { @JsonProperty("hotelId") - @VectorStoreRecordKeyAttribute + @VectorStoreRecordKey private final String id; - @VectorStoreRecordDataAttribute(isFilterable = true) + @VectorStoreRecordData(isFilterable = true) private final String name; - @VectorStoreRecordDataAttribute + @VectorStoreRecordData private final int code; @JsonProperty("summary") - @VectorStoreRecordDataAttribute() + @VectorStoreRecordData() private final String description; @JsonProperty("summaryEmbedding1") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) private final List euclidean; @JsonProperty("summaryEmbedding2") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE) + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List cosineDistance; @JsonProperty("summaryEmbedding3") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.DOT_PRODUCT) + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.DOT_PRODUCT) private final List dotProduct; @JsonProperty("indexedSummaryEmbedding") - @VectorStoreRecordVectorAttribute(dimensions = 8, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) + @VectorStoreRecordVector(dimensions = 8, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) private final List indexedEuclidean; - @VectorStoreRecordDataAttribute + + @VectorStoreRecordData + private final List tags; + + @VectorStoreRecordData private double rating; public Hotel() { - this(null, null, 0, null, null, null, null, null, 0.0); + this(null, null, 0, null, null, null, null, null, 0.0, null); } @JsonCreator @@ -57,7 +61,8 @@ public Hotel( @JsonProperty("summaryEmbedding2") List cosineDistance, @JsonProperty("summaryEmbedding3") List dotProduct, @JsonProperty("indexedSummaryEmbedding") List indexedEuclidean, - @JsonProperty("rating") double rating) { + @JsonProperty("rating") double rating, + @JsonProperty("tags") List tags) { this.id = id; this.name = name; this.code = code; @@ -67,6 +72,7 @@ public Hotel( this.dotProduct = euclidean; this.indexedEuclidean = euclidean; this.rating = rating; + this.tags = tags; } public String getId() { @@ -97,6 +103,10 @@ public double getRating() { return rating; } + public List getTags() { + return tags; + } + public void setRating(double rating) { this.rating = rating; } diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java index ec63d9ec..8342f489 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -8,7 +8,6 @@ import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.connectors.data.jdbc.filter.SQLEqualToFilterClause; import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.sqlite.SQLiteVectorStoreQueryProvider; @@ -159,19 +158,19 @@ private List getHotels() { return Arrays.asList( new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), null, null, null, - 4.0), + 4.0, Arrays.asList("luxury", "city")), new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), null, null, null, - 4.0), + 4.0, Arrays.asList("luxury", "city")), new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), null, null, null, - 5.0), + 5.0, Arrays.asList("luxury", "beach")), new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), null, null, null, - 4.0), + 4.0, Arrays.asList("luxury", "city")), new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), null, null, null, - 4.0) + 4.0, Arrays.asList("luxury", "city")) ); } @@ -396,14 +395,13 @@ public void getBatchWithNoVectors(QueryProvider provider) { } private static Stream provideSearchParameters() { - return Stream.of( - Arguments.of(QueryProvider.MySQL, "euclidean"), - Arguments.of(QueryProvider.MySQL, "cosineDistance"), - Arguments.of(QueryProvider.MySQL, "dotProduct"), - Arguments.of(QueryProvider.PostgreSQL, "euclidean"), - Arguments.of(QueryProvider.PostgreSQL, "cosineDistance"), - Arguments.of(QueryProvider.PostgreSQL, "dotProduct") - ); + return Arrays.stream(QueryProvider.values()).map(provider -> + Stream.of( + Arguments.of(provider, "euclidean"), + Arguments.of(provider, "cosineDistance"), + Arguments.of(provider, "dotProduct") + ) + ).flatMap(s -> s); } @ParameterizedTest @@ -464,13 +462,13 @@ public void approximateSearch(QueryProvider provider) { assertNotNull(results); assertEquals(5, results.size()); // The third hotel should be the most similar - assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + assertEquals("id_3", results.get(0).getRecord().getId()); } @ParameterizedTest @MethodSource("provideSearchParameters") - public void searchWithFilter(QueryProvider provider, String embeddingName) { - String collectionName = "searchWithFilter"; + public void searchWithFilterEqualToFilter(QueryProvider provider, String embeddingName) { + String collectionName = "searchWithFilterEqualToFilter"; JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); @@ -482,7 +480,7 @@ public void searchWithFilter(QueryProvider provider, String embeddingName) { .withLimit(3) .withVectorSearchFilter( VectorSearchFilter.builder() - .withEqualToFilterClause(new SQLEqualToFilterClause("rating", 4.0)).build()) + .equalTo("rating", 4.0).build()) .build(); // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned @@ -491,7 +489,34 @@ public void searchWithFilter(QueryProvider provider, String embeddingName) { assertNotNull(results); assertEquals(3, results.size()); // The first hotel should be the most similar - assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); + assertEquals("id_1", results.get(0).getRecord().getId()); + } + + @ParameterizedTest + @MethodSource("provideSearchParameters") + public void searchWithAnyTagEqualToFilter(QueryProvider provider, String embeddingName) { + String collectionName = "searchWithAnyTagEqualToFilter"; + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, + collectionName); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions options = VectorSearchOptions.builder() + .withVectorFieldName(embeddingName) + .withLimit(3) + .withVectorSearchFilter( + VectorSearchFilter.builder() + .anyTagEqualTo("tags", "city").build()) + .build(); + + // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, + options).block(); + assertNotNull(results); + assertEquals(3, results.size()); + // The first hotel should be the most similar + assertEquals("id_1", results.get(0).getRecord().getId()); } // MySQL will always return the vectors as they're needed to compute the distances @@ -520,7 +545,7 @@ public void postgresSearchIncludeAndNotIncludeVectors() { assertNotNull(results); assertEquals(3, results.size()); // The third hotel should be the most similar - assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + assertEquals("id_3", results.get(0).getRecord().getId()); assertNotNull(results.get(0).getRecord().getEuclidean()); } } diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java index 81b962bb..478adb32 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java @@ -2,9 +2,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVector; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; @@ -12,31 +12,32 @@ public class Hotel { - @VectorStoreRecordKeyAttribute + @VectorStoreRecordKey private final String id; - @VectorStoreRecordDataAttribute(isFilterable = true) + @VectorStoreRecordData(isFilterable = true) private final String name; - @VectorStoreRecordDataAttribute + @VectorStoreRecordData private final int code; @JsonProperty("summary") - @VectorStoreRecordDataAttribute() + @VectorStoreRecordData() private final String description; @JsonProperty("summaryEmbedding1") - @VectorStoreRecordVectorAttribute(dimensions = 8, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) + @VectorStoreRecordVector(dimensions = 8, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) private final List euclidean; @JsonProperty("summaryEmbedding2") - @VectorStoreRecordVectorAttribute(dimensions = 8) + @VectorStoreRecordVector(dimensions = 8) private final List cosineDistance; @JsonProperty("summaryEmbedding3") - @VectorStoreRecordVectorAttribute(dimensions = 8, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.DOT_PRODUCT) + @VectorStoreRecordVector(dimensions = 8, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.DOT_PRODUCT) private final List dotProduct; - @VectorStoreRecordDataAttribute + + @VectorStoreRecordData(isFilterable = true) private double rating; public Hotel() { diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index 8e07fdd2..a2b5e2c1 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -2,6 +2,7 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; @@ -94,6 +95,7 @@ static void setup() { fields.add(VectorStoreRecordDataField.builder() .withName("rating") .withFieldType(Double.class) + .isFilterable(true) .build()); VectorStoreRecordDefinition recordDefinition = VectorStoreRecordDefinition.fromFields(fields); @@ -440,4 +442,30 @@ public void searchWithOffSet(RecordCollectionOptions options, String embeddingNa // The first hotel should be the most similar assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId(), indexingFailureMessage); } + + @ParameterizedTest + @MethodSource("provideSearchParameters") + public void searchWithFilterEqualToFilter(RecordCollectionOptions recordCollectionOptions, String embeddingName) { + String collectionName = getCollectionName("search", recordCollectionOptions); + RedisHashSetVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(recordCollectionOptions), collectionName); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions options = VectorSearchOptions.builder() + .withVectorFieldName(embeddingName) + .withLimit(3) + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("rating", 4.0).build()) + .build(); + + // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, + options).block(); + assertNotNull(results); + assertEquals(3, results.size()); + // The first hotel should be the most similar + assertEquals("id_1", results.get(0).getRecord().getId()); + } } diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index df41e84e..393690fd 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -2,6 +2,7 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; @@ -94,6 +95,7 @@ static void setup() { fields.add(VectorStoreRecordDataField.builder() .withName("rating") .withFieldType(Double.class) + .isFilterable(true) .build()); VectorStoreRecordDefinition recordDefinition = VectorStoreRecordDefinition.fromFields(fields); @@ -440,4 +442,30 @@ public void searchWithOffSet(RecordCollectionOptions options, String embeddingNa // The first hotel should be the most similar assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId(), indexingFailureMessage); } + + @ParameterizedTest + @MethodSource("provideSearchParameters") + public void searchWithFilterEqualToFilter(RecordCollectionOptions recordCollectionOptions, String embeddingName) { + String collectionName = getCollectionName("search", recordCollectionOptions); + RedisJsonVectorStoreRecordCollection recordCollection = createCollection(optionsMap.get(recordCollectionOptions), collectionName); + + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions options = VectorSearchOptions.builder() + .withVectorFieldName(embeddingName) + .withLimit(3) + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("rating", 4.0).build()) + .build(); + + // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, + options).block(); + assertNotNull(results); + assertEquals(3, results.size()); + // The first hotel should be the most similar + assertEquals("id_1", results.get(0).getRecord().getId()); + } } diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java index f5d9d7f1..192f1189 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java @@ -11,9 +11,9 @@ import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.VolatileVectorStore; import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVector; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -39,13 +39,13 @@ public class InMemoryVolatileVectorStore { static class GitHubFile { @JsonProperty("fileId") // Set a different name for the storage field if needed - @VectorStoreRecordKeyAttribute() + @VectorStoreRecordKey() private final String id; - @VectorStoreRecordDataAttribute() + @VectorStoreRecordData() private final String description; - @VectorStoreRecordDataAttribute + @VectorStoreRecordData private final String link; - @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE) + @VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List embedding; public GitHubFile( diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java index 9de59ed7..59822601 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java @@ -17,9 +17,9 @@ import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVector; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import java.nio.charset.StandardCharsets; import java.util.Arrays; @@ -53,14 +53,13 @@ public class VectorStoreWithAzureAISearch { static class GitHubFile { @JsonProperty("fileId") // Set a different name for the storage field if needed - @VectorStoreRecordKeyAttribute() + @VectorStoreRecordKey() private final String id; - @VectorStoreRecordDataAttribute() - @VectorStoreRecordVectorAttribute(distanceFunction = DistanceFunction.COSINE_DISTANCE, dimensions = EMBEDDING_DIMENSIONS) + @VectorStoreRecordData() private final String description; - @VectorStoreRecordDataAttribute + @VectorStoreRecordData private final String link; - @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE) + @VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_SIMILARITY) private final List embedding; public GitHubFile() { @@ -161,8 +160,8 @@ private static Mono>> search( VectorStoreRecordCollection recordCollection, OpenAITextEmbeddingGenerationService embeddingGeneration) { // Generate embeddings for the search text and search for the closest records - return embeddingGeneration.generateEmbeddingsAsync(Collections.singletonList(searchText)) - .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); + return embeddingGeneration.generateEmbeddingAsync(searchText) + .flatMap(r -> recordCollection.searchAsync(r.getVector(), null)); } private static Mono> storeData( diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java index 6cd4953f..e842d6e1 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java @@ -13,9 +13,9 @@ import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVector; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.mysql.cj.jdbc.MysqlDataSource; import java.nio.charset.StandardCharsets; @@ -42,14 +42,14 @@ public class VectorStoreWithJDBC { private static final int EMBEDDING_DIMENSIONS = 1536; static class GitHubFile { - @JsonProperty("fileId") // Set a different name for the storage field if needed - @VectorStoreRecordKeyAttribute() + + @VectorStoreRecordKey() private final String id; - @VectorStoreRecordDataAttribute() + @VectorStoreRecordData() private final String description; - @VectorStoreRecordDataAttribute + @VectorStoreRecordData private final String link; - @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, distanceFunction = DistanceFunction.COSINE_DISTANCE) + @VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List embedding; public GitHubFile() { diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java index aee00a5b..7d6976ce 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java @@ -5,21 +5,17 @@ import com.azure.ai.openai.OpenAIClientBuilder; import com.azure.core.credential.AzureKeyCredential; import com.azure.core.credential.KeyCredential; -import com.azure.core.util.ClientOptions; -import com.azure.core.util.MetricsOptions; -import com.azure.core.util.TracingOptions; import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; -import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.redis.RedisStorageType; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStore; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreOptions; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVector; import java.util.Arrays; import java.util.Collections; @@ -47,13 +43,13 @@ public class VectorStoreWithRedis { public static class GitHubFile { @JsonProperty("fileId") // Set a different name for the storage field if needed - @VectorStoreRecordKeyAttribute() + @VectorStoreRecordKey() private final String id; - @VectorStoreRecordDataAttribute() + @VectorStoreRecordData() private final String description; - @VectorStoreRecordDataAttribute + @VectorStoreRecordData private final String link; - @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE) + @VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List embedding; public GitHubFile() { diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index f042a3ef..d4b8aec4 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -62,6 +62,16 @@ com.microsoft.semantic-kernel semantickernel-aiservices-openai + + com.microsoft.semantic-kernel + semantickernel-experimental + + + com.mysql + mysql-connector-j + 9.0.0 + compile + diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java index e0e763c1..adbbdb87 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java @@ -26,9 +26,9 @@ public class AzureAISearchVectorStore implements VectorStore { */ @SuppressFBWarnings("EI_EXPOSE_REP2") public AzureAISearchVectorStore(@Nonnull SearchIndexAsyncClient searchIndexAsyncClient, - @Nonnull AzureAISearchVectorStoreOptions options) { + @Nullable AzureAISearchVectorStoreOptions options) { this.searchIndexAsyncClient = searchIndexAsyncClient; - this.options = options; + this.options = options == null ? new AzureAISearchVectorStoreOptions() : options; } /** @@ -128,9 +128,6 @@ public AzureAISearchVectorStore build() { if (searchIndexAsyncClient == null) { throw new SKException("searchIndexAsyncClient is required"); } - if (options == null) { - throw new SKException("options is required"); - } return new AzureAISearchVectorStore(searchIndexAsyncClient, options); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java index eb3a5ff2..4abbe1fb 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java @@ -1,45 +1,92 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.azureaisearch; -import com.microsoft.semantickernel.connectors.data.azureaisearch.filter.AzureAISearchEqualToFilterClause; -import com.microsoft.semantickernel.connectors.data.azureaisearch.filter.AzureAISearchAnyTagEqualToFilterClause; +import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; +import com.microsoft.semantickernel.data.filter.EqualToFilterClause; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.filter.FilterMapping; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; import java.util.stream.Collectors; -class AzureAISearchVectorStoreCollectionSearchMapping { - public static String buildFilterString(VectorSearchFilter vectorSearchFilter, +class AzureAISearchVectorStoreCollectionSearchMapping + implements FilterMapping { + + private AzureAISearchVectorStoreCollectionSearchMapping() { + } + + private static class AzureAISearchVectorStoreCollectionSearchMappingHolder { + private static final AzureAISearchVectorStoreCollectionSearchMapping INSTANCE = new AzureAISearchVectorStoreCollectionSearchMapping(); + } + + static AzureAISearchVectorStoreCollectionSearchMapping getInstance() { + return AzureAISearchVectorStoreCollectionSearchMappingHolder.INSTANCE; + } + + public String getFilter(VectorSearchFilter vectorSearchFilter, VectorStoreRecordDefinition recordDefinition) { if (vectorSearchFilter == null || vectorSearchFilter.getFilterClauses().isEmpty()) { return ""; } - return String.join(" and ", - vectorSearchFilter.getFilterClauses().stream().map(filterClause -> { - if (filterClause instanceof AzureAISearchEqualToFilterClause) { - AzureAISearchEqualToFilterClause azureFilterClause = (AzureAISearchEqualToFilterClause) filterClause; - // Create new instance with the storage name of the field - return new AzureAISearchEqualToFilterClause( - recordDefinition.getField(azureFilterClause.getFieldName()) - .getEffectiveStorageName(), - azureFilterClause.getValue()) - .getFilter(); - } else if (filterClause instanceof AzureAISearchAnyTagEqualToFilterClause) { - AzureAISearchAnyTagEqualToFilterClause azureFilterClause = (AzureAISearchAnyTagEqualToFilterClause) filterClause; - // Create new instance with the storage name of the field - return new AzureAISearchAnyTagEqualToFilterClause( - recordDefinition.getField(azureFilterClause.getFieldName()) - .getEffectiveStorageName(), - azureFilterClause.getValue()) - .getFilter(); - } else { - throw new SKException("Unsupported filter clause type '" - + filterClause.getClass().getSimpleName() + "'."); - } - }) - .collect(Collectors.toList())); + return vectorSearchFilter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof EqualToFilterClause) { + EqualToFilterClause equalToFilterClause = (EqualToFilterClause) filterClause; + // Create new instance with the storage name of the field + return getEqualToFilter(new EqualToFilterClause( + recordDefinition.getField(equalToFilterClause.getFieldName()) + .getEffectiveStorageName(), + equalToFilterClause.getValue())); + } else if (filterClause instanceof AnyTagEqualToFilterClause) { + AnyTagEqualToFilterClause anyTagEqualToFilterClause = (AnyTagEqualToFilterClause) filterClause; + // Create new instance with the storage name of the field + return getAnyTagEqualToFilter(new AnyTagEqualToFilterClause( + recordDefinition.getField(anyTagEqualToFilterClause.getFieldName()) + .getEffectiveStorageName(), + anyTagEqualToFilterClause.getValue())); + } else { + throw new SKException("Unsupported filter clause type '" + + filterClause.getClass().getSimpleName() + "'."); + } + }).collect(Collectors.joining(" and ")); + } + + @Override + public String getEqualToFilter(EqualToFilterClause filterClause) { + String fieldName = filterClause.getFieldName(); + Object value = filterClause.getValue(); + + if (value instanceof String) { + return String.format("%s eq '%s'", fieldName, value); + } else if (value instanceof Boolean) { + return String.format("%s eq %s", fieldName, + value.toString().toLowerCase()); + } else if (value instanceof Integer) { + return String.format("%s eq %d", fieldName, (Integer) value); + } else if (value instanceof Long) { + return String.format("%s eq %d", fieldName, (Long) value); + } else if (value instanceof Float) { + return String.format("%s eq %f", fieldName, (Float) value); + } else if (value instanceof Double) { + return String.format("%s eq %f", fieldName, (Double) value); + } else if (value instanceof OffsetDateTime) { + return String.format("%s eq %s", fieldName, ((OffsetDateTime) value) + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + } else if (value == null) { + return String.format("%s eq null", fieldName); + } else { + throw new SKException("Unsupported filter value type '" + + value.getClass().getSimpleName() + "'."); + } + } + + @Override + public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { + return String.format("%s/any(t: t eq '%s')", filterClause.getFieldName(), + filterClause.getValue()); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index fad24fe5..b01a0a76 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -65,7 +65,8 @@ public class AzureAISearchVectorStoreRecordCollection implements double.class, Boolean.class, boolean.class, - OffsetDateTime.class)); + OffsetDateTime.class, + List.class)); private static final HashSet> supportedVectorTypes = new HashSet<>( Arrays.asList( @@ -278,8 +279,8 @@ private Mono>> searchAndMapAsync( List vectorQueries, VectorSearchOptions options, GetRecordOptions getRecordOptions) { - String filter = AzureAISearchVectorStoreCollectionSearchMapping - .buildFilterString(options.getVectorSearchFilter(), recordDefinition); + String filter = AzureAISearchVectorStoreCollectionSearchMapping.getInstance() + .getFilter(options.getVectorSearchFilter(), recordDefinition); SearchOptions searchOptions = new SearchOptions() .setFilter(filter) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchAnyTagEqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchAnyTagEqualToFilterClause.java deleted file mode 100644 index 7c84174e..00000000 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchAnyTagEqualToFilterClause.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.azureaisearch.filter; - -import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; - -public class AzureAISearchAnyTagEqualToFilterClause extends AnyTagEqualToFilterClause { - - /** - * Initializes a new instance of the AzureAISearchTagListContainsFilterClause class. - * - * @param fieldName The field name to filter on. - * @param value The value. - */ - public AzureAISearchAnyTagEqualToFilterClause(String fieldName, Object value) { - super(fieldName, value); - } - - /** - * Gets the filter string. - * - * @return The filter string. - */ - @Override - public String getFilter() { - return String.format("%s/any(t: t eq '%s')", getFieldName(), getValue()); - } -} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchEqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchEqualToFilterClause.java deleted file mode 100644 index 0bacb35c..00000000 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchEqualToFilterClause.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.azureaisearch.filter; - -import com.microsoft.semantickernel.data.filter.EqualToFilterClause; -import com.microsoft.semantickernel.exceptions.SKException; - -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; - -public class AzureAISearchEqualToFilterClause extends EqualToFilterClause { - - /** - * Initializes a new instance of the AzureAISearchEqualityFilterClause class. - * - * @param fieldName The field name to filter on. - * @param value The value. - */ - public AzureAISearchEqualToFilterClause(String fieldName, Object value) { - super(fieldName, value); - } - - /** - * Gets the filter string. - * - * @return The filter string. - */ - @Override - public String getFilter() { - String fieldName = getFieldName(); - Object value = getValue(); - - if (value instanceof String) { - return String.format("%s eq '%s'", fieldName, value); - } else if (value instanceof Boolean) { - return String.format("%s eq %s", fieldName, - value.toString().toLowerCase()); - } else if (value instanceof Integer) { - return String.format("%s eq %d", fieldName, (Integer) value); - } else if (value instanceof Long) { - return String.format("%s eq %d", fieldName, (Long) value); - } else if (value instanceof Float) { - return String.format("%s eq %f", fieldName, (Float) value); - } else if (value instanceof Double) { - return String.format("%s eq %f", fieldName, (Double) value); - } else if (value instanceof OffsetDateTime) { - return String.format("%s eq %s", fieldName, ((OffsetDateTime) value) - .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); - } else if (value == null) { - return String.format("%s eq null", fieldName); - } else { - throw new SKException("Unsupported filter value type '" - + value.getClass().getSimpleName() + "'."); - } - } -} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/hsqldb/HSQLDBVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/hsqldb/HSQLDBVectorStoreQueryProvider.java index e8a0549d..85b33d1f 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/hsqldb/HSQLDBVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/hsqldb/HSQLDBVectorStoreQueryProvider.java @@ -2,8 +2,10 @@ package com.microsoft.semantickernel.connectors.data.hsqldb; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; @@ -25,12 +27,15 @@ public class HSQLDBVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider { + private final ObjectMapper objectMapper; + @SuppressFBWarnings("EI_EXPOSE_REP2") private HSQLDBVectorStoreQueryProvider( DataSource dataSource, String collectionsTable, String prefixForCollectionTables, - int defaultVarCharLength) { + int defaultVarCharLength, + ObjectMapper objectMapper) { super( dataSource, collectionsTable, @@ -38,6 +43,7 @@ private HSQLDBVectorStoreQueryProvider( buildSupportedKeyTypes(defaultVarCharLength), buildSupportedDataTypes(defaultVarCharLength), buildSupportedVectorTypes(defaultVarCharLength)); + this.objectMapper = objectMapper; } private static Map, String> buildSupportedVectorTypes(int defaultVarCharLength) { @@ -62,6 +68,7 @@ private static Map, String> buildSupportedDataTypes(int defaultVarCharL supportedDataTypes.put(Boolean.class, "BOOLEAN"); supportedDataTypes.put(boolean.class, "BOOLEAN"); supportedDataTypes.put(OffsetDateTime.class, "TIMESTAMPTZ"); + supportedDataTypes.put(List.class, "TEXT"); return supportedDataTypes; } @@ -71,34 +78,32 @@ private static HashMap, String> buildSupportedKeyTypes(int defaultVarCh return supportedKeyTypes; } - private void setStatementValues(PreparedStatement statement, Object record, + private void setUpsertStatementValues(PreparedStatement statement, Object record, List fields) { + JsonNode jsonNode = objectMapper.valueToTree(record); + for (int i = 0; i < fields.size(); ++i) { VectorStoreRecordField field = fields.get(i); try { - Field recordField = record.getClass().getDeclaredField(field.getName()); - recordField.setAccessible(true); - Object value = recordField.get(record); - - if (field instanceof VectorStoreRecordKeyField) { - statement.setObject(i + 1, (String) value); - } else if (field instanceof VectorStoreRecordVectorField) { - Class vectorType = record.getClass().getDeclaredField(field.getName()) - .getType(); - - // If the vector field is other than String, serialize it to JSON - if (vectorType.equals(String.class)) { - statement.setObject(i + 1, value); - } else { - // Serialize the vector to JSON - statement.setObject(i + 1, new ObjectMapper().writeValueAsString(value)); + JsonNode valueNode = jsonNode.get(field.getEffectiveStorageName()); + + if (field instanceof VectorStoreRecordVectorField) { + // Convert the vector field to a string + if (!field.getFieldType().equals(String.class)) { + statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); + continue; + } + } else if (field instanceof VectorStoreRecordDataField) { + // Convert List field to a string + if (field.getFieldType().equals(List.class)) { + statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); + continue; } - } else { - statement.setObject(i + 1, value); } - } catch (NoSuchFieldException | IllegalAccessException | SQLException e) { - throw new SKException("Failed to set statement values", e); - } catch (JsonProcessingException e) { + + statement.setObject(i + 1, + objectMapper.convertValue(valueNode, field.getFieldType())); + } catch (SQLException | JsonProcessingException e) { throw new RuntimeException(e); } } @@ -152,7 +157,7 @@ public void upsertRecords(String collectionName, List records, try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(query)) { for (Object record : records) { - setStatementValues(statement, record, recordDefinition.getAllFields()); + setUpsertStatementValues(statement, record, recordDefinition.getAllFields()); statement.addBatch(); } @@ -178,6 +183,7 @@ public static class Builder private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; private int defaultVarCharLength = 255; + private ObjectMapper objectMapper = new ObjectMapper(); @SuppressFBWarnings("EI_EXPOSE_REP2") public Builder withDataSource(DataSource dataSource) { @@ -212,6 +218,18 @@ public Builder setDefaultVarCharLength(int defaultVarCharLength) { return this; } + /** + * Sets the object mapper. + * + * @param objectMapper the object mapper + * @return the builder + */ + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withObjectMapper(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + return this; + } + public HSQLDBVectorStoreQueryProvider build() { if (dataSource == null) { throw new SKException("DataSource is required"); @@ -221,7 +239,8 @@ public HSQLDBVectorStoreQueryProvider build() { dataSource, collectionsTable, prefixForCollectionTables, - defaultVarCharLength); + defaultVarCharLength, + objectMapper); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index 0c07588c..dfa0d0f6 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -1,7 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; +import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; +import com.microsoft.semantickernel.data.filter.EqualToFilterClause; import com.microsoft.semantickernel.data.vectorsearch.VectorOperations; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; @@ -34,14 +37,16 @@ import org.slf4j.LoggerFactory; public class JDBCVectorStoreQueryProvider - implements SQLVectorStoreQueryProvider { + implements SQLVectorStoreQueryProvider, + SQLVectorStoreFilterQueryProvider { private static final Logger LOGGER = LoggerFactory .getLogger(JDBCVectorStoreQueryProvider.class); - private final Map, String> supportedKeyTypes; - private final Map, String> supportedDataTypes; - private final Map, String> supportedVectorTypes; + protected final Map, String> supportedKeyTypes; + protected final Map, String> supportedDataTypes; + protected final Map, String> supportedVectorTypes; + protected final DataSource dataSource; private final String collectionsTable; private final String prefixForCollectionTables; @@ -71,6 +76,7 @@ protected JDBCVectorStoreQueryProvider( supportedDataTypes.put(Boolean.class, "BOOLEAN"); supportedDataTypes.put(boolean.class, "BOOLEAN"); supportedDataTypes.put(OffsetDateTime.class, "TIMESTAMPTZ"); + supportedDataTypes.put(List.class, "TEXT"); supportedVectorTypes = new HashMap<>(); supportedVectorTypes.put(String.class, "TEXT"); @@ -527,10 +533,8 @@ public List> search(String collectionName, : (VectorStoreRecordVectorField) recordDefinition .getField(options.getVectorFieldName()); - String filter = SQLVectorStoreRecordCollectionSearchMapping - .buildFilter(options.getVectorSearchFilter(), recordDefinition); - List parameters = SQLVectorStoreRecordCollectionSearchMapping - .getFilterParameters(options.getVectorSearchFilter()); + String filter = getFilter(options.getVectorSearchFilter(), recordDefinition); + List parameters = getFilterParameters(options.getVectorSearchFilter()); List records = getRecordsWithFilter(collectionName, recordDefinition, mapper, new GetRecordOptions(true), filter, parameters); @@ -569,6 +573,89 @@ public String formatQuery(String query, String... args) { return String.format(query, (Object[]) args); } + @Override + public String getFilter(VectorSearchFilter filter, + VectorStoreRecordDefinition recordDefinition) { + if (filter == null + || filter.getFilterClauses().isEmpty()) { + return ""; + } + + return filter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof EqualToFilterClause) { + EqualToFilterClause equalToFilterClause = (EqualToFilterClause) filterClause; + return getEqualToFilter(new EqualToFilterClause( + recordDefinition.getField(equalToFilterClause.getFieldName()) + .getEffectiveStorageName(), + equalToFilterClause.getValue())); + } else if (filterClause instanceof AnyTagEqualToFilterClause) { + AnyTagEqualToFilterClause anyTagEqualToFilterClause = (AnyTagEqualToFilterClause) filterClause; + return getAnyTagEqualToFilter(new AnyTagEqualToFilterClause( + recordDefinition.getField(anyTagEqualToFilterClause.getFieldName()) + .getEffectiveStorageName(), + anyTagEqualToFilterClause.getValue())); + } else { + throw new SKException("Unsupported filter clause type '" + + filterClause.getClass().getSimpleName() + "'."); + } + }).collect(Collectors.joining(" AND ")); + } + + @Override + public List getFilterParameters(VectorSearchFilter vectorSearchFilter) { + if (vectorSearchFilter == null + || vectorSearchFilter.getFilterClauses().isEmpty()) { + return Collections.emptyList(); + } + + return vectorSearchFilter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof EqualToFilterClause) { + EqualToFilterClause equalToFilterClause = (EqualToFilterClause) filterClause; + return equalToFilterClause.getValue(); + } else if (filterClause instanceof AnyTagEqualToFilterClause) { + AnyTagEqualToFilterClause anyTagEqualToFilterClause = (AnyTagEqualToFilterClause) filterClause; + return String.format("%%\"%s\"%%", anyTagEqualToFilterClause.getValue()); + } else { + throw new SKException("Unsupported filter clause type '" + + filterClause.getClass().getSimpleName() + "'."); + } + }).collect(Collectors.toList()); + } + + @Override + public String getEqualToFilter(EqualToFilterClause filterClause) { + String fieldName = JDBCVectorStoreQueryProvider + .validateSQLidentifier(filterClause.getFieldName()); + Object value = filterClause.getValue(); + + if (value instanceof String) { + return String.format("%s = ?", fieldName); + } else if (value instanceof Boolean) { + return String.format("%s = ?", fieldName); + } else if (value instanceof Integer) { + return String.format("%s = ?", fieldName); + } else if (value instanceof Long) { + return String.format("%s = ?", fieldName); + } else if (value instanceof Float) { + return String.format("%s = ?", fieldName); + } else if (value instanceof Double) { + return String.format("%s = ?", fieldName); + } else if (value instanceof OffsetDateTime) { + return String.format("%s = ?", fieldName); + } else { + throw new SKException("Unsupported filter value type '" + + value.getClass().getSimpleName() + "'."); + } + } + + @Override + public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { + String fieldName = JDBCVectorStoreQueryProvider + .validateSQLidentifier(filterClause.getFieldName()); + + return String.format("%s LIKE ?", fieldName); + } + /** * The builder for {@link JDBCVectorStoreQueryProvider}. */ diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java index 8393c9f2..7db33984 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; @@ -113,7 +114,6 @@ public JDBCVectorStoreRecordMapper build() { // Select fields from the record definition. List fields; - ResultSetMetaData metaData = resultSet.getMetaData(); if (options != null && options.isIncludeVectors()) { fields = vectorStoreRecordDefinition.getAllFields(); } else { @@ -122,13 +122,17 @@ public JDBCVectorStoreRecordMapper build() { for (VectorStoreRecordField field : fields) { Object value = resultSet.getObject(field.getEffectiveStorageName()); + Class fieldType = field.getFieldType(); if (field instanceof VectorStoreRecordVectorField) { - Class vectorType = field.getFieldType(); - // If the vector field is other than String, deserialize it from the JSON string - if (!vectorType.equals(String.class)) { - value = objectMapper.readValue((String) value, vectorType); + if (!fieldType.equals(String.class)) { + value = objectMapper.readValue((String) value, fieldType); + } + } else if (field instanceof VectorStoreRecordDataField) { + // If the field is List, deserialize it from the JSON string + if (fieldType.equals(List.class)) { + value = objectMapper.readValue((String) value, fieldType); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreFilterQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreFilterQueryProvider.java new file mode 100644 index 00000000..330df60d --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreFilterQueryProvider.java @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.jdbc; + +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.filter.FilterMapping; + +import java.util.List; + +public interface SQLVectorStoreFilterQueryProvider extends FilterMapping { + /** + * Gets the filter parameters for the given vector search filter to associate with the filter string + * generated by the getFilter method. + * + * @param filter The filter to get the filter parameters for. + * @return The filter parameters. + */ + List getFilterParameters(VectorSearchFilter filter); +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java index 2c7109f8..8ae9c1ea 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java @@ -2,6 +2,7 @@ package com.microsoft.semantickernel.connectors.data.jdbc; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.data.filter.FilterClause; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollectionSearchMapping.java deleted file mode 100644 index 17164acc..00000000 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollectionSearchMapping.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.jdbc; - -import com.microsoft.semantickernel.connectors.data.jdbc.filter.SQLEqualToFilterClause; -import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; -import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.exceptions.SKException; - -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - -public class SQLVectorStoreRecordCollectionSearchMapping { - - public static String buildFilter(VectorSearchFilter vectorSearchFilter, - VectorStoreRecordDefinition recordDefinition) { - if (vectorSearchFilter == null - || vectorSearchFilter.getFilterClauses().isEmpty()) { - return ""; - } - - return vectorSearchFilter.getFilterClauses().stream().map(filterClause -> { - if (filterClause instanceof SQLEqualToFilterClause) { - SQLEqualToFilterClause equalityFilterClause = (SQLEqualToFilterClause) filterClause; - // Create new instance with the storage name of the field - return new SQLEqualToFilterClause( - recordDefinition.getField(equalityFilterClause.getFieldName()) - .getEffectiveStorageName(), - equalityFilterClause.getValue()).getFilter(); - } else { - throw new SKException("Unsupported filter clause type '" - + filterClause.getClass().getSimpleName() + "'."); - } - }).collect(Collectors.joining(" AND ")); - } - - public static List getFilterParameters(VectorSearchFilter vectorSearchFilter) { - if (vectorSearchFilter == null - || vectorSearchFilter.getFilterClauses().isEmpty()) { - return Collections.emptyList(); - } - - return vectorSearchFilter.getFilterClauses().stream().map(filterClause -> { - if (filterClause instanceof SQLEqualToFilterClause) { - SQLEqualToFilterClause equalityFilterClause = (SQLEqualToFilterClause) filterClause; - return equalityFilterClause.getValue(); - } else { - throw new SKException("Unsupported filter clause type '" - + filterClause.getClass().getSimpleName() + "'."); - } - }).collect(Collectors.toList()); - } -} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/filter/SQLEqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/filter/SQLEqualToFilterClause.java deleted file mode 100644 index d90b27b7..00000000 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/filter/SQLEqualToFilterClause.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.jdbc.filter; - -import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.filter.EqualToFilterClause; -import com.microsoft.semantickernel.exceptions.SKException; - -import java.time.OffsetDateTime; - -public class SQLEqualToFilterClause extends EqualToFilterClause { - - /** - * Initializes a new instance of the SQLEqualityFilterClause class. - * - * @param fieldName The field name to filter on. - * @param value The value. - */ - public SQLEqualToFilterClause(String fieldName, Object value) { - super(fieldName, value); - } - - /** - * Gets the filter string. - * - * @return The filter string. - */ - @Override - public String getFilter() { - String fieldName = JDBCVectorStoreQueryProvider.validateSQLidentifier(getFieldName()); - Object value = getValue(); - - if (value instanceof String) { - return String.format("%s = ?", fieldName); - } else if (value instanceof Boolean) { - return String.format("%s = ?", fieldName); - } else if (value instanceof Integer) { - return String.format("%s = ?", fieldName); - } else if (value instanceof Long) { - return String.format("%s = ?", fieldName); - } else if (value instanceof Float) { - return String.format("%s = ?", fieldName); - } else if (value instanceof Double) { - return String.format("%s = ?", fieldName); - } else if (value instanceof OffsetDateTime) { - return String.format("%s = ?", fieldName); - } else { - throw new SKException("Unsupported filter value type '" - + value.getClass().getSimpleName() + "'."); - } - } -} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java index 87aa4d69..54a379cc 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; @@ -24,7 +25,6 @@ public class MySQLVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider implements SQLVectorStoreQueryProvider { - private final DataSource dataSource; private final ObjectMapper objectMapper; @SuppressFBWarnings("EI_EXPOSE_REP2") @@ -34,7 +34,6 @@ private MySQLVectorStoreQueryProvider( @Nonnull String prefixForCollectionTables, @Nonnull ObjectMapper objectMapper) { super(dataSource, collectionsTable, prefixForCollectionTables); - this.dataSource = dataSource; this.objectMapper = objectMapper; } @@ -61,6 +60,12 @@ private void setUpsertStatementValues(PreparedStatement statement, Object record statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); continue; } + } else if (field instanceof VectorStoreRecordDataField) { + // Convert List field to a string + if (field.getFieldType().equals(List.class)) { + statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); + continue; + } } statement.setObject(i + 1, diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index 15f7971b..3778d74a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -6,9 +6,12 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreRecordCollectionSearchMapping; +import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; +import com.microsoft.semantickernel.data.filter.EqualToFilterClause; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; @@ -29,6 +32,7 @@ import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -36,12 +40,6 @@ public class PostgreSQLVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider implements SQLVectorStoreQueryProvider { - - private final Map, String> supportedKeyTypes; - private final Map, String> supportedDataTypes; - private final Map, String> supportedVectorTypes; - - private final DataSource dataSource; private final String collectionsTable; private final String prefixForCollectionTables; private final ObjectMapper objectMapper; @@ -52,16 +50,28 @@ private PostgreSQLVectorStoreQueryProvider( @Nonnull String collectionsTable, @Nonnull String prefixForCollectionTables, @Nonnull ObjectMapper objectMapper) { - super(dataSource, collectionsTable, prefixForCollectionTables); - this.dataSource = dataSource; + super( + dataSource, + collectionsTable, + prefixForCollectionTables, + buildSupportedKeyTypes(), + buildSupportedDataTypes(), + buildSupportedVectorTypes()); this.collectionsTable = collectionsTable; this.prefixForCollectionTables = prefixForCollectionTables; this.objectMapper = objectMapper; + } - supportedKeyTypes = new HashMap<>(); - supportedKeyTypes.put(String.class, "VARCHAR(255)"); + private static Map, String> buildSupportedVectorTypes() { + HashMap, String> supportedVectorTypes = new HashMap<>(); + supportedVectorTypes.put(String.class, "TEXT"); + supportedVectorTypes.put(List.class, "VECTOR(%d)"); + supportedVectorTypes.put(Collection.class, "VECTOR(%d)"); + return supportedVectorTypes; + } - supportedDataTypes = new HashMap<>(); + private static Map, String> buildSupportedDataTypes() { + HashMap, String> supportedDataTypes = new HashMap<>(); supportedDataTypes.put(String.class, "TEXT"); supportedDataTypes.put(Integer.class, "INTEGER"); supportedDataTypes.put(int.class, "INTEGER"); @@ -74,41 +84,14 @@ private PostgreSQLVectorStoreQueryProvider( supportedDataTypes.put(Boolean.class, "BOOLEAN"); supportedDataTypes.put(boolean.class, "BOOLEAN"); supportedDataTypes.put(OffsetDateTime.class, "TIMESTAMPTZ"); - - supportedVectorTypes = new HashMap<>(); - supportedDataTypes.put(String.class, "TEXT"); - supportedVectorTypes.put(List.class, "VECTOR(%d)"); - supportedVectorTypes.put(Collection.class, "VECTOR(%d)"); + supportedDataTypes.put(List.class, "JSONB"); + return supportedDataTypes; } - /** - * Gets the supported key types and their corresponding SQL types. - * - * @return the supported key types - */ - @Override - public Map, String> getSupportedKeyTypes() { - return new HashMap<>(this.supportedKeyTypes); - } - - /** - * Gets the supported data types and their corresponding SQL types. - * - * @return the supported data types - */ - @Override - public Map, String> getSupportedDataTypes() { - return new HashMap<>(this.supportedDataTypes); - } - - /** - * Gets the supported vector types and their corresponding SQL types. - * - * @return the supported vector types - */ - @Override - public Map, String> getSupportedVectorTypes() { - return new HashMap<>(this.supportedVectorTypes); + private static HashMap, String> buildSupportedKeyTypes() { + HashMap, String> supportedKeyTypes = new HashMap<>(); + supportedKeyTypes.put(String.class, "VARCHAR(255)"); + return supportedKeyTypes; } /** @@ -253,6 +236,12 @@ private void setUpsertStatementValues(PreparedStatement statement, Object record statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); continue; } + } else if (field instanceof VectorStoreRecordDataField) { + // Convert List field to a string + if (field.getFieldType().equals(List.class)) { + statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); + continue; + } } statement.setObject(i + 1, @@ -271,6 +260,12 @@ private String getWildcardStringWithCast(List fields) { if (field instanceof VectorStoreRecordVectorField) { wildcard += "::vector"; } + if (field instanceof VectorStoreRecordDataField) { + // Add casting for List fields + if (field.getFieldType().equals(List.class)) { + wildcard += "::jsonb"; + } + } return wildcard; }) .collect(Collectors.joining(", ")); @@ -367,11 +362,8 @@ public List> search(String collectionName, "Distance function is required for vector field: " + vectorField.getName()); } - String filter = SQLVectorStoreRecordCollectionSearchMapping.buildFilter( - options.getVectorSearchFilter(), - recordDefinition); - List parameters = SQLVectorStoreRecordCollectionSearchMapping - .getFilterParameters(options.getVectorSearchFilter()); + String filter = getFilter(options.getVectorSearchFilter(), recordDefinition); + List parameters = getFilterParameters(options.getVectorSearchFilter()); String filterClause = filter.isEmpty() ? "" : "WHERE " + filter; String searchQuery = formatQuery( @@ -413,6 +405,35 @@ public List> search(String collectionName, } } + @Override + public List getFilterParameters(VectorSearchFilter vectorSearchFilter) { + if (vectorSearchFilter == null + || vectorSearchFilter.getFilterClauses().isEmpty()) { + return Collections.emptyList(); + } + + return vectorSearchFilter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof EqualToFilterClause) { + EqualToFilterClause equalToFilterClause = (EqualToFilterClause) filterClause; + return equalToFilterClause.getValue(); + } else if (filterClause instanceof AnyTagEqualToFilterClause) { + AnyTagEqualToFilterClause anyTagEqualToFilterClause = (AnyTagEqualToFilterClause) filterClause; + return String.format("[\"%s\"]", anyTagEqualToFilterClause.getValue()); + } else { + throw new SKException("Unsupported filter clause type '" + + filterClause.getClass().getSimpleName() + "'."); + } + }).collect(Collectors.toList()); + } + + @Override + public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { + String fieldName = JDBCVectorStoreQueryProvider + .validateSQLidentifier(filterClause.getFieldName()); + + return String.format("%s @> ?::jsonb", fieldName); + } + public static class Builder extends JDBCVectorStoreQueryProvider.Builder { private DataSource dataSource; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java index 9550ade8..410f708e 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; @@ -113,15 +114,20 @@ public PostgreSQLVectorStoreRecordMapper build() { for (VectorStoreRecordField field : fields) { Object value = resultSet.getObject(field.getEffectiveStorageName()); + Class fieldType = field.getFieldType(); if (field instanceof VectorStoreRecordVectorField) { - Class vectorType = field.getFieldType(); - // If the vector field is other than String, deserialize it from the JSON string - if (!vectorType.equals(String.class)) { + if (!fieldType.equals(String.class)) { // Deserialize the pgvector string to the vector type value = objectMapper.readValue(((PGobject) value).getValue(), - vectorType); + fieldType); + } + } else if (field instanceof VectorStoreRecordDataField) { + // If the field is List, deserialize it from the JSON string + if (fieldType.equals(List.class)) { + value = objectMapper.readValue(((PGobject) value).getValue(), + fieldType); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java index 115fc43b..71cc79aa 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java @@ -395,6 +395,7 @@ public Mono>> searchAsync(List vector, return createCollectionIfNotExistsAsync().flatMap(collection -> Mono.fromCallable(() -> { Pair ftSearchParams = RedisVectorStoreCollectionSearchMapping + .getInstance() .buildQuery(vector, options, recordDefinition, RedisStorageType.HASH_SET); SearchResult searchResult = client.ftSearch(collectionName, ftSearchParams.getLeft(), diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java index d2df1766..0a7c2c15 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java @@ -411,7 +411,7 @@ public Mono>> searchAsync(List vector, return createCollectionIfNotExistsAsync().flatMap(collection -> Mono.fromCallable(() -> { Pair ftSearchParams = RedisVectorStoreCollectionSearchMapping - .buildQuery(vector, options, recordDefinition, RedisStorageType.JSON); + .getInstance().buildQuery(vector, options, recordDefinition, RedisStorageType.JSON); SearchResult searchResult = client.ftSearch(collectionName, ftSearchParams.getLeft(), ftSearchParams.getRight()); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java index f9ea409b..0b792996 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java @@ -1,8 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; -import com.microsoft.semantickernel.connectors.data.redis.filter.RedisEqualToFilterClause; +import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; +import com.microsoft.semantickernel.data.filter.EqualToFilterClause; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.filter.FilterMapping; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; @@ -16,11 +18,22 @@ import redis.clients.jedis.args.SortingOrder; import redis.clients.jedis.search.FTSearchParams; -public class RedisVectorStoreCollectionSearchMapping { +public class RedisVectorStoreCollectionSearchMapping implements FilterMapping { static final String VECTOR_SCORE_FIELD = "vector_score"; - public static Pair buildQuery(List vector, + private RedisVectorStoreCollectionSearchMapping() { + } + + static class RedisVectorStoreCollectionSearchMappingHolder { + static final RedisVectorStoreCollectionSearchMapping INSTANCE = new RedisVectorStoreCollectionSearchMapping(); + } + + static RedisVectorStoreCollectionSearchMapping getInstance() { + return RedisVectorStoreCollectionSearchMappingHolder.INSTANCE; + } + + public Pair buildQuery(List vector, VectorSearchOptions options, VectorStoreRecordDefinition recordDefinition, RedisStorageType storageType) { @@ -34,7 +47,7 @@ public static Pair buildQuery(List vector, : (VectorStoreRecordVectorField) recordDefinition .getField(options.getVectorFieldName()); - String filter = buildFilter(options.getVectorSearchFilter(), recordDefinition); + String filter = getFilter(options.getVectorSearchFilter(), recordDefinition); String knn = String.format("%s=>[KNN $K @%s $BLOB AS %s]", filter, vectorField.getEffectiveStorageName(), VECTOR_SCORE_FIELD); @@ -82,25 +95,68 @@ public static List convertByteArrayToList(byte[] bytes) { return embeddings; } - public static String buildFilter(VectorSearchFilter vectorSearchFilter, + /** + * Gets the filter string for the given vector search filter and record definition. + * + * @param filter The filter to get the filter string for. + * @param recordDefinition The record definition to get the filter string for. + * @return The filter string. + */ + @Override + public String getFilter(VectorSearchFilter filter, VectorStoreRecordDefinition recordDefinition) { - if (vectorSearchFilter == null - || vectorSearchFilter.getFilterClauses().isEmpty()) { + if (filter == null + || filter.getFilterClauses().isEmpty()) { return "*"; } return String.format("(%s)", - vectorSearchFilter.getFilterClauses().stream().map(filterClause -> { - if (filterClause instanceof RedisEqualToFilterClause) { - RedisEqualToFilterClause equalToFilterClause = (RedisEqualToFilterClause) filterClause; - return new RedisEqualToFilterClause( + filter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof EqualToFilterClause) { + EqualToFilterClause equalToFilterClause = (EqualToFilterClause) filterClause; + return getEqualToFilter(new EqualToFilterClause( recordDefinition.getField(equalToFilterClause.getFieldName()) .getEffectiveStorageName(), - equalToFilterClause.getValue()).getFilter(); + equalToFilterClause.getValue())); } else { throw new SKException("Unsupported filter clause type '" + filterClause.getClass().getSimpleName() + "'."); } }).collect(Collectors.joining(" "))); } + + /** + * Gets the filter string for the given equal to filter clause. + * + * @param filterClause The equal to filter clause to get the filter string for. + * @return The filter string. + */ + @Override + public String getEqualToFilter(EqualToFilterClause filterClause) { + String fieldName = filterClause.getFieldName(); + Object value = filterClause.getValue(); + String formattedValue; + + if (value instanceof String) { + formattedValue = String.format("\"%s\"", value); + } else if (value instanceof Number) { + formattedValue = String.format("[%s %s]", value, value); + } else { + throw new SKException("Unsupported filter value type '" + + value.getClass().getSimpleName() + "'."); + } + + return String.format("@%s:%s", fieldName, formattedValue); + } + + /** + * Gets the filter string for the given any tag equal to filter clause. + * + * @param filterClause The any tag equal to filter clause to get the filter string for. + * @return The filter string. + */ + @Override + public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { + return String.format("@%s:\"%s\"", filterClause.getFieldName(), filterClause.getValue()); + } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/filter/RedisEqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/filter/RedisEqualToFilterClause.java deleted file mode 100644 index aa823ef9..00000000 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/filter/RedisEqualToFilterClause.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.redis.filter; - -import com.microsoft.semantickernel.data.filter.EqualToFilterClause; -import com.microsoft.semantickernel.exceptions.SKException; - -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; - -public class RedisEqualToFilterClause extends EqualToFilterClause { - public RedisEqualToFilterClause(String fieldName, Object value) { - super(fieldName, value); - } - - /** - * Gets the filter string. - * - * @return The filter. - */ - @Override - public String getFilter() { - String fieldName = getFieldName(); - Object value = getValue(); - String formattedValue; - - if (value instanceof String) { - formattedValue = String.format("\"%s\"", value); - } else if (value instanceof Boolean) { - formattedValue = value.toString().toLowerCase(); - } else if (value instanceof Number) { - formattedValue = value.toString(); - } else if (value instanceof OffsetDateTime) { - formattedValue = ((OffsetDateTime) value) - .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); - } else { - throw new SKException("Unsupported filter value type '" - + value.getClass().getSimpleName() + "'."); - } - - return String.format("@%s:%s", fieldName, formattedValue); - } -} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/sqlite/SQLiteVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/sqlite/SQLiteVectorStoreQueryProvider.java index c501c99f..35d38150 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/sqlite/SQLiteVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/sqlite/SQLiteVectorStoreQueryProvider.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; @@ -61,6 +62,12 @@ private void setUpsertStatementValues(PreparedStatement statement, Object record statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); continue; } + } else if (field instanceof VectorStoreRecordDataField) { + // Convert List field to a string + if (field.getFieldType().equals(List.class)) { + statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); + continue; + } } statement.setObject(i + 1, diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java index 1b940717..ab1a8754 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.filter; -public abstract class AnyTagEqualToFilterClause implements FilterClause { +public class AnyTagEqualToFilterClause implements FilterClause { private final String fieldName; private final Object value; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java index 3590b252..68b0b5ec 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java @@ -2,7 +2,6 @@ package com.microsoft.semantickernel.data.filter; public class EqualToFilterClause implements FilterClause { - private final String fieldName; private final Object value; @@ -28,15 +27,4 @@ public String getFieldName() { public Object getValue() { return value; } - - /** - * Gets the filter string. - * - * @return The filter. - */ - @Override - public String getFilter() { - throw new UnsupportedOperationException(String.format( - "Not implemented. Use one of %s derived classes.", this.getClass().getSimpleName())); - } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java index e3425be6..2c309922 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java @@ -2,11 +2,4 @@ package com.microsoft.semantickernel.data.filter; public interface FilterClause { - - /** - * Gets the filter string. - * - * @return The filter. - */ - String getFilter(); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java new file mode 100644 index 00000000..6483ab40 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.filter; + +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; + +public interface FilterMapping { + /** + * Gets the filter string for the given vector search filter and record definition. + * + * @param filter The filter to get the filter string for. + * @param recordDefinition The record definition to get the filter string for. + * @return The filter string. + */ + String getFilter(VectorSearchFilter filter, VectorStoreRecordDefinition recordDefinition); + + /** + * Gets the filter string for the given equal to filter clause. + * + * @param filterClause The equal to filter clause to get the filter string for. + * @return The filter string. + */ + String getEqualToFilter(EqualToFilterClause filterClause); + + /** + * Gets the filter string for the given any tag equal to filter clause. + * + * @param filterClause The any tag equal to filter clause to get the filter string for. + * @return The filter string. + */ + String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause); +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java index cde9594f..18a2b7e9 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java @@ -54,23 +54,24 @@ public static class Builder { /** * Adds an EqualToFilterClause to the filter. * - * @param equalToFilterClause The EqualToFilterClause to add. + * @param fieldName The field name to filter on. + * @param value The value. * @return The builder. */ - public Builder withEqualToFilterClause(EqualToFilterClause equalToFilterClause) { - filterClauses.add(equalToFilterClause); + public Builder equalTo(String fieldName, Object value) { + filterClauses.add(new EqualToFilterClause(fieldName, value)); return this; } /** * Adds an AnyTagEqualToFilterClause to the filter. * - * @param anyTagEqualToFilterClause The AnyTagEqualToFilterClause clause to add. + * @param fieldName The field name to filter on. + * @param value The value. * @return The builder. */ - public Builder withAnyTagEqualToFilterClause( - AnyTagEqualToFilterClause anyTagEqualToFilterClause) { - filterClauses.add(anyTagEqualToFilterClause); + public Builder anyTagEqualTo(String fieldName, Object value) { + filterClauses.add(new AnyTagEqualToFilterClause(fieldName, value)); return this; } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordDataAttribute.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordData.java similarity index 94% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordDataAttribute.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordData.java index a5103105..2deac3e0 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordDataAttribute.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordData.java @@ -11,7 +11,7 @@ */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.FIELD) -public @interface VectorStoreRecordDataAttribute { +public @interface VectorStoreRecordData { /** * Storage name of the field. * This value is only used when JSON Serialization using Jackson is not supported in a VectorStore. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKeyAttribute.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKey.java similarity index 90% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKeyAttribute.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKey.java index 8d54d60b..cdecdcd2 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKeyAttribute.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKey.java @@ -11,7 +11,7 @@ */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.FIELD) -public @interface VectorStoreRecordKeyAttribute { +public @interface VectorStoreRecordKey { /** * Storage name of the field. */ diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVector.java similarity index 95% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVector.java index a72c2c69..6594c469 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVector.java @@ -15,7 +15,7 @@ */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.FIELD) -public @interface VectorStoreRecordVectorAttribute { +public @interface VectorStoreRecordVector { /** * Number of dimensions in the vector. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java index eeedfe73..3a9550f0 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java @@ -2,9 +2,9 @@ package com.microsoft.semantickernel.data.vectorstorage.definition; import com.fasterxml.jackson.annotation.JsonProperty; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVector; import com.microsoft.semantickernel.exceptions.SKException; import java.lang.reflect.Field; import java.util.ArrayList; @@ -159,9 +159,9 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) storageName = field.getAnnotation(JsonProperty.class).value(); } - if (field.isAnnotationPresent(VectorStoreRecordKeyAttribute.class)) { - VectorStoreRecordKeyAttribute keyAttribute = field - .getAnnotation(VectorStoreRecordKeyAttribute.class); + if (field.isAnnotationPresent(VectorStoreRecordKey.class)) { + VectorStoreRecordKey keyAttribute = field + .getAnnotation(VectorStoreRecordKey.class); if (storageName == null) { storageName = keyAttribute.storageName().isEmpty() ? field.getName() @@ -174,9 +174,9 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) .build()); } - if (field.isAnnotationPresent(VectorStoreRecordDataAttribute.class)) { - VectorStoreRecordDataAttribute dataAttribute = field - .getAnnotation(VectorStoreRecordDataAttribute.class); + if (field.isAnnotationPresent(VectorStoreRecordData.class)) { + VectorStoreRecordData dataAttribute = field + .getAnnotation(VectorStoreRecordData.class); if (storageName == null) { storageName = dataAttribute.storageName().isEmpty() ? field.getName() @@ -190,9 +190,9 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) .build()); } - if (field.isAnnotationPresent(VectorStoreRecordVectorAttribute.class)) { - VectorStoreRecordVectorAttribute vectorAttribute = field - .getAnnotation(VectorStoreRecordVectorAttribute.class); + if (field.isAnnotationPresent(VectorStoreRecordVector.class)) { + VectorStoreRecordVector vectorAttribute = field + .getAnnotation(VectorStoreRecordVector.class); if (storageName == null) { storageName = vectorAttribute.storageName().isEmpty() ? field.getName() diff --git a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java index 7d5b62e5..26af9f95 100644 --- a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java +++ b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java @@ -3,43 +3,43 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVector; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import java.util.List; public class Hotel { - @VectorStoreRecordKeyAttribute + @VectorStoreRecordKey private final String id; - @VectorStoreRecordDataAttribute(isFilterable = true) + @VectorStoreRecordData(isFilterable = true) private final String name; - @VectorStoreRecordDataAttribute + @VectorStoreRecordData private final int code; @JsonProperty("summary") - @VectorStoreRecordDataAttribute() + @VectorStoreRecordData() private final String description; @JsonProperty("summaryEmbedding1") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE) private final List euclidean; @JsonProperty("summaryEmbedding2") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE) + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List cosineDistance; @JsonProperty("summaryEmbedding3") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.COSINE_SIMILARITY) + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_SIMILARITY) private final List cosineSimilarity; @JsonProperty("summaryEmbedding4") - @VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = DistanceFunction.DOT_PRODUCT) + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.DOT_PRODUCT) private final List dotProduct; - @VectorStoreRecordDataAttribute + @VectorStoreRecordData private double rating; public Hotel() { diff --git a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java index d2084967..23b1ac78 100644 --- a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java +++ b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java @@ -6,13 +6,10 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; -import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollection; -import com.microsoft.semantickernel.connectors.data.jdbc.filter.SQLEqualToFilterClause; import com.microsoft.semantickernel.data.filter.EqualToFilterClause; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; @@ -23,7 +20,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; -import org.junit.jupiter.params.provider.MethodSource; public class VolatileVectorStoreRecordCollectionTest { @@ -198,7 +194,7 @@ public void searchWithFilter(DistanceFunction distanceFunction) { .withLimit(3) .withVectorSearchFilter( VectorSearchFilter.builder() - .withEqualToFilterClause(new EqualToFilterClause("rating", 4.0)).build()) + .equalTo("rating", 4.0).build()) .build(); // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned From 72d6c3109fcf594291be25ea0433de3a6d9fac2f Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Wed, 16 Oct 2024 16:46:36 -0700 Subject: [PATCH 076/277] Learn exampels --- .../semantickernel-learn-resources/pom.xml | 9 +++ .../connectors/azureaisearch/Main.java | 30 ++++++++++ .../connectors/inmemory/Main.java | 18 ++++++ .../vectorstores/connectors/jdbc/Main.java | 43 ++++++++++++++ .../vectorstores/connectors/redis/Main.java | 36 ++++++++++++ .../vectorstores/index/Hotel.java | 37 ++++++++++++ .../vectorstores/index/Main.java | 55 ++++++++++++++++++ .../vectorstores/recorddefinition/Main.java | 58 +++++++++++++++++++ 8 files changed, 286 insertions(+) create mode 100644 samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/connectors/azureaisearch/Main.java create mode 100644 samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/connectors/inmemory/Main.java create mode 100644 samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/connectors/jdbc/Main.java create mode 100644 samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/connectors/redis/Main.java create mode 100644 samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/index/Hotel.java create mode 100644 samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/index/Main.java create mode 100644 samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/recorddefinition/Main.java diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index f042a3ef..055d1bf2 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -62,6 +62,15 @@ com.microsoft.semantic-kernel semantickernel-aiservices-openai + + com.microsoft.semantic-kernel + semantickernel-experimental + + + com.mysql + mysql-connector-j + 9.0.0 + diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/connectors/azureaisearch/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/connectors/azureaisearch/Main.java new file mode 100644 index 00000000..738b56c9 --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/connectors/azureaisearch/Main.java @@ -0,0 +1,30 @@ +package com.microsoft.semantickernel.samples.documentationexamples.vectorstores.connectors.azureaisearch; + +import com.azure.core.credential.AzureKeyCredential; +import com.azure.search.documents.indexes.SearchIndexClientBuilder; +import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStore; +import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreOptions; +import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreRecordCollection; +import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.samples.documentationexamples.vectorstores.index.Hotel; + +public class Main { + public static void main(String[] args) { + // Build the Azure AI Search client + var searchClient = new SearchIndexClientBuilder() + .endpoint("https://.search.windows.net") + .credential(new AzureKeyCredential("")) + .buildAsyncClient(); + + // Build an Azure AI Search Vector Store + var vectorStore = AzureAISearchVectorStore.builder() + .withSearchIndexAsyncClient(searchClient) + .withOptions(new AzureAISearchVectorStoreOptions()) + .build(); + + var collection = new AzureAISearchVectorStoreRecordCollection<>(searchClient, "skhotels", + AzureAISearchVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); + } +} diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/connectors/inmemory/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/connectors/inmemory/Main.java new file mode 100644 index 00000000..1fd5db2c --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/connectors/inmemory/Main.java @@ -0,0 +1,18 @@ +package com.microsoft.semantickernel.samples.documentationexamples.vectorstores.connectors.inmemory; + +import com.microsoft.semantickernel.data.VolatileVectorStore; +import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollection; +import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.samples.documentationexamples.vectorstores.index.Hotel; + +public class Main { + public static void main(String[] args) { + // Build an Azure AI Search Vector Store + var vectorStore = new VolatileVectorStore(); + + var collection = new VolatileVectorStoreRecordCollection<>("skhotels", + VolatileVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); + } +} diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/connectors/jdbc/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/connectors/jdbc/Main.java new file mode 100644 index 00000000..6da2af82 --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/connectors/jdbc/Main.java @@ -0,0 +1,43 @@ +package com.microsoft.semantickernel.samples.documentationexamples.vectorstores.connectors.jdbc; + +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollection; +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.samples.documentationexamples.vectorstores.index.Hotel; +import com.mysql.cj.jdbc.MysqlDataSource; + +public class Main { + public static void main(String[] args) { + // Configure the data source + var dataSource = new MysqlDataSource(); + dataSource.setUrl("jdbc:mysql://localhost:3306/sk"); + dataSource.setPassword("root"); + dataSource.setUser("root"); + + // Build a query provider + // Other available query providers are PostgreSQLVectorStoreQueryProvider, SQLiteVectorStoreQueryProvider + // and HSQDBVectorStoreQueryProvider + var queryProvider = MySQLVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build(); + + // Build a vector store + var vectorStore = JDBCVectorStore.builder() + .withDataSource(dataSource) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + + var collection = new JDBCVectorStoreRecordCollection<>( + dataSource, + "skhotels", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build() + ); + } +} diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/connectors/redis/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/connectors/redis/Main.java new file mode 100644 index 00000000..792f9d47 --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/connectors/redis/Main.java @@ -0,0 +1,36 @@ +package com.microsoft.semantickernel.samples.documentationexamples.vectorstores.connectors.redis; + +import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.connectors.data.redis.RedisStorageType; +import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStore; +import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreOptions; +import com.microsoft.semantickernel.samples.documentationexamples.vectorstores.index.Hotel; +import redis.clients.jedis.JedisPooled; + +public class Main { + public static void main(String[] args) { + JedisPooled jedis = new JedisPooled(""); + + // Build a Redis Vector Store + // Available storage types are JSON and HASHSET. Default is JSON. + var vectorStore = RedisVectorStore.builder() + .withClient(jedis) + .withOptions( + RedisVectorStoreOptions.builder() + .withStorageType(RedisStorageType.JSON).build()) + .build(); + + var collection = vectorStore.getCollection("skhotels", + RedisJsonVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); + + collection = vectorStore.getCollection("skhotels", + RedisJsonVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .withPrefixCollectionName(false) + .build()); + + collection.getAsync("myprefix_h1", null).block(); + } +} diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/index/Hotel.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/index/Hotel.java new file mode 100644 index 00000000..5a0d3377 --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/index/Hotel.java @@ -0,0 +1,37 @@ +package com.microsoft.semantickernel.samples.documentationexamples.vectorstores.index; + +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; + +import java.util.List; + +public class Hotel { + @VectorStoreRecordKeyAttribute + private String hotelId; + + @VectorStoreRecordDataAttribute(isFilterable = true) + private String name; + + @VectorStoreRecordDataAttribute(isFullTextSearchable = true) + private String description; + + @VectorStoreRecordVectorAttribute(dimensions = 4, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE) + private List descriptionEmbedding; + + public Hotel() { } + + public Hotel(String hotelId, String name, String description, List descriptionEmbedding) { + this.hotelId = hotelId; + this.name = name; + this.description = description; + this.descriptionEmbedding = descriptionEmbedding; + } + + public String getHotelId() { return hotelId; } + public String getName() { return name; } + public String getDescription() { return description; } + public List getDescriptionEmbedding() { return descriptionEmbedding; } +} diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/index/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/index/Main.java new file mode 100644 index 00000000..ba5bb02d --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/index/Main.java @@ -0,0 +1,55 @@ +package com.microsoft.semantickernel.samples.documentationexamples.vectorstores.index; + +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; +import com.mysql.cj.jdbc.MysqlDataSource; + +import java.util.List; + +public class Main { + public static void main(String[] args) { + // Create a MySQL data source + var dataSource = new MysqlDataSource(); + dataSource.setUrl("jdbc:mysql://localhost:3306/sk"); + dataSource.setPassword("root"); + dataSource.setUser("root"); + + // Create a JDBC vector store + var vectorStore = JDBCVectorStore.builder() + .withDataSource(dataSource) + .withOptions( + JDBCVectorStoreOptions.builder() + .withQueryProvider(MySQLVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build()) + .build() + ) + .build(); + + // Get a collection from the vector store + var collection = vectorStore.getCollection("skhotels", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build() + ); + + // Create the collection if it doesn't exist yet. + collection.createCollectionAsync().block(); + + // Upsert a record. + var description = "A place where everyone can be happy"; + var hotelId = "hotel1"; + var hotel = new Hotel(hotelId, "Hotel Happy", description, generateEmbeddings(description)); + + collection.upsertAsync(hotel, null).block(); + + // Retrieve the upserted record. + var retrievedHotel = collection.getAsync(hotelId, null).block(); + } + + private static List generateEmbeddings(String text) { + return List.of(1.0f, 2.0f, 3.0f, 4.0f); + } +} diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/recorddefinition/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/recorddefinition/Main.java new file mode 100644 index 00000000..32310f57 --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/vectorstores/recorddefinition/Main.java @@ -0,0 +1,58 @@ +package com.microsoft.semantickernel.samples.documentationexamples.vectorstores.recorddefinition; + +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorstorage.definition.*; +import com.mysql.cj.jdbc.MysqlDataSource; + +import java.util.Arrays; +import java.util.List; + +public class Main { + public static void main(String[] args) { + // Create a MySQL data source + var dataSource = new MysqlDataSource(); + dataSource.setUrl("jdbc:mysql://localhost:3306/sk"); + dataSource.setPassword("root"); + dataSource.setUser("root"); + + // Create a JDBC vector store + var vectorStore = JDBCVectorStore.builder() + .withDataSource(dataSource) + .withOptions( + JDBCVectorStoreOptions.builder() + .withQueryProvider(MySQLVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build()) + .build() + ) + .build(); + + var hotelDefinition = VectorStoreRecordDefinition.fromFields( + Arrays.asList( + VectorStoreRecordKeyField.builder().withName("hotelId").withFieldType(String.class).build(), + VectorStoreRecordDataField.builder() + .withName("name") + .withFieldType(String.class) + .isFilterable(true).build(), + VectorStoreRecordDataField.builder() + .withName("description") + .withFieldType(String.class) + .isFullTextSearchable(true).build(), + VectorStoreRecordVectorField.builder().withName("descriptionEmbedding") + .withDimensions(4) + .withIndexKind(IndexKind.HNSW) + .withDistanceFunction(DistanceFunction.COSINE_DISTANCE) + .withFieldType(List.class).build() + ) + ); + + var collection = vectorStore.getCollection("skhotels", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordDefinition(hotelDefinition) + .build() + ); + } +} From b14bd82ea4857f3f5513bb0d2453580f8c984710 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Wed, 16 Oct 2024 16:48:16 -0700 Subject: [PATCH 077/277] Update --- .../semantickernel/tests/connectors/memory/jdbc/Hotel.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java index a7a11901..b1a2a294 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java @@ -8,6 +8,7 @@ import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; +import java.util.Collections; import java.util.List; public class Hotel { @@ -72,7 +73,7 @@ public Hotel( this.dotProduct = euclidean; this.indexedEuclidean = euclidean; this.rating = rating; - this.tags = tags; + this.tags = Collections.unmodifiableList(tags); } public String getId() { From ede15f4c78b27c0eac97e02a336fa15d8f5ecd09 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Mon, 21 Oct 2024 23:28:13 -0700 Subject: [PATCH 078/277] Update --- .../semantickernel/tests/connectors/memory/jdbc/Hotel.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java index b1a2a294..d69b73af 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java @@ -73,7 +73,7 @@ public Hotel( this.dotProduct = euclidean; this.indexedEuclidean = euclidean; this.rating = rating; - this.tags = Collections.unmodifiableList(tags); + this.tags = tags; } public String getId() { From be089f9f9e788e8f642eb0e7f2bf21ee469bef38 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 22 Oct 2024 00:54:38 -0700 Subject: [PATCH 079/277] Update annotations package --- .../semantickernel/tests/connectors/memory/jdbc/Hotel.java | 7 +++---- .../tests/connectors/memory/redis/Hotel.java | 6 +++--- .../syntaxexamples/memory/InMemoryVolatileVectorStore.java | 6 +++--- .../memory/VectorStoreWithAzureAISearch.java | 6 +++--- .../samples/syntaxexamples/memory/VectorStoreWithJDBC.java | 7 +++---- .../syntaxexamples/memory/VectorStoreWithRedis.java | 6 +++--- .../{attributes => annotations}/VectorStoreRecordData.java | 2 +- .../{attributes => annotations}/VectorStoreRecordKey.java | 2 +- .../VectorStoreRecordVector.java | 2 +- .../definition/VectorStoreRecordDefinition.java | 6 +++--- .../test/java/com/microsoft/semantickernel/data/Hotel.java | 6 +++--- 11 files changed, 27 insertions(+), 29 deletions(-) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/{attributes => annotations}/VectorStoreRecordData.java (92%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/{attributes => annotations}/VectorStoreRecordKey.java (86%) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/{attributes => annotations}/VectorStoreRecordVector.java (93%) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java index d69b73af..d78a333d 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java @@ -2,13 +2,12 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordData; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKey; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; -import java.util.Collections; import java.util.List; public class Hotel { diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java index 478adb32..da64efb0 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java @@ -2,9 +2,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordData; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKey; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java index 192f1189..8dc1372d 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java @@ -11,9 +11,9 @@ import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.VolatileVectorStore; import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordData; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKey; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import java.util.Arrays; import java.util.Collections; import java.util.List; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java index 59822601..dca8599b 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java @@ -17,9 +17,9 @@ import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordData; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKey; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import java.nio.charset.StandardCharsets; import java.util.Arrays; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java index e842d6e1..2099886b 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java @@ -5,7 +5,6 @@ import com.azure.ai.openai.OpenAIClientBuilder; import com.azure.core.credential.AzureKeyCredential; import com.azure.core.credential.KeyCredential; -import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStore; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions; @@ -13,9 +12,9 @@ import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordData; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKey; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.mysql.cj.jdbc.MysqlDataSource; import java.nio.charset.StandardCharsets; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java index 7d6976ce..0310c9dd 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java @@ -13,9 +13,9 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreOptions; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordData; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKey; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import java.util.Arrays; import java.util.Collections; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordData.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java similarity index 92% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordData.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java index 2deac3e0..b27423ce 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordData.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorstorage.attributes; +package com.microsoft.semantickernel.data.vectorstorage.annotations; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKey.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java similarity index 86% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKey.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java index cdecdcd2..cd462dfc 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKey.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorstorage.attributes; +package com.microsoft.semantickernel.data.vectorstorage.annotations; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVector.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java similarity index 93% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVector.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java index 6594c469..91d5571d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVector.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorstorage.attributes; +package com.microsoft.semantickernel.data.vectorstorage.annotations; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java index 3a9550f0..6b72d529 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java @@ -2,9 +2,9 @@ package com.microsoft.semantickernel.data.vectorstorage.definition; import com.fasterxml.jackson.annotation.JsonProperty; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordData; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKey; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import com.microsoft.semantickernel.exceptions.SKException; import java.lang.reflect.Field; import java.util.ArrayList; diff --git a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java index 26af9f95..f6741e5a 100644 --- a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java +++ b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java @@ -3,9 +3,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordData; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKey; -import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import java.util.List; From 3e7f6c83466a84559244c4b42bf0ad4b60b44a3f Mon Sep 17 00:00:00 2001 From: David Grieve Date: Tue, 22 Oct 2024 12:38:33 -0600 Subject: [PATCH 080/277] Eliminate javadoc warnings (#246) * fix javadoc warnings in semantickernel-api * fix javadoc warnings in aiservices/google * fix javadoc warnings in aiservices/huggingface * fix javadoc warnings in aiservices/openai * fix javadoc warnings in semantickernel-experimental * HuggingFaceClient Builder missing 'public' --- .../aiservices/google/GeminiService.java | 13 +++ .../google/GeminiServiceBuilder.java | 2 + .../chatcompletion/GeminiChatCompletion.java | 21 ++++- .../chatcompletion/GeminiFunctionCall.java | 24 ++++++ .../google/chatcompletion/GeminiRole.java | 3 + .../GeminiStreamingChatMessageContent.java | 1 + .../chatcompletion/GeminiXMLPromptParser.java | 24 ++++++ .../GeminiTextGenerationService.java | 16 ++++ .../huggingface/HuggingFaceClient.java | 43 +++++++++- .../huggingface/models/GeneratedTextItem.java | 82 ++++++++++++++++++ .../models/TextGenerationRequest.java | 84 ++++++++++++++++++- .../HuggingFacePromptExecutionSettings.java | 51 +++++++++++ .../HuggingFaceTextGenerationService.java | 32 +++++++ .../aiservices/openai/OpenAiService.java | 9 ++ .../audio/OpenAiAudioToTextService.java | 1 + .../audio/OpenAiTextToAudioService.java | 2 +- .../OpenAIStreamingChatMessageContent.java | 17 ++++ ...atCompletionsJsonSchemaResponseFormat.java | 10 +++ .../JacksonResponseFormatGenerator.java | 11 +++ .../OpenAITextEmbeddingGenerationService.java | 9 ++ pom.xml | 1 + .../com/microsoft/semantickernel/Kernel.java | 20 +++++ .../ContextVariableTypeConverter.java | 16 +++- .../ContextVariableTypes.java | 8 ++ ...nVariableContextVariableTypeConverter.java | 11 ++- ...eVariableContextVariableTypeConverter.java | 1 + ...gVariableContextVariableTypeConverter.java | 8 ++ ...tVariableContextVariableTypeConverter.java | 6 ++ .../exceptions/SKCheckedException.java | 27 +++--- .../exceptions/SKException.java | 24 ++++-- .../hooks/PreToolCallEvent.java | 10 ++- .../localization/SemanticKernelResources.java | 31 ++++++- .../orchestration/FunctionResultMetadata.java | 4 + .../orchestration/InvocationReturnMode.java | 3 + .../orchestration/ToolCallBehavior.java | 3 + .../JsonObjectResponseFormat.java | 7 ++ .../responseformat/JsonResponseSchema.java | 23 +++++ .../JsonSchemaResponseFormat.java | 55 ++++++++++++ .../responseformat/ResponseFormat.java | 28 ++++++- .../ResponseSchemaGenerator.java | 15 +++- .../responseformat/TextResponseFormat.java | 7 ++ .../plugin/KernelPluginFactory.java | 1 + .../HandlebarsPromptTemplateFactory.java | 6 ++ .../semanticfunctions/InputVariable.java | 1 + .../semanticfunctions/KernelFunction.java | 3 + .../KernelFunctionFromMethod.java | 16 +++- .../KernelFunctionFromPrompt.java | 1 - .../KernelPromptTemplateFactory.java | 6 ++ .../semanticfunctions/OutputVariable.java | 1 + .../PromptTemplateConfig.java | 21 +++-- .../PromptTemplateOption.java | 3 + .../services/KernelContent.java | 4 +- .../services/StreamingKernelContent.java | 4 + .../services/audio/AudioToTextService.java | 4 + .../audio/TextToAudioExecutionSettings.java | 3 + .../services/chatcompletion/ChatHistory.java | 13 +++ .../chatcompletion/StreamingChatContent.java | 8 ++ .../message/ChatMessageImageContent.java | 26 ++++-- .../message/ChatMessageTextContent.java | 3 + .../services/textembedding/Embedding.java | 4 + .../EmbeddingGenerationService.java | 3 +- .../AzureAISearchVectorStore.java | 3 + ...rchVectorStoreCollectionCreateMapping.java | 34 ++++++++ .../AzureAISearchVectorStoreOptions.java | 3 + ...reAISearchVectorStoreRecordCollection.java | 12 +++ ...rchVectorStoreRecordCollectionFactory.java | 1 + ...rchVectorStoreRecordCollectionOptions.java | 5 ++ ...zureAISearchAnyTagEqualToFilterClause.java | 5 +- .../AzureAISearchEqualToFilterClause.java | 5 +- .../HSQLDBVectorStoreQueryProvider.java | 25 +++++- .../connectors/data/jdbc/JDBCVectorStore.java | 1 + .../data/jdbc/JDBCVectorStoreOptions.java | 6 +- .../jdbc/JDBCVectorStoreQueryProvider.java | 12 +++ .../jdbc/JDBCVectorStoreRecordCollection.java | 14 +++- ...DBCVectorStoreRecordCollectionFactory.java | 1 + ...DBCVectorStoreRecordCollectionOptions.java | 8 ++ .../jdbc/JDBCVectorStoreRecordMapper.java | 10 +++ .../connectors/data/jdbc/SQLVectorStore.java | 3 + .../jdbc/SQLVectorStoreQueryProvider.java | 1 + .../jdbc/SQLVectorStoreRecordCollection.java | 6 ++ ...torStoreRecordCollectionSearchMapping.java | 16 ++++ .../jdbc/filter/SQLEqualToFilterClause.java | 3 + .../mysql/MySQLVectorStoreQueryProvider.java | 8 ++ .../PostgreSQLVectorDistanceFunction.java | 34 +++++++- .../postgres/PostgreSQLVectorIndexKind.java | 25 +++++- .../PostgreSQLVectorStoreQueryProvider.java | 8 ++ .../PostgreSQLVectorStoreRecordMapper.java | 10 +++ ...disHashSetVectorStoreRecordCollection.java | 5 ++ ...SetVectorStoreRecordCollectionOptions.java | 4 + .../RedisHashSetVectorStoreRecordMapper.java | 4 + .../RedisJsonVectorStoreRecordCollection.java | 6 ++ ...sonVectorStoreRecordCollectionOptions.java | 5 ++ .../RedisJsonVectorStoreRecordMapper.java | 4 + .../data/redis/RedisVectorStore.java | 12 ++- ...disVectorStoreCollectionCreateMapping.java | 10 +++ ...disVectorStoreCollectionSearchMapping.java | 27 ++++++ .../data/redis/RedisVectorStoreOptions.java | 5 +- ...disVectorStoreRecordCollectionFactory.java | 1 + .../filter/RedisEqualToFilterClause.java | 9 ++ .../SQLiteVectorStoreQueryProvider.java | 6 ++ .../data/VolatileVectorStore.java | 10 ++- ...ileVectorStoreCollectionSearchMapping.java | 13 +++ .../VolatileVectorStoreRecordCollection.java | 11 +++ ...ileVectorStoreRecordCollectionOptions.java | 6 ++ .../filter/AnyTagEqualToFilterClause.java | 8 ++ .../data/filter/EqualToFilterClause.java | 9 ++ .../data/filter/FilterClause.java | 3 + .../data/vectorsearch/VectorOperations.java | 3 + .../data/vectorsearch/VectorSearchFilter.java | 19 +++++ .../data/vectorsearch/VectorSearchResult.java | 4 + .../vectorsearch/VectorizableTextSearch.java | 5 ++ .../data/vectorsearch/VectorizedSearch.java | 5 ++ .../data/vectorstorage/VectorStore.java | 2 + .../VectorStoreRecordCollection.java | 6 ++ .../VectorStoreRecordCollectionOptions.java | 6 ++ .../VectorStoreRecordMapper.java | 1 + .../VectorStoreRecordDataAttribute.java | 5 +- .../VectorStoreRecordKeyAttribute.java | 1 + .../VectorStoreRecordVectorAttribute.java | 4 + .../definition/DistanceFunction.java | 7 ++ .../vectorstorage/definition/IndexKind.java | 8 ++ .../VectorStoreRecordDataField.java | 11 +++ .../VectorStoreRecordDefinition.java | 14 ++++ .../definition/VectorStoreRecordField.java | 7 ++ .../definition/VectorStoreRecordKeyField.java | 8 ++ .../VectorStoreRecordVectorField.java | 7 ++ .../options/GetRecordOptions.java | 16 ++++ .../options/VectorSearchOptions.java | 45 ++++++++++ 128 files changed, 1431 insertions(+), 64 deletions(-) diff --git a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/GeminiService.java b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/GeminiService.java index 7155074f..d4012be1 100644 --- a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/GeminiService.java +++ b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/GeminiService.java @@ -6,10 +6,19 @@ import javax.annotation.Nullable; + +/** + * Makes a Gemini service available to the Semantic Kernel. + */ public class GeminiService implements AIService { private final VertexAI client; private final String modelId; + /** + * Creates a new Gemini service. + * @param client The VertexAI client + * @param modelId The Gemini model ID + */ protected GeminiService(VertexAI client, String modelId) { this.client = client; this.modelId = modelId; @@ -27,6 +36,10 @@ public String getServiceId() { return null; } + /** + * Gets the VertexAI client. + * @return The VertexAI client + */ protected VertexAI getClient() { return client; } diff --git a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/GeminiServiceBuilder.java b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/GeminiServiceBuilder.java index c5d3f658..3312299c 100644 --- a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/GeminiServiceBuilder.java +++ b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/GeminiServiceBuilder.java @@ -7,6 +7,8 @@ /** * Builder for a Gemini service. + * @param The type of the service + * @param The type of the builder */ public abstract class GeminiServiceBuilder> implements SemanticKernelBuilder { diff --git a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java index 8ba16bc4..e45bfa23 100644 --- a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java +++ b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java @@ -50,14 +50,23 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; + +/** + * A chat completion service that uses the Gemini model to generate chat completions. + */ public class GeminiChatCompletion extends GeminiService implements ChatCompletionService { private static final Logger LOGGER = LoggerFactory.getLogger(GeminiChatCompletion.class); + /** + * Constructor for {@link GeminiChatCompletion}. + * @param client The VertexAI client + * @param modelId The model ID + */ public GeminiChatCompletion(VertexAI client, String modelId) { super(client, modelId); } - + /** * Create a new instance of {@link GeminiChatCompletion.Builder}. * @@ -391,6 +400,13 @@ private Tool getTool(@Nullable Kernel kernel, @Nullable ToolCallBehavior toolCal return toolBuilder.build(); } + /** + * Invoke the Gemini function call. + * @param kernel The semantic kernel + * @param invocationContext Additional context for the invocation + * @param geminiFunction The Gemini function call + * @return The result of the function call + */ public Mono performFunctionCall(@Nullable Kernel kernel, @Nullable InvocationContext invocationContext, GeminiFunctionCall geminiFunction) { if (kernel == null) { @@ -433,6 +449,9 @@ public Mono performFunctionCall(@Nullable Kernel kernel, .map(result -> new GeminiFunctionCall(geminiFunction.getFunctionCall(), result)); } + /** + * Builder for {@link GeminiChatCompletion}. + */ public static class Builder extends GeminiServiceBuilder { @Override diff --git a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiFunctionCall.java b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiFunctionCall.java index fc64c830..0a6e87d4 100644 --- a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiFunctionCall.java +++ b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiFunctionCall.java @@ -9,6 +9,9 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +/** + * Represents a function call in Gemini. + */ public class GeminiFunctionCall { @Nonnull private final FunctionCall functionCall; @@ -17,6 +20,11 @@ public class GeminiFunctionCall { private final String pluginName; private final String functionName; + /** + * Creates a new Gemini function call. + * @param functionCall The function call + * @param functionResult The result of the function invocation + */ @SuppressFBWarnings("EI_EXPOSE_REP2") public GeminiFunctionCall( @Nonnull FunctionCall functionCall, @@ -29,19 +37,35 @@ public GeminiFunctionCall( this.functionName = name[1]; } + /** + * Gets the plugin name. + * @return The plugin name + */ public String getPluginName() { return pluginName; } + /** + * Gets the function name. + * @return The function name + */ public String getFunctionName() { return functionName; } + /** + * Gets the function call. + * @return The function call + */ @SuppressFBWarnings("EI_EXPOSE_REP") public FunctionCall getFunctionCall() { return functionCall; } + /** + * Gets the function result. + * @return The function result + */ @Nullable public FunctionResult getFunctionResult() { return functionResult; diff --git a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiRole.java b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiRole.java index 74590723..03983553 100644 --- a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiRole.java +++ b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiRole.java @@ -1,6 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.aiservices.google.chatcompletion; +/** + * Represents the role of a message in a Gemini conversation. + */ public enum GeminiRole { /** * A user message is a message generated by the user. diff --git a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiStreamingChatMessageContent.java b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiStreamingChatMessageContent.java index 45aed993..e6f0d551 100644 --- a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiStreamingChatMessageContent.java +++ b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiStreamingChatMessageContent.java @@ -27,6 +27,7 @@ public class GeminiStreamingChatMessageContent extends GeminiChatMessageConte * @param innerContent The inner content. * @param encoding The encoding. * @param metadata The metadata. + * @param id The id of the message. * @param geminiFunctionCalls The function calls. */ public GeminiStreamingChatMessageContent(AuthorRole authorRole, String content, diff --git a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiXMLPromptParser.java b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiXMLPromptParser.java index afde9a33..e43032dc 100644 --- a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiXMLPromptParser.java +++ b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiXMLPromptParser.java @@ -17,15 +17,26 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Parses an XML prompt for a Gemini chat. + */ public class GeminiXMLPromptParser { private static final Logger LOGGER = LoggerFactory.getLogger(GeminiXMLPromptParser.class); + /** + * Represents a parsed prompt for Gemini chat. + */ public static class GeminiParsedPrompt { private final ChatHistory chatHistory; private final List functions; + /** + * Creates a new parsed prompt. + * @param parsedChatHistory The chat history + * @param parsedFunctions The functions declarations. + */ protected GeminiParsedPrompt( ChatHistory parsedChatHistory, @Nullable List parsedFunctions) { @@ -36,10 +47,18 @@ protected GeminiParsedPrompt( this.functions = parsedFunctions; } + /** + * Gets the chat history. + * @return A copy of the chat history. + */ public ChatHistory getChatHistory() { return new ChatHistory(chatHistory.getMessages()); } + /** + * Gets the functions declarations. + * @return A copy of the functions declarations. + */ public List getFunctions() { return Collections.unmodifiableList(functions); } @@ -131,6 +150,11 @@ public ChatPromptParseVisitor reset() { } } + /** + * Create a GeminiParsedPrompt by parsing a raw prompt. + * @param rawPrompt the raw prompt to parse. + * @return The parsed prompt. + */ public static GeminiParsedPrompt parse(String rawPrompt) { ChatPromptParseVisitor visitor = ChatXMLPromptParser.parse( rawPrompt, diff --git a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/textcompletion/GeminiTextGenerationService.java b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/textcompletion/GeminiTextGenerationService.java index beec7d2c..5e555116 100644 --- a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/textcompletion/GeminiTextGenerationService.java +++ b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/textcompletion/GeminiTextGenerationService.java @@ -28,14 +28,27 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +/** + * A Gemini service for text generation. + * @see TextGenerationService + */ public class GeminiTextGenerationService extends GeminiService implements TextGenerationService { private static final Logger LOGGER = LoggerFactory.getLogger(GeminiTextGenerationService.class); + /** + * Creates a new Gemini text generation service. + * @param client The VertexAI client + * @param modelId The Gemini model ID + */ public GeminiTextGenerationService(VertexAI client, String modelId) { super(client, modelId); } + /** + * Creates a new builder for a Gemini text generation service. + * @return The builder + */ public static Builder builder() { return new Builder(); } @@ -121,6 +134,9 @@ private GenerativeModel getGenerativeModel( return modelBuilder.build(); } + /** + * Builder for a Gemini text generation service. + */ public static class Builder extends GeminiServiceBuilder { diff --git a/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/HuggingFaceClient.java b/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/HuggingFaceClient.java index 93fe64fd..28952880 100644 --- a/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/HuggingFaceClient.java +++ b/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/HuggingFaceClient.java @@ -19,12 +19,21 @@ import reactor.core.publisher.Mono; import javax.annotation.Nullable; +/** + * A client for the Hugging Face API. + */ public class HuggingFaceClient { private final KeyCredential key; private final String endpoint; private final HttpClient httpClient; + /** + * Creates a new Hugging Face client. + * @param key The key credential for endpoint authentication. + * @param endpoint The endpoint for the Hugging Face API. + * @param httpClient The HTTP client to use for requests. + */ public HuggingFaceClient( KeyCredential key, String endpoint, @@ -74,6 +83,12 @@ public GeneratedTextItemList( } + /** + * Gets the text contents from the Hugging Face API. + * @param modelId The model ID. + * @param textGenerationRequest The text generation request. + * @return The generated text items. + */ public Mono> getTextContentsAsync( String modelId, TextGenerationRequest textGenerationRequest) { @@ -131,11 +146,18 @@ private Mono performRequest(String modelId, return responseBody; } + /** + * Creates a new builder for a Hugging Face client. + * @return The builder + */ public static Builder builder() { return new Builder(); } - public static class Builder { + /** + * Builder for a Hugging Face client. + */ + public static class Builder { @Nullable private KeyCredential key = null; @@ -144,6 +166,10 @@ public static class Builder { @Nullable private HttpClient httpClient = null; + /** + * Builds the Hugging Face client. + * @return The client + */ public HuggingFaceClient build() { if (httpClient == null) { httpClient = HttpClient.createDefault(); @@ -160,16 +186,31 @@ public HuggingFaceClient build() { httpClient); } + /** + * Sets the key credential for the client. + * @param key The key credential + * @return The builder + */ public Builder credential(KeyCredential key) { this.key = key; return this; } + /** + * Sets the endpoint for the client. + * @param endpoint The endpoint + * @return The builder + */ public Builder endpoint(String endpoint) { this.endpoint = endpoint; return this; } + /** + * Sets the HTTP client for the client. + * @param httpClient The HTTP client + * @return The builder + */ public Builder httpClient(HttpClient httpClient) { this.httpClient = httpClient; return this; diff --git a/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/models/GeneratedTextItem.java b/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/models/GeneratedTextItem.java index 6f4125c6..12ed5be6 100644 --- a/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/models/GeneratedTextItem.java +++ b/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/models/GeneratedTextItem.java @@ -8,6 +8,9 @@ import java.util.List; import javax.annotation.Nullable; +/** + * Represents a generated text item deserialized from a JSON response. + */ public class GeneratedTextItem { @Nullable @@ -18,6 +21,11 @@ public class GeneratedTextItem { @JsonProperty("details") private final TextGenerationDetails details; + /** + * Constructor used by Jackson to deserialize a generated text item. + * @param generatedText The generated text. + * @param details The details of the generation. + */ @JsonCreator public GeneratedTextItem( @JsonProperty("generated_text") @Nullable String generatedText, @@ -26,16 +34,27 @@ public GeneratedTextItem( this.details = details; } + /** + * Gets the generated text. + * @return The generated text. + */ @Nullable public String getGeneratedText() { return generatedText; } + /** + * Gets the details of the generation. + * @return The details of the generation. + */ @Nullable public TextGenerationDetails getDetails() { return details; } + /** + * Represents the details of a text generation deserialized from a JSON response. + */ public static class TextGenerationDetails { @Nullable @@ -57,6 +76,14 @@ public static class TextGenerationDetails { @JsonProperty("tokens") private final List tokens; + /** + * Constructor used by Jackson to deserialize text generation details. + * @param finishReason The reason the generation finished. + * @param generatedTokens The number of tokens generated. + * @param seed The seed used for generation. + * @param prefill The prefill tokens. + * @param tokens The generated tokens. + */ @JsonCreator public TextGenerationDetails( @JsonProperty("finish_reason") @Nullable String finishReason, @@ -79,31 +106,54 @@ public TextGenerationDetails( } } + /** + * Gets the reason the generation finished. + * @return The reason the generation finished. + */ @Nullable public String getFinishReason() { return finishReason; } + /** + * Gets the number of tokens generated. + * @return The number of tokens generated. + */ public int getGeneratedTokens() { return generatedTokens; } + /** + * Gets the seed used for generation. + * @return The seed used for generation. + */ @Nullable public Long getSeed() { return seed; } + /** + * Gets the prefill tokens. + * @return The prefill tokens. + */ @Nullable public List getPrefill() { return Collections.unmodifiableList(prefill); } + /** + * Gets the generated tokens. + * @return The generated tokens. + */ @Nullable public List getTokens() { return Collections.unmodifiableList(tokens); } } + /** + * Represents a prefill token deserialized from a JSON response. + */ public static class TextGenerationPrefillToken { @JsonProperty("id") @@ -116,6 +166,12 @@ public static class TextGenerationPrefillToken { @JsonProperty("logprob") private final double logProb; + /** + * Constructor used by Jackson to deserialize a prefill token. + * @param id The token ID. + * @param text The token text. + * @param logProb The log probability of the token. + */ @JsonCreator public TextGenerationPrefillToken( @JsonProperty("id") int id, @@ -126,25 +182,47 @@ public TextGenerationPrefillToken( this.logProb = logProb; } + /** + * Gets the token ID. + * @return The token ID. + */ public int getId() { return id; } + /** + * Gets the token text. + * @return The token text. + */ @Nullable public String getText() { return text; } + /** + * Gets the log probability of the token. + * @return The log probability of the token. + */ public double getLogProb() { return logProb; } } + /** + * Represents a generated token deserialized from a JSON response. + */ public static class TextGenerationToken extends TextGenerationPrefillToken { @JsonProperty("special") private final boolean special; + /** + * Constructor used by Jackson to deserialize a generated token. + * @param special Whether the token is special. + * @param id The token ID. + * @param text The token text. + * @param logProb The log probability of the token. + */ @JsonCreator public TextGenerationToken( @JsonProperty("special") boolean special, @@ -155,6 +233,10 @@ public TextGenerationToken( this.special = special; } + /** + * Gets whether the token is special. + * @return Whether the token is special. + */ public boolean isSpecial() { return special; } diff --git a/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/models/TextGenerationRequest.java b/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/models/TextGenerationRequest.java index fd0b6077..0e52b568 100644 --- a/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/models/TextGenerationRequest.java +++ b/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/models/TextGenerationRequest.java @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.aiservices.huggingface.models; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonInclude.Include; import com.fasterxml.jackson.annotation.JsonProperty; @@ -11,6 +10,9 @@ import java.util.Arrays; import java.util.List; +/** + * Represents a request to generate text using the Hugging Face API. + */ @JsonInclude(Include.NON_NULL) public class TextGenerationRequest { @@ -42,6 +44,13 @@ public class TextGenerationRequest { @JsonProperty("options") private final HuggingFaceTextOptions options; + /** + * Create a new instance of TextGenerationRequest. + * @param inputs The input string to generate text for. + * @param stream Enable streaming. + * @param parameters Parameters used by the model for generation. + * @param options Options used by the model for generation. + */ public TextGenerationRequest( @Nullable String inputs, boolean stream, @@ -53,6 +62,12 @@ public TextGenerationRequest( this.options = options; } + /** + * Create a new instance of TextGenerationRequest. + * @param prompt The prompt to generate text for. + * @param executionSettings The settings for executing the prompt. + * @return A new instance of TextGenerationRequest. + */ public static TextGenerationRequest fromPromptAndExecutionSettings(String prompt, HuggingFacePromptExecutionSettings executionSettings) { return new TextGenerationRequest( @@ -72,6 +87,9 @@ public static TextGenerationRequest fromPromptAndExecutionSettings(String prompt new HuggingFaceTextOptions()); } + /** + * Parameters used by the model for generation. + */ public static class HuggingFaceTextParameters { /// @@ -156,6 +174,19 @@ public static class HuggingFaceTextParameters { @JsonProperty("details") private final Boolean details; + /** + * Creator method for jackson deserialization. + * @param topK The number of top tokens considered within the sample operation to create new text. + * @param topP The tokens that are within the sample operation of text generation. + * @param temperature The temperature of the sampling operation. + * @param repetitionPenalty The repetition penalty. + * @param maxNewTokens The amount of new tokens to be generated. + * @param maxTime The amount of time in seconds that the query should take maximum. + * @param returnFullText A value indicating whether the return results will contain the original query. + * @param numReturnSequences The number of propositions to be returned. + * @param doSample A value indicating whether to use sampling. + * @param details A value indicating whether to include the details of the generation. + */ public HuggingFaceTextParameters( @JsonProperty("top_k") @Nullable Integer topK, @JsonProperty("top_p") @Nullable Double topP, @@ -179,56 +210,99 @@ public HuggingFaceTextParameters( this.details = details; } + /** + * Gets the number of top tokens considered within the sample operation to create new text. + * @return The number of top tokens considered within the sample operation to create new text. + */ @Nullable public Integer getTopK() { return topK; } + /** + * Gets the tokens that are within the sample operation of text generation. + * @return The tokens that are within the sample operation of text generation. + */ @Nullable public Double getTopP() { return topP; } + /** + * Gets the temperature of the sampling operation. + * @return The temperature of the sampling operation. + */ @Nullable public Double getTemperature() { return temperature; } + /** + * Gets the repetition penalty. + * @return The repetition penalty. + */ @Nullable public Double getRepetitionPenalty() { return repetitionPenalty; } + /** + * Gets the amount of new tokens to be generated. + * @return The amount of new tokens to be generated. + */ @Nullable public Integer getMaxNewTokens() { return maxNewTokens; } + /** + * Gets the amount of time in seconds that the query should take maximum. + * @return The amount of time in seconds that the query should take maximum. + */ @Nullable public Double getMaxTime() { return maxTime; } + /** + * Gets a value indicating whether the return results will contain the original query. + * @return A value indicating whether the return results will contain the original query. + */ public boolean isReturnFullText() { return returnFullText; } + /** + * Gets the number of propositions to be returned. + * @return The number of propositions to be returned. + */ @Nullable public Integer getNumReturnSequences() { return numReturnSequences; } + /** + * Gets a value indicating whether to use sampling. + * @return A value indicating whether to use sampling. + */ @Nullable public Boolean getDoSample() { return doSample; } + /** + * Gets a value indicating whether to include the details of the generation. + * @return A value indicating whether to include the details of the generation. + */ @Nullable public Boolean getDetails() { return details; } } + /** + * Options used by the model for generation. + */ @SuppressFBWarnings("SS_SHOULD_BE_STATIC") public static class HuggingFaceTextOptions { @@ -249,10 +323,18 @@ public static class HuggingFaceTextOptions { @JsonProperty("wait_for_model") private final boolean waitForModel = false; + /** + * Gets a value indicating whether to use the cache layer on the inference API. + * @return A value indicating whether to use the cache layer on the inference API. + */ public boolean isUseCache() { return useCache; } + /** + * Gets a value indicating whether to wait for the model if it is not ready. + * @return A value indicating whether to wait for the model if it is not ready. + */ public boolean isWaitForModel() { return waitForModel; } diff --git a/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFacePromptExecutionSettings.java b/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFacePromptExecutionSettings.java index be6cd2ca..adcabe7c 100644 --- a/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFacePromptExecutionSettings.java +++ b/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFacePromptExecutionSettings.java @@ -8,6 +8,9 @@ import java.util.Objects; import javax.annotation.Nullable; +/** + * Represents the settings for executing a prompt with the Hugging Face API. + */ public class HuggingFacePromptExecutionSettings extends PromptExecutionSettings { @Nullable @@ -25,6 +28,11 @@ public class HuggingFacePromptExecutionSettings extends PromptExecutionSettings @Nullable private final Long seed; + /** + * Create a new instance of HuggingFacePromptExecutionSettings. + * + * @param copy The PromptExecutionSettings to copy. + */ public HuggingFacePromptExecutionSettings(PromptExecutionSettings copy) { super( copy.getServiceId(), @@ -65,6 +73,13 @@ public HuggingFacePromptExecutionSettings(PromptExecutionSettings copy) { * @param stopSequences The stop sequences to use for prompt execution. * @param tokenSelectionBiases The token selection biases to use for prompt execution. * @param responseFormat The response format to use for prompt execution + * @param topK The topK setting for prompt execution. + * @param repetitionPenalty The repetition penalty setting for prompt execution. + * @param maxTime The max time setting for prompt execution. + * @param details The details setting for prompt execution. + * @param logProbs The logprobs setting for prompt execution. + * @param topLogProbs The top log probs setting for prompt execution. + * @param seed The seed setting for prompt execution */ public HuggingFacePromptExecutionSettings( String serviceId, @@ -100,6 +115,14 @@ public HuggingFacePromptExecutionSettings( this.seed = seed; } + /** + * Create a new instance of PromptExecutionSettings from a PromptExecutionSettings. + * This method handles the whether the PromptExecutionSettings is already a + * HuggingFacePromptExecutionSettings or a new instance needs to be created + * from the provided PromptExecutionSettings. + * @param promptExecutionSettings The PromptExecutionSettings to copy. + * @return The PromptExecutionSettings mapped to a HuggingFacePromptExecutionSettings. + */ public static HuggingFacePromptExecutionSettings fromExecutionSettings( PromptExecutionSettings promptExecutionSettings) { if (promptExecutionSettings instanceof HuggingFacePromptExecutionSettings) { @@ -131,36 +154,64 @@ public static HuggingFacePromptExecutionSettings fromExecutionSettings( null); } + /** + * Gets the topK setting for prompt execution. + * @return The topK setting for prompt execution + */ @Nullable public Integer getTopK() { return topK; } + /** + * Gets the repetition penalty setting for prompt execution. + * @return The repetition penalty setting for prompt execution + */ @Nullable public Double getRepetitionPenalty() { return repetitionPenalty; } + /** + * Gets the max time setting for prompt execution. + * @return The max time setting for prompt execution + */ @Nullable public Double getMaxTime() { return maxTime; } + /** + * Gets the details setting for prompt execution. + * @return The details setting for prompt execution + */ @Nullable public Boolean getDetails() { return details; } + /** + * Gets the logprobs setting for prompt execution. + * @return The logprobs setting for prompt execution + */ @Nullable public Boolean getLogprobs() { return logProbs; } + /** + * Gets the top log probs setting for prompt execution. + * @return The top log probs setting for prompt execution + */ @Nullable public Integer getTopLogProbs() { return topLogProbs; } + /** + * Gets the seed setting for prompt execution. + * @return The seed setting for prompt execution + */ @Nullable public Long getSeed() { return seed; diff --git a/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFaceTextGenerationService.java b/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFaceTextGenerationService.java index 0f5260d3..a08b4232 100644 --- a/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFaceTextGenerationService.java +++ b/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFaceTextGenerationService.java @@ -19,12 +19,21 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +/** + * A service that generates text using the Hugging Face API. + */ public class HuggingFaceTextGenerationService implements TextGenerationService { private final String modelId; private final String serviceId; private final HuggingFaceClient client; + /** + * Create a new instance of HuggingFaceTextGenerationService. + * @param modelId The model ID. + * @param serviceId The service ID. + * @param client The Hugging Face client. + */ public HuggingFaceTextGenerationService( String modelId, String serviceId, @@ -34,6 +43,13 @@ public HuggingFaceTextGenerationService( this.client = client; } + /** + * Get the response to a prompt. + * @param prompt The prompt. + * @param huggingFacePromptExecutionSettings The settings for executing the prompt. + * @param kernel The semantic kernel. + * @return The response to the prompt. + */ public Mono> getTextContentsAsync( String prompt, @Nullable HuggingFacePromptExecutionSettings huggingFacePromptExecutionSettings, @@ -117,10 +133,17 @@ public String getServiceId() { return serviceId; } + /** + * Create a new builder for HuggingFaceTextGenerationService. + * @return The builder. + */ public static Builder builder() { return new Builder(); } + /** + * A builder for HuggingFaceTextGenerationService. + */ public static class Builder { @Nullable @@ -152,11 +175,20 @@ public Builder withServiceId(String serviceId) { return this; } + /** + * Sets the HuggingFaceClient for the service + * @param client The HuggingFaceClient + * @return The builder + */ public Builder withHuggingFaceClient(HuggingFaceClient client) { this.client = client; return this; } + /** + * Builds the HuggingFaceTextGenerationService + * @return The HuggingFaceTextGenerationService + */ public HuggingFaceTextGenerationService build() { if (this.modelId == null) { diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/OpenAiService.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/OpenAiService.java index 4a7a196b..0edee476 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/OpenAiService.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/OpenAiService.java @@ -6,6 +6,7 @@ /** * Provides OpenAI service. + * @param the client type */ public abstract class OpenAiService implements AIService { @@ -38,10 +39,18 @@ public String getServiceId() { return serviceId; } + /** + * Gets the client. + * @return the client + */ protected Client getClient() { return client; } + /** + * Gets the deployment name. + * @return the deployment name + */ public String getDeploymentName() { return deploymentName; } diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiAudioToTextService.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiAudioToTextService.java index 631f2cac..7a126f6d 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiAudioToTextService.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiAudioToTextService.java @@ -28,6 +28,7 @@ public class OpenAiAudioToTextService extends OpenAiService * * @param client OpenAI client. * @param modelId The model ID. + * @param deploymentName The deployment name. */ public OpenAiAudioToTextService( OpenAIAsyncClient client, diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiTextToAudioService.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiTextToAudioService.java index c698fab3..dc748af7 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiTextToAudioService.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiTextToAudioService.java @@ -27,6 +27,7 @@ public class OpenAiTextToAudioService extends OpenAiService * * @param client OpenAI client. * @param modelId The model ID. + * @param deploymentName The deployment name. */ public OpenAiTextToAudioService( OpenAIAsyncClient client, @@ -79,7 +80,6 @@ public static Builder builder() { /** * Represents a builder for OpenAi text to audio service. */ - public static class Builder extends TextToAudioService.Builder { /** diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIStreamingChatMessageContent.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIStreamingChatMessageContent.java index 21162c8f..c919f5c6 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIStreamingChatMessageContent.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIStreamingChatMessageContent.java @@ -8,11 +8,28 @@ import java.util.List; import javax.annotation.Nullable; +/** + * Represents the content of a chat message. + * + * @param The type of the inner content. + */ public class OpenAIStreamingChatMessageContent extends OpenAIChatMessageContent implements StreamingChatContent { private final String id; + /** + * Creates a new instance of the {@link OpenAIChatMessageContent} class. + * + * @param id The id of the message. + * @param authorRole The author role that generated the content. + * @param content The content. + * @param modelId The model id. + * @param innerContent The inner content. + * @param encoding The encoding. + * @param metadata The metadata. + * @param toolCall The tool call. + */ public OpenAIStreamingChatMessageContent( String id, AuthorRole authorRole, diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/responseformat/ChatCompletionsJsonSchemaResponseFormat.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/responseformat/ChatCompletionsJsonSchemaResponseFormat.java index 96fa2526..f5a3b1c4 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/responseformat/ChatCompletionsJsonSchemaResponseFormat.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/responseformat/ChatCompletionsJsonSchemaResponseFormat.java @@ -6,19 +6,29 @@ import com.microsoft.semantickernel.orchestration.responseformat.JsonResponseSchema; import java.io.IOException; +/** + * Represents a response format for chat completions that uses a JSON schema. + */ public class ChatCompletionsJsonSchemaResponseFormat extends ChatCompletionsResponseFormat { private final JsonResponseSchema schema; private String type = "json_schema"; + /** + * Creates a new instance of the {@link ChatCompletionsJsonSchemaResponseFormat} class. + * + * @param schema The JSON schema. + */ public ChatCompletionsJsonSchemaResponseFormat(JsonResponseSchema schema) { this.schema = schema; } + @Override public String getType() { return this.type; } + @Override public JsonWriter toJson(JsonWriter jsonWriter) throws IOException { jsonWriter.writeStartObject(); jsonWriter.writeStringField("type", this.type); diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/responseformat/JacksonResponseFormatGenerator.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/responseformat/JacksonResponseFormatGenerator.java index 93ea5b5f..97060cab 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/responseformat/JacksonResponseFormatGenerator.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/responseformat/JacksonResponseFormatGenerator.java @@ -12,10 +12,16 @@ import com.github.victools.jsonschema.module.jackson.JacksonModule; import com.microsoft.semantickernel.orchestration.responseformat.ResponseSchemaGenerator; +/** + * Represents a response format generator that uses Jackson. + */ public class JacksonResponseFormatGenerator implements ResponseSchemaGenerator { private final SchemaGenerator generator; + /** + * Creates a new instance of the {@link JacksonResponseFormatGenerator} class. + */ public JacksonResponseFormatGenerator() { JacksonModule module = new JacksonModule(); SchemaGeneratorConfigBuilder builder = new SchemaGeneratorConfigBuilder( @@ -31,6 +37,11 @@ public JacksonResponseFormatGenerator() { generator = new SchemaGenerator(builder.build()); } + /** + * Creates a new instance of the {@link JacksonResponseFormatGenerator} class. + * + * @param generator The schema generator. + */ public JacksonResponseFormatGenerator(SchemaGenerator generator) { this.generator = generator; } diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textembedding/OpenAITextEmbeddingGenerationService.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textembedding/OpenAITextEmbeddingGenerationService.java index f8ccaeb0..89b52831 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textembedding/OpenAITextEmbeddingGenerationService.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textembedding/OpenAITextEmbeddingGenerationService.java @@ -29,7 +29,15 @@ public class OpenAITextEmbeddingGenerationService extends OpenAiService3.2.5 3.9.3 2.16.2 + 3.7.0 5.10.0 0.9.1 6.55.0 diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/Kernel.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/Kernel.java index 4ab1553f..57e46165 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/Kernel.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/Kernel.java @@ -163,6 +163,15 @@ public FunctionInvocation invokePromptAsync(@Nonnull String prompt) { return invokeAsync(KernelFunction.createFromPrompt(prompt).build()); } + /** + * Invokes a Prompt. + * + * @param The return type of the prompt. + * @param prompt The prompt to invoke. + * @param arguments The arguments to pass to the prompt. + * @return The result of the prompt invocation. + * @see KernelFunction#invokeAsync(Kernel) + */ public FunctionInvocation invokePromptAsync(@Nonnull String prompt, @Nonnull KernelFunctionArguments arguments) { KernelFunction function = KernelFunction.createFromPrompt(prompt).build(); @@ -171,6 +180,17 @@ public FunctionInvocation invokePromptAsync(@Nonnull String prompt, .withArguments(arguments); } + /** + * Invokes a Prompt. + * + * @param The return type of the prompt. + * @param prompt The prompt to invoke. + * @param arguments The arguments to pass to the prompt. + * @param invocationContext Additional context to used when invoking the prompt. + * @return The result of the prompt invocation. + * @see KernelFunction#invokeAsync(Kernel) + */ + public FunctionInvocation invokePromptAsync(@Nonnull String prompt, @Nonnull KernelFunctionArguments arguments, @Nonnull InvocationContext invocationContext) { diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypeConverter.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypeConverter.java index ac1b9dd2..22d53ec0 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypeConverter.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypeConverter.java @@ -25,8 +25,19 @@ public class ContextVariableTypeConverter { private static final Logger LOGGER = LoggerFactory.getLogger( ContextVariableTypeConverter.class); + /** + * A function to convert ContextVariable to a prompt string. + * @param the type of ContextVariable to convert + */ public interface ToPromptStringFunction { + /** + * Convert the type to a prompt string. + * + * @param types the context variable types + * @param t the type to convert + * @return the prompt string + */ String toPromptString(ContextVariableTypes types, T t); } @@ -116,6 +127,7 @@ public ContextVariableTypeConverter( * Use this converter to convert the object to the type of the context variable. * * @param the type to convert to + * @param types the context variable types * @param t the object to convert * @param clazz the class of the type to convert to * @return the converted object @@ -129,10 +141,11 @@ public U toObject(ContextVariableTypes types, @Nullable Object t, Class c /** * Use this converter to convert the object to the type of the context variable. * - * @param the type to convert to + * @param types the context variable types * @param t the object to convert * @param clazz the class of the type to convert to * @param logWarnings whether to log warnings + * @param the type to convert to * @return the converted object */ @Nullable @@ -314,6 +327,7 @@ public static Builder builder(Class clazz) { /** * A builder for a context variable type converter. + * @param the type of the context variable */ public static class Builder { diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypes.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypes.java index 76ed87fa..dd12bb70 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypes.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypes.java @@ -94,6 +94,14 @@ public ContextVariableTypes() { variableTypes = new HashMap<>(); } + /** + * Get the globally available collectors, which is the + * default collection of context variable types and those that + * have been added with + * {@link #addGlobalConverter(ContextVariableTypeConverter)}. + * + * @return The collection of globally available converters. + */ public static ContextVariableTypes getGlobalTypes() { return new ContextVariableTypes(DEFAULT_TYPES); } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/CollectionVariableContextVariableTypeConverter.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/CollectionVariableContextVariableTypeConverter.java index 5a5c14cf..d7bf72e0 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/CollectionVariableContextVariableTypeConverter.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/CollectionVariableContextVariableTypeConverter.java @@ -15,12 +15,15 @@ * A {@link ContextVariableTypeConverter} for {@code java.util.Collection} variables. Use * {@code ContextVariableTypes.getGlobalVariableTypeForClass(String.class)} to get an instance of * this class. - * * @see ContextVariableTypes#getGlobalVariableTypeForClass(Class) */ public class CollectionVariableContextVariableTypeConverter extends ContextVariableTypeConverter { + /** + * Creates a new instance of the {@link CollectionVariableContextVariableTypeConverter} class. + * @param delimiter The delimiter to use joining elements of the collection. + */ @SuppressFBWarnings("CT_CONSTRUCTOR_THROW") public CollectionVariableContextVariableTypeConverter(String delimiter) { super( @@ -35,12 +38,16 @@ public CollectionVariableContextVariableTypeConverter(String delimiter) { /** * Creates a new instance of the {@link CollectionVariableContextVariableTypeConverter} class. */ - @SuppressFBWarnings("CT_CONSTRUCTOR_THROW") public CollectionVariableContextVariableTypeConverter() { this(","); } + /** + * Gets a function that converts a collection to a string. + * @param delimiter The delimiter to use joining elements of the collection. + * @return A function that converts a collection to a string. + */ @SuppressWarnings("NullAway") public static ToPromptStringFunction getString(String delimiter) { return (types, collection) -> { diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/PrimitiveVariableContextVariableTypeConverter.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/PrimitiveVariableContextVariableTypeConverter.java index d41b3a33..310ceab0 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/PrimitiveVariableContextVariableTypeConverter.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/PrimitiveVariableContextVariableTypeConverter.java @@ -25,6 +25,7 @@ public class PrimitiveVariableContextVariableTypeConverter extends * @param clazz the class * @param fromPromptString the function to convert from a prompt string * @param fromObject the function to convert from an object to primitive + * @param toPromptString the function to convert to a prompt string */ public PrimitiveVariableContextVariableTypeConverter( Class clazz, diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/StringVariableContextVariableTypeConverter.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/StringVariableContextVariableTypeConverter.java index 4a431e60..3e303bff 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/StringVariableContextVariableTypeConverter.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/StringVariableContextVariableTypeConverter.java @@ -29,6 +29,14 @@ public StringVariableContextVariableTypeConverter() { s -> s); } + /** + * Converts the specified object to a string. + * Has special handling for {@link ContextVariable} objects and + * for objects that look like an object reference + * @param s the object to convert + * @return the string representation of the object, or {@code null} + * if the object cannot be converted to a string or is an object reference. + */ @Nullable public static String convertToString(@Nullable Object s) { String converted = convert(s, String.class); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/TextContentVariableContextVariableTypeConverter.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/TextContentVariableContextVariableTypeConverter.java index e43ed0f0..920ac4a1 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/TextContentVariableContextVariableTypeConverter.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/TextContentVariableContextVariableTypeConverter.java @@ -29,6 +29,12 @@ public TextContentVariableContextVariableTypeConverter() { }); } + /** + * Escapes the XML string value. + * @param value The value containing the content to escape. + * @return The escaped XML string value. + * @see ContextVariableTypeConverter#escapeXmlString(String) + */ @Nullable public static String escapeXmlStringValue(@Nullable TextContent value) { if (value == null) { diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java index 7b8364ea..98e310fd 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java @@ -37,32 +37,39 @@ public SKCheckedException(@Nullable String message, @Nullable Throwable cause) { super(message, cause); } - public SKCheckedException(Throwable e) { - super(e); + /** + * Initializes a new instance of the {@code SKCheckedException} class with its + * message set to {@code null} and the cause set to {@code e}. + * + * @param cause The exception that is the cause of the current exception. + */ + public SKCheckedException(Throwable cause) { + super(cause); } /** - * Forms a checked exception, if the exception is already an SK exception, it will be unwrapped + * Forms a checked exception, if the exception is already an SK exception, it + * will be unwrapped * and the cause extracted. * * @param message The message to be displayed - * @param e The exception to be thrown + * @param cause The exception that is the cause of the current exception. * @return A checked exception */ public static SKCheckedException build( String message, - @Nullable Exception e) { + @Nullable Exception cause) { - if (e == null) { + if (cause == null) { return new SKCheckedException(message); } - Throwable cause = e.getCause(); + Throwable wrappedCause = cause.getCause(); - if ((e instanceof SKCheckedException || e instanceof SKException) && cause != null) { - return new SKCheckedException(message, cause); + if ((cause instanceof SKCheckedException || cause instanceof SKException) && wrappedCause != null) { + return new SKCheckedException(message, wrappedCause); } else { - return new SKCheckedException(message, e); + return new SKCheckedException(message, cause); } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java index 99aef056..45ea84b8 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java @@ -37,8 +37,14 @@ public SKException(@Nullable String message, @Nullable Throwable cause) { super(message, cause); } - public SKException(Throwable e) { - super(e); + /** + * Initializes a new instance of the {@code SKException} class with its + * message set to {@code null} and the cause set to {@code e}. + * + * @param cause The exception that is the cause of the current exception. + */ + public SKException(Throwable cause) { + super(cause); } /** @@ -46,23 +52,23 @@ public SKException(Throwable e) { * unwrapped and the cause extracted. * * @param message The message to be displayed - * @param e The exception to be thrown + * @param cause The exception that is the cause of the current exception. * @return An unchecked exception */ public static SKException build( String message, - @Nullable Exception e) { + @Nullable Exception cause) { - if (e == null) { + if (cause == null) { return new SKException(message); } - Throwable cause = e.getCause(); + Throwable wrappedCause = cause.getCause(); - if ((e instanceof SKCheckedException || e instanceof SKException) && cause != null) { - return new SKException(message, cause); + if ((cause instanceof SKCheckedException || cause instanceof SKException) && wrappedCause != null) { + return new SKException(message, wrappedCause); } else { - return new SKException(message, e); + return new SKException(message, cause); } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PreToolCallEvent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PreToolCallEvent.java index 9ef0d16c..c728a917 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PreToolCallEvent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PreToolCallEvent.java @@ -38,12 +38,20 @@ public PreToolCallEvent( this.contextVariableTypes = contextVariableTypes; } + /** + * Gets the tool call arguments. + * @return The tool call arguments. + */ @SuppressFBWarnings("EI_EXPOSE_REP") @Nullable public KernelFunctionArguments getArguments() { return arguments; } - + + /** + * Get the tool call function. + * @return The tool call function. + */ @SuppressFBWarnings("EI_EXPOSE_REP2") public KernelFunction getFunction() { return function; diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java index 405c32a1..2ebc9c41 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java @@ -6,6 +6,9 @@ import java.util.PropertyResourceBundle; import java.util.ResourceBundle; +/** + * Provides access to the resources used by the Semantic Kernel. + */ public class SemanticKernelResources { private static final String RESOURCE_BUNDLE_CLASS = "com.microsoft.semantickernel.localization.ResourceBundle"; @@ -20,6 +23,13 @@ public class SemanticKernelResources { RESOURCE_BUNDLE = setResourceBundle(LOCALE); } + /** + * Load the localized resource bundle for the Semantic Kernel. + * If there is no resource bundle for the specified locale, the default + * resource bundle will be loaded. + * @param locale The locale to use. + * @return the resource bundle. + */ public static ResourceBundle setResourceBundle(Locale locale) { ResourceBundle resourceBundle; try { @@ -33,12 +43,24 @@ public static ResourceBundle setResourceBundle(Locale locale) { return resourceBundle; } + /** + * Set the locale for the Semantic Kernel. As a side effect, + * the localized resource bundle will be loaded. + * @param locale The locale to use. + * @return the locale. + */ public static Locale setLocale(Locale locale) { LOCALE = locale; setResourceBundle(locale); return locale; } + /** + * Get the string for the specified id from the resource bundle. + * @param id The id of the string. + * @param defaultValue The default value to return if the string is not found. + * @return the localized string, or the default value if the string is not found. + */ public static String localize(String id, String defaultValue) { if (RESOURCE_BUNDLE.containsKey(id)) { return RESOURCE_BUNDLE.getString(id); @@ -47,8 +69,13 @@ public static String localize(String id, String defaultValue) { } } - public static String getString(String s) { - return localize(s, s); + /** + * Get the string for the specified id from the resource bundle. + * @param id The id of the string. + * @return the localized string, or the id if the string is not found. + */ + public static String getString(String id) { + return localize(id, id); } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionResultMetadata.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionResultMetadata.java index f9e79c7f..9bb55d11 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionResultMetadata.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionResultMetadata.java @@ -12,6 +12,7 @@ * Metadata about the result of a function invocation. *

* This class is used to return metadata about the result of a function invocation. + * @param The result type of the function invocation. */ public class FunctionResultMetadata { @@ -50,6 +51,8 @@ public FunctionResultMetadata(CaseInsensitiveMap> metadata) { /** * Create a new instance of FunctionResultMetadata. + * @param id The id of the result of the function invocation. + * @return A new instance of FunctionResultMetadata. */ public static FunctionResultMetadata build(String id) { return build(id, null, null); @@ -61,6 +64,7 @@ public static FunctionResultMetadata build(String id) { * @param id The id of the result of the function invocation. * @param usage The usage of the result of the function invocation. * @param createdAt The time the result was created. + * @param The result type of the function invocation. * @return A new instance of FunctionResultMetadata. */ public static FunctionResultMetadata build( diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/InvocationReturnMode.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/InvocationReturnMode.java index 1f03f22d..48c279f2 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/InvocationReturnMode.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/InvocationReturnMode.java @@ -1,6 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.orchestration; +/** + * Represents the mode in which a function invocation should return its results. + */ public enum InvocationReturnMode { /** * Function invocations that build upon a history of previous invocations, such as Chat diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/ToolCallBehavior.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/ToolCallBehavior.java index affa13ac..6454a7fd 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/ToolCallBehavior.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/ToolCallBehavior.java @@ -70,6 +70,9 @@ public static ToolCallBehavior allowOnlyKernelFunctions(boolean autoInvoke, } private static final int DEFAULT_MAXIMUM_AUTO_INVOKE_ATTEMPTS = 5; + /** + * The separator between the plugin name and the function name. + */ public static final String FUNCTION_NAME_SEPARATOR = "-"; private final int maximumAutoInvokeAttempts; diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonObjectResponseFormat.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonObjectResponseFormat.java index 6d9a107a..dde82691 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonObjectResponseFormat.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonObjectResponseFormat.java @@ -3,8 +3,15 @@ import com.fasterxml.jackson.annotation.JsonCreator; +/** + * A response represented in a JSON format. + */ public class JsonObjectResponseFormat extends ResponseFormat { + /** + * Used by Jackson deserialization to create a new instance + * of the {@link JsonObjectResponseFormat} class. + */ @JsonCreator public JsonObjectResponseFormat() { super(Type.JSON_OBJECT); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonResponseSchema.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonResponseSchema.java index be92d417..c952cfea 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonResponseSchema.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonResponseSchema.java @@ -4,12 +4,23 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +/** + * The schema for a response in JSON format. + */ public class JsonResponseSchema extends ResponseSchema { private final String name; private final String schema; private final boolean strict; + /** + * Used by Jackson deserialization to create a new + * instance of the {@link JsonResponseSchema} class. + * + * @param name The name of the schema. + * @param schema The schema. + * @param strict Whether the schema is strict. + */ @JsonCreator public JsonResponseSchema( @JsonProperty("name") String name, @@ -20,14 +31,26 @@ public JsonResponseSchema( this.strict = strict; } + /** + * Gets the name of the schema. + * @return The name of the schema. + */ public String getName() { return name; } + /** + * Gets the schema. + * @return The schema. + */ public String getSchema() { return schema; } + /** + * Gets whether the schema is strict. + * @return Whether the schema is strict. + */ public boolean isStrict() { return strict; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonSchemaResponseFormat.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonSchemaResponseFormat.java index 68547d3a..ebc43b93 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonSchemaResponseFormat.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/JsonSchemaResponseFormat.java @@ -6,10 +6,18 @@ import com.microsoft.semantickernel.exceptions.SKException; import javax.annotation.Nullable; +/** + * A response represented in a JSON schema format. + */ public class JsonSchemaResponseFormat extends ResponseFormat { private final JsonResponseSchema jsonSchema; + /** + * Used by Jackson deserialization to create a new instance + * of the {@link JsonSchemaResponseFormat} class. + * @param jsonSchema The JSON schema. + */ @JsonCreator public JsonSchemaResponseFormat( @JsonProperty("json_schema") JsonResponseSchema jsonSchema) { @@ -17,15 +25,26 @@ public JsonSchemaResponseFormat( this.jsonSchema = jsonSchema; } + /** + * Gets the JSON schema. + * @return The JSON schema. + */ @JsonProperty("json_schema") public JsonResponseSchema getJsonSchema() { return jsonSchema; } + /** + * Creates a new instance of the {@link JsonSchemaResponseFormat} class. + * @return The new instance. + */ public static Builder builder() { return new Builder(); } + /** + * A builder for the {@link JsonSchemaResponseFormat} class. + */ public static class Builder { @Nullable @@ -36,38 +55,74 @@ public static class Builder { private String name = null; private boolean strict = true; + /** + * Sets the response format. + * @param clazz The class. + * @param responseSchemaGenerator The response schema generator. + * @return The builder. + */ public Builder setResponseFormat(Class clazz, ResponseSchemaGenerator responseSchemaGenerator) { name = clazz.getSimpleName(); return setJsonSchema(responseSchemaGenerator.generateSchema(clazz)); } + /** + * Sets the response format. Uses Jackson to generate the schema + * from the {@code clazz} + * @param clazz The class. + * @return The builder. + */ public Builder setResponseFormat(Class clazz) { name = clazz.getSimpleName(); setJsonSchema(ResponseSchemaGenerator.jacksonGenerator().generateSchema(clazz)); return this; } + /** + * Sets the JSON response schema. + * @param jsonResponseSchema The JSON response schema. + * @return The builder. + */ public Builder setJsonResponseSchema(JsonResponseSchema jsonResponseSchema) { this.jsonResponseSchema = jsonResponseSchema; return this; } + /** + * Sets the JSON schema. + * @param jsonSchema The JSON schema. + * @return The builder. + */ public Builder setJsonSchema(String jsonSchema) { this.jsonSchema = jsonSchema; return this; } + /** + * Sets the name of the JSON schema. + * @param name The schema name. + * @return The builder. + */ public Builder setName(String name) { this.name = name; return this; } + /** + * Sets whether the schema is strict. + * @param strict Whether the schema is strict. + * @return The builder. + */ public Builder setStrict(boolean strict) { this.strict = strict; return this; } + /** + * Builds the {@link JsonSchemaResponseFormat} instance. + * @return The {@link JsonSchemaResponseFormat} instance. + */ public JsonSchemaResponseFormat build() { if (jsonResponseSchema != null) { diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseFormat.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseFormat.java index beb68760..c4caf6c4 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseFormat.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseFormat.java @@ -7,6 +7,9 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo.As; import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; +/** + * Base class for response formats. + */ @JsonTypeInfo(use = Id.NAME, include = As.EXISTING_PROPERTY, property = "type") @JsonSubTypes({ @JsonSubTypes.Type(value = JsonSchemaResponseFormat.class, name = "json_schema", names = { @@ -15,23 +18,44 @@ "TEXT" }), @JsonSubTypes.Type(value = JsonObjectResponseFormat.class, name = "json_object", names = { "json_object", "JSON_OBJECT" }), -}) + + }) public abstract class ResponseFormat { + /** + * The type of the response format. + */ public static enum Type { + /** + * Only valid for openai chat completion, with GPT-4 and gpt-3.5-turbo-1106+ models. + */ JSON_OBJECT, /** * Only valid for openai chat completion, with GPT-4 and gpt-3.5-turbo-1106+ models. */ - JSON_SCHEMA, TEXT; + JSON_SCHEMA, + /** + * The response is in text format. + */ + TEXT; } private final Type type; + /** + * Creates a new instance of the {@link ResponseFormat} class. + * + * @param type The type of the response format. + */ public ResponseFormat(Type type) { this.type = type; } + /** + * Gets the type of the response format. + * + * @return The type. + */ @JsonProperty("type") public Type getType() { return type; diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseSchemaGenerator.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseSchemaGenerator.java index f9f66849..93d933a1 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseSchemaGenerator.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/ResponseSchemaGenerator.java @@ -30,15 +30,22 @@ public interface ResponseSchemaGenerator { public static ResponseSchemaGenerator jacksonGenerator() { try { return loadGenerator( - "com.microsoft.semantickernel.aiservices.openai.chatcompletion.responseformat.JacksonResponseFormatGenerator"); + "com.microsoft.semantickernel.aiservices.openai.chatcompletion.responseformat.JacksonResponseFormatGenerator"); } catch (NoClassDefFoundError e) { LOGGER.error( - "The Jackson response schema generator relies on the optional dependencies 'com.github.victools:jsonschema-generator', and 'com.github.victools:jsonschema-module-jackson'. To use this feature, please add this dependency to your project."); + "The Jackson response schema generator relies on the optional dependencies 'com.github.victools:jsonschema-generator', and 'com.github.victools:jsonschema-module-jackson'. To use this feature, please add this dependency to your project."); throw new SKException( - "The Jackson response schema generator relies on the optional dependency 'com.github.victools:jsonschema-generator', and 'com.github.victools:jsonschema-module-jackson'. To use this feature, please add this dependency to your project."); + "The Jackson response schema generator relies on the optional dependency 'com.github.victools:jsonschema-generator', and 'com.github.victools:jsonschema-module-jackson'. To use this feature, please add this dependency to your project."); } } - + + /** + * Load a response schema generator based on the given class name. + * The class must implement the {@link ResponseSchemaGenerator} interface. + * + * @param className The class name of the generator. + * @return The response schema generator. + */ public static ResponseSchemaGenerator loadGenerator(String className) { return ServiceLoadUtil .findServiceLoader(ResponseSchemaGenerator.class, diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/TextResponseFormat.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/TextResponseFormat.java index daf0e021..45214f31 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/TextResponseFormat.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/responseformat/TextResponseFormat.java @@ -3,8 +3,15 @@ import com.fasterxml.jackson.annotation.JsonCreator; +/** + * Represents a text response format. + */ public class TextResponseFormat extends ResponseFormat { + /** + * Used by Jackson to creates a new instance of the + * {@link TextResponseFormat} class. + */ @JsonCreator public TextResponseFormat() { super(Type.TEXT); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/plugin/KernelPluginFactory.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/plugin/KernelPluginFactory.java index a6a23c3b..afa31cb0 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/plugin/KernelPluginFactory.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/plugin/KernelPluginFactory.java @@ -428,6 +428,7 @@ private static KernelFunction getKernelFunction( } /** + * Imports a plugin from a resource directory on the filesystem. * @param parentDirectory The parent directory containing the plugin directories. * @param pluginDirectoryName The name of the plugin directory. * @param functionName The name of the function to import. diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/HandlebarsPromptTemplateFactory.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/HandlebarsPromptTemplateFactory.java index 786df87c..2a9b991f 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/HandlebarsPromptTemplateFactory.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/HandlebarsPromptTemplateFactory.java @@ -18,6 +18,12 @@ public class HandlebarsPromptTemplateFactory implements PromptTemplateFactory { */ public static final String HANDLEBARS_TEMPLATE_FORMAT = "handlebars"; + /** + * Initializes a new instance of the {@link HandlebarsPromptTemplateFactory} class. + */ + public HandlebarsPromptTemplateFactory() { + } + @Override public PromptTemplate tryCreate(@NonNull PromptTemplateConfig templateConfig) { if (templateConfig.getTemplateFormat() != null && diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/InputVariable.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/InputVariable.java index a819b2ee..80042263 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/InputVariable.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/InputVariable.java @@ -79,6 +79,7 @@ public InputVariable(String name) { * @param type the type of the input variable * @param description the description of the input variable * @param defaultValue the default value of the input variable + * @param enumValues the enum values of the input variable * @param required whether the input variable is required * @return a new instance of {@link InputVariable} */ diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunction.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunction.java index 8dd31b6e..70f33bd7 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunction.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunction.java @@ -99,6 +99,7 @@ public static FromPromptBuilder createFromPrompt( } /** + * Get the plugin name of the function. * @return The name of the plugin that this function is within */ @Nullable @@ -107,6 +108,7 @@ public String getPluginName() { } /** + * Get the name of the function. * @return The name of this function */ public String getName() { @@ -114,6 +116,7 @@ public String getName() { } /** + * Get the description of the function. * @return A description of the function */ @Nullable diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java index 9a67b575..a760194a 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java @@ -148,8 +148,15 @@ private static MethodDetails getMethodDetails( method.getReturnType())); } + /** + * Gets the function from the method. + * @param method the method to invoke + * @param instance the instance to invoke the method on + * @param the return type of the function + * @return the function representing the method + */ @SuppressWarnings("unchecked") - public static ImplementationFunc getFunction(Method method, Object instance) { + public static ImplementationFunc getFunction(Method method, Object instance) { return (kernel, function, arguments, variableType, invocationContext) -> { InvocationContext context; if (invocationContext == null) { @@ -683,6 +690,11 @@ private static InputVariable toKernelParameterMetadata(Parameter parameter) { isRequired); } + /** + * Gets the constants from an enum type. + * @param type the type to get the enum constants from + * @return a list of the enum constants or {@code null} if the type is not an enum + */ public static @Nullable List getEnumOptions(Class type) { List enumValues = null; if (type.isEnum()) { @@ -719,6 +731,7 @@ public Mono> invokeAsync( /** * Concrete implementation of the abstract method in KernelFunction. + * @param the return type of the function */ public interface ImplementationFunc { @@ -762,6 +775,7 @@ default FunctionResult invoke( /** * A builder for {@link KernelFunction}. + * @param the return type of the function */ public static class Builder { diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java index 697da535..adcea185 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java @@ -450,6 +450,5 @@ public KernelFunction build() { return new KernelFunctionFromPrompt<>(temp, config, executionSettings); } - } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelPromptTemplateFactory.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelPromptTemplateFactory.java index b10c6fb9..45dd8428 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelPromptTemplateFactory.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelPromptTemplateFactory.java @@ -16,6 +16,12 @@ */ public class KernelPromptTemplateFactory implements PromptTemplateFactory { + /** + * Initializes a new instance of the {@code KernelPromptTemplateFactory} class. + */ + public KernelPromptTemplateFactory() { + } + @Override public PromptTemplate tryCreate(@Nonnull PromptTemplateConfig templateConfig) { if (templateConfig.getTemplate() == null) { diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/OutputVariable.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/OutputVariable.java index 8871c03f..833aef17 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/OutputVariable.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/OutputVariable.java @@ -9,6 +9,7 @@ /** * Metadata for an output variable of a kernel function. + * @param The type of the output variable. */ public class OutputVariable { diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateConfig.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateConfig.java index 93df46f3..cbc0478b 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateConfig.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateConfig.java @@ -408,7 +408,7 @@ private Builder(PromptTemplateConfig promptTemplateConfig) { * Set the name of the prompt template config. * * @param name The name of the prompt template config. - * @return {@code this} prompt template config. + * @return {@code this} builder */ public Builder withName(String name) { this.name = name; @@ -419,7 +419,7 @@ public Builder withName(String name) { * Add an input variable to the prompt template config. * * @param inputVariable The input variable to add. - * @return {@code this} prompt template config. + * @return {@code this} builder */ public Builder addInputVariable(InputVariable inputVariable) { inputVariables.add(inputVariable); @@ -430,7 +430,7 @@ public Builder addInputVariable(InputVariable inputVariable) { * Set the template of the prompt template config. * * @param template The template of the prompt template config. - * @return {@code this} prompt template config. + * @return {@code this} builder */ public Builder withTemplate(String template) { this.template = template; @@ -441,7 +441,7 @@ public Builder withTemplate(String template) { * Set the description of the prompt template config. * * @param description The description of the prompt template config. - * @return {@code this} prompt template config. + * @return {@code this} builder */ public Builder withDescription(String description) { this.description = description; @@ -452,13 +452,18 @@ public Builder withDescription(String description) { * Set the template format of the prompt template config. * * @param templateFormat The template format of the prompt template config. - * @return {@code this} prompt template config. + * @return {@code this} builder */ public Builder withTemplateFormat(String templateFormat) { this.templateFormat = templateFormat; return this; } + /** + * Set the prompt template options. + * @param option The prompt template option to add. + * @return {@code this} builder. + */ public Builder addPromptTemplateOption(PromptTemplateOption option) { promptTemplateOptions.add(option); return this; @@ -468,7 +473,7 @@ public Builder addPromptTemplateOption(PromptTemplateOption option) { * Set the inputVariables of the prompt template config. * * @param inputVariables The input variables of the prompt template config. - * @return {@code this} prompt template config. + * @return {@code this} builder */ public Builder withInputVariables(List inputVariables) { this.inputVariables = new ArrayList<>(inputVariables); @@ -479,7 +484,7 @@ public Builder withInputVariables(List inputVariables) { * Set the output variable of the prompt template config. * * @param outputVariable The output variable of the prompt template config. - * @return {@code this} prompt template config. + * @return {@code this} builder */ public Builder withOutputVariable(OutputVariable outputVariable) { this.outputVariable = outputVariable; @@ -490,7 +495,7 @@ public Builder withOutputVariable(OutputVariable outputVariable) { * Set the prompt execution settings of the prompt template config. * * @param executionSettings The prompt execution settings of the prompt template config. - * @return {@code this} prompt template config. + * @return {@code this} builder */ public Builder withExecutionSettings( Map executionSettings) { diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateOption.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateOption.java index 5d244613..56f5a2e2 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateOption.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateOption.java @@ -1,6 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.semanticfunctions; +/** + * Options to customize the behavior of a prompt. + */ public enum PromptTemplateOption { /** * Allow methods on objects provided as arguments to an invocation, to be invoked when rendering diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/KernelContent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/KernelContent.java index 2a7ddf00..6c8c80bc 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/KernelContent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/KernelContent.java @@ -11,16 +11,18 @@ */ public interface KernelContent { - /* + /** * The inner content representation. Use this to bypass the current * abstraction. The usage of this property is considered "unsafe". * Use it only if strictly necessary. + * @return The inner content. */ @Nullable T getInnerContent(); /** * The metadata associated with the content. + * @return The metadata. */ @Nullable FunctionResultMetadata getMetadata(); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/StreamingKernelContent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/StreamingKernelContent.java index 02d9d737..dd1dcffe 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/StreamingKernelContent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/StreamingKernelContent.java @@ -1,6 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.services; +/** + * Base class which represents the content returned by an AI service. + * @param The type of the content. + */ public interface StreamingKernelContent extends KernelContent { } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/AudioToTextService.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/AudioToTextService.java index 871d4cb4..deeec0f6 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/AudioToTextService.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/AudioToTextService.java @@ -24,6 +24,10 @@ Mono getTextContentsAsync( AudioContent content, @Nullable AudioToTextExecutionSettings executionSettings); + /** + * Builder for the AudioToTextService. + * @return The builder. + */ static Builder builder() { return ServiceLoadUtil.findServiceLoader(Builder.class, "com.microsoft.semantickernel.aiservices.openai.audio.OpenAiAudioToTextService$Builder") diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/TextToAudioExecutionSettings.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/TextToAudioExecutionSettings.java index c367e37f..f85659c5 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/TextToAudioExecutionSettings.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/TextToAudioExecutionSettings.java @@ -126,6 +126,9 @@ public TextToAudioExecutionSettings build() { } return new TextToAudioExecutionSettings(voice, responseFormat, speed); } + + private Builder() { + } } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java index ea910c01..25cd8ea8 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java @@ -83,6 +83,7 @@ public void addAll(ChatHistory value) { /** * Create an {@code Iterator} from the chat history. + * @return An {@code Iterator} from the chat history. */ @Override public Iterator> iterator() { @@ -101,6 +102,7 @@ public void forEach(Consumer> action) { /** * Create a {@code Spliterator} from the chat history + * @return A {@code Spliterator} from the chat history */ @Override public Spliterator> spliterator() { @@ -114,6 +116,7 @@ public Spliterator> spliterator() { * @param content The content of the message * @param encoding The encoding of the message * @param metadata The metadata of the message + * @return {@code this} ChatHistory */ public ChatHistory addMessage(AuthorRole authorRole, String content, Charset encoding, FunctionResultMetadata metadata) { @@ -132,6 +135,7 @@ public ChatHistory addMessage(AuthorRole authorRole, String content, Charset enc * * @param authorRole The role of the author of the message * @param content The content of the message + * @return {@code this} ChatHistory */ public ChatHistory addMessage(AuthorRole authorRole, String content) { chatMessageContents.add( @@ -146,6 +150,7 @@ public ChatHistory addMessage(AuthorRole authorRole, String content) { * Add a message to the chat history * * @param content The content of the message + * @return {@code this} ChatHistory */ public ChatHistory addMessage(ChatMessageContent content) { chatMessageContents.add(content); @@ -156,6 +161,7 @@ public ChatHistory addMessage(ChatMessageContent content) { * Add a user message to the chat history * * @param content The content of the user message + * @return {@code this} ChatHistory */ public ChatHistory addUserMessage(String content) { return addMessage(AuthorRole.USER, content); @@ -165,6 +171,7 @@ public ChatHistory addUserMessage(String content) { * Add an assistant message to the chat history * * @param content The content of the assistant message + * @return {@code this} ChatHistory */ public ChatHistory addAssistantMessage(String content) { return addMessage(AuthorRole.ASSISTANT, content); @@ -174,11 +181,17 @@ public ChatHistory addAssistantMessage(String content) { * Add an system message to the chat history * * @param content The content of the system message + * @return {@code this} ChatHistory */ public ChatHistory addSystemMessage(String content) { return addMessage(AuthorRole.SYSTEM, content); } + /** + * Add all messages to the chat history + * @param messages The messages to add to the chat history + * @return {@code this} ChatHistory + */ public ChatHistory addAll(List> messages) { chatMessageContents.addAll(messages); return this; diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/StreamingChatContent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/StreamingChatContent.java index 46182913..52bb4131 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/StreamingChatContent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/StreamingChatContent.java @@ -3,7 +3,15 @@ import com.microsoft.semantickernel.services.StreamingKernelContent; +/** + * Base class which represents the content returned by a chat completion service. + * @param The type of the content. + */ public interface StreamingChatContent extends StreamingKernelContent { + /** + * Gets the ID of the content. + * @return The ID. + */ public String getId(); } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/message/ChatMessageImageContent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/message/ChatMessageImageContent.java index e87f7fcc..75ab0a3d 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/message/ChatMessageImageContent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/message/ChatMessageImageContent.java @@ -9,10 +9,21 @@ import java.util.Base64; import javax.annotation.Nullable; +/** + * Represents an image content in a chat message. + * + * @param the type of the inner content within the message + */ public class ChatMessageImageContent extends ChatMessageContent { private final ImageDetail detail; + /** + * Create a new instance of the {@link ChatMessageImageContent} class. + * @param content The chat message content + * @param modelId The LLM id to use for the chat + * @param detail The detail level of the image to include in the chat message + */ public ChatMessageImageContent( String content, @Nullable String modelId, @@ -69,6 +80,10 @@ public static Builder builder() { return new Builder<>(); } + /** + * Builder for the {@link ChatMessageImageContent} class. + * @param the type of the inner content within the message + */ public static class Builder implements SemanticKernelBuilder> { @Nullable @@ -82,7 +97,7 @@ public static class Builder implements SemanticKernelBuilder withModelId(String modelId) { this.modelId = modelId; @@ -90,10 +105,11 @@ public Builder withModelId(String modelId) { } /** + * Set the image content to include in the chat message. * @param imageType For instance jpg or png. For known types known to OpenAI see: docs. * @param content the image content - * @return this builder + * @return {@code this} builder */ public Builder withImage( String imageType, @@ -108,7 +124,7 @@ public Builder withImage( * Set the URL of the image to include in the chat message. * * @param url the URL of the image - * @return this builder + * @return {@code this} builder */ public Builder withImageUrl(String url) { this.content = url; @@ -119,7 +135,7 @@ public Builder withImageUrl(String url) { * Set the URL of the image to include in the chat message. * * @param url the URL of the image - * @return this builder + * @return {@code this} builder */ public Builder withImageUrl(URL url) { this.content = url.toString(); @@ -130,7 +146,7 @@ public Builder withImageUrl(URL url) { * Set the detail level of the image to include in the chat message. * * @param detail the detail level of the image - * @return this builder + * @return {@code this} builder */ public Builder withDetail(ImageDetail detail) { this.detail = detail; diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/message/ChatMessageTextContent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/message/ChatMessageTextContent.java index ccce7e4d..f065ed2f 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/message/ChatMessageTextContent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/message/ChatMessageTextContent.java @@ -81,6 +81,9 @@ public static ChatMessageTextContent systemMessage(String content) { return buildContent(AuthorRole.SYSTEM, content); } + /** + * Builder for the {@link ChatMessageTextContent} class. + */ public static class Builder implements SemanticKernelBuilder { @Nullable diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java index 66ea2791..dcdce1c6 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java @@ -15,6 +15,10 @@ public class Embedding { private static final Embedding EMPTY = new Embedding(); + /** + * Returns an empty {@code Embedding} instance. + * @return An empty {@code Embedding} instance. + */ public static Embedding empty() { return EMPTY; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java index 79ce998e..561dbe1d 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java @@ -6,7 +6,8 @@ import reactor.core.publisher.Mono; /** - * Interface for text embedding generation services + * Interface for text embedding generation services + * @param The type of the data to generate embeddings for */ public interface EmbeddingGenerationService extends AIService { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java index e0e763c1..5aeb5ef3 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java @@ -13,6 +13,9 @@ import javax.annotation.Nullable; import reactor.core.publisher.Mono; +/** + * Represents an Azure AI Search vector store. + */ public class AzureAISearchVectorStore implements VectorStore { private final SearchIndexAsyncClient searchIndexAsyncClient; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java index 7afe3c35..996790f3 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java @@ -21,6 +21,9 @@ import java.util.List; import javax.annotation.Nonnull; +/** + * Maps vector store record fields to Azure AI Search fields. + */ class AzureAISearchVectorStoreCollectionCreateMapping { private static String getVectorSearchProfileName(VectorStoreRecordVectorField vectorField) { @@ -71,12 +74,24 @@ private static VectorSearchAlgorithmConfiguration getAlgorithmConfig( } } + /** + * Maps a key field to a search field. + * + * @param keyField The key field. + * @return The search field. + */ public static SearchField mapKeyField(VectorStoreRecordKeyField keyField) { return new SearchField(keyField.getEffectiveStorageName(), SearchFieldDataType.STRING) .setKey(true) .setFilterable(true); } + /** + * Maps a data field to a search field. + * + * @param dataField The data field. + * @return The search field. + */ public static SearchField mapDataField(VectorStoreRecordDataField dataField) { if (dataField.getFieldType() == null) { throw new SKException( @@ -89,6 +104,12 @@ public static SearchField mapDataField(VectorStoreRecordDataField dataField) { .setSearchable(dataField.isFullTextSearchable()); } + /** + * Maps a vector field to a search field. + * + * @param vectorField The vector field. + * @return The search field. + */ public static SearchField mapVectorField(VectorStoreRecordVectorField vectorField) { return new SearchField(vectorField.getEffectiveStorageName(), SearchFieldDataType.collection(SearchFieldDataType.SINGLE)) @@ -97,6 +118,13 @@ public static SearchField mapVectorField(VectorStoreRecordVectorField vectorFiel .setVectorSearchProfileName(getVectorSearchProfileName(vectorField)); } + /** + * Updates the vector search parameters for the specified vector field. + * + * @param algorithms The list of vector search algorithms. + * @param profiles The list of vector search profiles. + * @param vectorField The vector field. + */ public static void updateVectorSearchParameters( List algorithms, List profiles, @@ -110,6 +138,12 @@ public static void updateVectorSearchParameters( getVectorSearchProfileName(vectorField), getAlgorithmConfigName(vectorField))); } + /** + * Gets the search field data type for the specified field type. + * + * @param fieldType The field type. + * @return The search field data type. + */ public static SearchFieldDataType getSearchFieldDataType(Class fieldType) { if (fieldType == String.class) { return SearchFieldDataType.STRING; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreOptions.java index db3b7ab6..2a5d71b8 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreOptions.java @@ -3,6 +3,9 @@ import javax.annotation.Nullable; +/** + * Represents the options for the Azure AI Search vector store. + */ public class AzureAISearchVectorStoreOptions { @Nullable diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index fad24fe5..622bc6d0 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -43,6 +43,11 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +/** + * Represents an Azure AI Search vector store record collection. + * + * @param The type of the record. + */ public class AzureAISearchVectorStoreRecordCollection implements VectorStoreRecordCollection, VectorizedSearch, @@ -82,6 +87,13 @@ public class AzureAISearchVectorStoreRecordCollection implements private final List nonVectorFields = new ArrayList<>(); private final String firstVectorFieldName; + /** + * Creates a new instance of {@link AzureAISearchVectorStoreRecordCollection}. + * + * @param searchIndexAsyncClient The Azure AI Search client. + * @param collectionName The name of the collection. + * @param options The options for the collection. + */ @SuppressFBWarnings("EI_EXPOSE_REP2") public AzureAISearchVectorStoreRecordCollection( @Nonnull SearchIndexAsyncClient searchIndexAsyncClient, diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java index b0acf545..6856b3c1 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java @@ -16,6 +16,7 @@ public interface AzureAISearchVectorStoreRecordCollectionFactory { * @param collectionName The name of the collection. * @param recordClass The class type of the record. * @param recordDefinition The record definition. + * @param The record type. * @return The new Azure AI Search vector store record collection. */ AzureAISearchVectorStoreRecordCollection createVectorStoreRecordCollection( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java index d1fad192..e62f1020 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java @@ -95,6 +95,11 @@ public static class Builder { @Nullable private VectorStoreRecordDefinition recordDefinition; + /** + * Sets the record class. + * @param recordClass the record Class + * @return the builder + */ public Builder withRecordClass(Class recordClass) { this.recordClass = recordClass; return this; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchAnyTagEqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchAnyTagEqualToFilterClause.java index 7c84174e..81a4bb72 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchAnyTagEqualToFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchAnyTagEqualToFilterClause.java @@ -3,13 +3,16 @@ import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; +/** + * A filter clause that filters on any tag equal to a value. + */ public class AzureAISearchAnyTagEqualToFilterClause extends AnyTagEqualToFilterClause { /** * Initializes a new instance of the AzureAISearchTagListContainsFilterClause class. * * @param fieldName The field name to filter on. - * @param value The value. + * @param value The value to filter on. */ public AzureAISearchAnyTagEqualToFilterClause(String fieldName, Object value) { super(fieldName, value); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchEqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchEqualToFilterClause.java index 0bacb35c..aa03b532 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchEqualToFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/filter/AzureAISearchEqualToFilterClause.java @@ -7,13 +7,16 @@ import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; +/** + * A filter clause that filters on equality. + */ public class AzureAISearchEqualToFilterClause extends EqualToFilterClause { /** * Initializes a new instance of the AzureAISearchEqualityFilterClause class. * * @param fieldName The field name to filter on. - * @param value The value. + * @param value The value to filter on. */ public AzureAISearchEqualToFilterClause(String fieldName, Object value) { super(fieldName, value); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/hsqldb/HSQLDBVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/hsqldb/HSQLDBVectorStoreQueryProvider.java index e8a0549d..86f7ec6e 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/hsqldb/HSQLDBVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/hsqldb/HSQLDBVectorStoreQueryProvider.java @@ -23,6 +23,10 @@ import java.util.stream.Collectors; import javax.sql.DataSource; +/** + * The HSQLDB vector store query provider. + * Provides the necessary methods to interact with a HSQLDB vector store and vector store collections. + */ public class HSQLDBVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider { @SuppressFBWarnings("EI_EXPOSE_REP2") @@ -116,7 +120,6 @@ private void setStatementValues(PreparedStatement statement, Object record, @Override @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") // SQL query is generated dynamically with valid identifiers - public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { validateSQLidentifier(getCollectionTableName(collectionName)); @@ -171,6 +174,9 @@ public static Builder builder() { return new Builder(); } + /** + * The builder for the HSQLDB vector store query provider. + */ public static class Builder extends JDBCVectorStoreQueryProvider.Builder { @@ -179,6 +185,12 @@ public static class Builder private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; private int defaultVarCharLength = 255; + /** + * Sets the data source. + * + * @param dataSource the data source + * @return the builder + */ @SuppressFBWarnings("EI_EXPOSE_REP2") public Builder withDataSource(DataSource dataSource) { this.dataSource = dataSource; @@ -207,11 +219,22 @@ public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { return this; } + /** + * Sets the default VARCHAR length. + * + * @param defaultVarCharLength the default VARCHAR length + * @return the builder + */ public Builder setDefaultVarCharLength(int defaultVarCharLength) { this.defaultVarCharLength = defaultVarCharLength; return this; } + /** + * Builds the HSQLDB vector store query provider. + * + * @return the HSQLDB vector store query provider + */ public HSQLDBVectorStoreQueryProvider build() { if (dataSource == null) { throw new SKException("DataSource is required"); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java index 4e349eb1..3d045e58 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java @@ -102,6 +102,7 @@ public VectorStoreRecordCollection getCollection( * @param collectionName The name of the collection. * @param recordClass The class type of the record. * @param recordDefinition The record definition. + * @param The record type. * @return The collection. */ public VectorStoreRecordCollection getCollection( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java index 58e2c85e..db2e6c8d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java @@ -5,6 +5,9 @@ import javax.annotation.Nullable; +/** + * Options for the JDBC vector store. + */ public class JDBCVectorStoreOptions { @Nullable private final JDBCVectorStoreRecordCollectionFactory vectorStoreRecordCollectionFactory; @@ -15,6 +18,8 @@ public class JDBCVectorStoreOptions { * Creates a new instance of the JDBC vector store options. * * @param vectorStoreRecordCollectionFactory The vector store record collection factory. + * @param queryProvider The query provider. + * */ @SuppressFBWarnings("EI_EXPOSE_REP2") // DataSource in queryProvider is not exposed public JDBCVectorStoreOptions( @@ -63,7 +68,6 @@ public JDBCVectorStoreRecordCollectionFactory getVectorStoreRecordCollectionFact /** * Builder for JDBC vector store options. - * */ public static class Builder { @Nullable diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index 0c07588c..7970ffad 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -33,6 +33,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * A JDBC vector store query provider. + */ public class JDBCVectorStoreQueryProvider implements SQLVectorStoreQueryProvider { @@ -78,6 +81,15 @@ protected JDBCVectorStoreQueryProvider( supportedVectorTypes.put(Collection.class, "TEXT"); } + /** + * Creates a new instance of the JDBCVectorStoreQueryProvider class. + * @param dataSource the data source + * @param collectionsTable the collections table + * @param prefixForCollectionTables the prefix for collection tables + * @param supportedKeyTypes the supported key types + * @param supportedDataTypes the supported data types + * @param supportedVectorTypes the supported vector types + */ public JDBCVectorStoreQueryProvider( @SuppressFBWarnings("EI_EXPOSE_REP2") @Nonnull DataSource dataSource, @Nonnull String collectionsTable, diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java index f43fca6e..c7f839c2 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -27,6 +27,14 @@ import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; +/** + * The JDBCVectorStoreRecordCollection class represents a collection of records + * in a JDBC vector store. It implements the SQLVectorStoreRecordCollection + * interface and provides methods for managing the collection, such as creating, + * deleting, and upserting records. + * + * @param the type of the records in the collection + */ public class JDBCVectorStoreRecordCollection implements SQLVectorStoreRecordCollection { @@ -312,7 +320,11 @@ public Mono>> searchAsync(List vector, vectorStoreRecordMapper)) .subscribeOn(Schedulers.boundedElastic()); } - + + /** + * Builder for a JDBCVectorStoreRecordCollection. + * @param the type of the records in the collection + */ public static class Builder implements SemanticKernelBuilder> { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java index 14cdcc27..50063627 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java @@ -17,6 +17,7 @@ public interface JDBCVectorStoreRecordCollectionFactory { * @param collectionName The name of the collection. * @param recordClass The class type of the * @param recordDefinition The record definition. + * @param The type of record in the collection. * @return The new JDBC vector store record collection. */ JDBCVectorStoreRecordCollection createVectorStoreRecordCollection( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java index 8ae826d4..5d2e439c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java @@ -13,6 +13,10 @@ import static com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider.DEFAULT_COLLECTIONS_TABLE; import static com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider.DEFAULT_PREFIX_FOR_COLLECTION_TABLES; +/** + * Options for a JDBC vector store record collection. + * @param the record type + */ public class JDBCVectorStoreRecordCollectionOptions implements VectorStoreRecordCollectionOptions { private final Class recordClass; @@ -105,6 +109,10 @@ public SQLVectorStoreQueryProvider getQueryProvider() { return queryProvider; } + /** + * Builder for JDBC vector store record collection options. + * @param the record type + */ public static class Builder { private Class recordClass; private VectorStoreRecordDefinition recordDefinition; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java index 8393c9f2..35fafedd 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java @@ -20,6 +20,11 @@ import java.sql.SQLException; import java.util.function.BiFunction; +/** + * Maps a JDBC result set to a record. + * + * @param the record type + */ public class JDBCVectorStoreRecordMapper extends VectorStoreRecordMapper { @@ -51,6 +56,11 @@ public ResultSet mapRecordToStorageModel(Record record) { throw new UnsupportedOperationException("Not implemented"); } + /** + * Builder for {@link JDBCVectorStoreRecordMapper}. + * + * @param the record type + */ public static class Builder implements SemanticKernelBuilder> { private Class recordClass; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java index ee45f8be..5fc68a64 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java @@ -4,6 +4,9 @@ import com.microsoft.semantickernel.data.vectorstorage.VectorStore; import reactor.core.publisher.Mono; +/** + * Represents a SQL vector store. + */ public interface SQLVectorStore extends VectorStore { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java index 2c7109f8..d33a686f 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java @@ -101,6 +101,7 @@ public interface SQLVectorStoreQueryProvider { * @param recordDefinition the record definition * @param mapper the mapper * @param options the options + * @param the record type * @return the records */ List getRecords(String collectionName, List keys, diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java index 711344fa..120d0273 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java @@ -4,6 +4,12 @@ import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import reactor.core.publisher.Mono; +/** + * Represents a SQL vector store record collection. + * + * @param the key type + * @param the record type + */ public interface SQLVectorStoreRecordCollection extends VectorStoreRecordCollection { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollectionSearchMapping.java index 17164acc..81408b8e 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollectionSearchMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollectionSearchMapping.java @@ -10,8 +10,18 @@ import java.util.List; import java.util.stream.Collectors; +/** + * A mapping for searching a collection of vector records in SQL. + */ public class SQLVectorStoreRecordCollectionSearchMapping { + /** + * Builds a filter for searching a collection of vector records in SQL. + * + * @param vectorSearchFilter the search filter + * @param recordDefinition the record definition + * @return the filter + */ public static String buildFilter(VectorSearchFilter vectorSearchFilter, VectorStoreRecordDefinition recordDefinition) { if (vectorSearchFilter == null @@ -34,6 +44,12 @@ public static String buildFilter(VectorSearchFilter vectorSearchFilter, }).collect(Collectors.joining(" AND ")); } + /** + * Gets the filter parameters. + * + * @param vectorSearchFilter the search filter + * @return the filter parameters + */ public static List getFilterParameters(VectorSearchFilter vectorSearchFilter) { if (vectorSearchFilter == null || vectorSearchFilter.getFilterClauses().isEmpty()) { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/filter/SQLEqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/filter/SQLEqualToFilterClause.java index d90b27b7..488c1473 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/filter/SQLEqualToFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/filter/SQLEqualToFilterClause.java @@ -7,6 +7,9 @@ import java.time.OffsetDateTime; +/** + * Represents an equality filter clause for SQL. + */ public class SQLEqualToFilterClause extends EqualToFilterClause { /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java index 87aa4d69..08b133cf 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java @@ -21,6 +21,11 @@ import java.util.List; import java.util.stream.Collectors; +/** + * The MySQL vector store query provider. + * Provides the necessary methods to interact with a MySQL + * vector store and vector store collections. + */ public class MySQLVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider implements SQLVectorStoreQueryProvider { @@ -110,6 +115,9 @@ public void upsertRecords(String collectionName, List records, } } + /** + * The MySQL vector store query provider builder. + */ public static class Builder extends JDBCVectorStoreQueryProvider.Builder { private DataSource dataSource; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java index 001a5499..669f3652 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java @@ -3,9 +3,26 @@ import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +/** + * Represents a PostgreSQL vector distance function. + */ public enum PostgreSQLVectorDistanceFunction { - L2("vector_l2_ops", "<->"), COSINE("vector_cosine_ops", "<=>"), INNER_PRODUCT("vector_ip_ops", - "<#>"), UNDEFINED(null, null); + /** + * Euclidean L2 distance function. + */ + L2("vector_l2_ops", "<->"), + /** + * The cosine distance function. + */ + COSINE("vector_cosine_ops", "<=>"), + /** + * The inner product distance function. + */ + INNER_PRODUCT("vector_ip_ops", "<#>"), + /** + * The distance function is undefined. + */ + UNDEFINED(null, null); private final String value; private final String operator; @@ -15,14 +32,27 @@ public enum PostgreSQLVectorDistanceFunction { this.operator = operator; } + /** + * Gets the value of the distance function. + * @return the value of the distance function + */ public String getValue() { return value; } + /** + * Gets the operator of the distance function. + * @return the operator of the distance function + */ public String getOperator() { return operator; } + /** + * Converts a distance function to a PostgreSQL vector distance function. + * @param function the distance function + * @return the PostgreSQL vector distance function + */ public static PostgreSQLVectorDistanceFunction fromDistanceFunction(DistanceFunction function) { switch (function) { case EUCLIDEAN_DISTANCE: diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java index 1a04fb18..6681ef83 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java @@ -3,8 +3,22 @@ import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; +/** + * Represents a PostgreSQL vector index kind. + */ public enum PostgreSQLVectorIndexKind { - HNSW("hnsw"), IVFFLAT("ivfflat"), UNDEFINED(null); + /** + * The vector is indexed using an HNSW algorithm. + */ + HNSW("hnsw"), + /** + * The vector is indexed using a Flat algorithm. + */ + IVFFLAT("ivfflat"), + /** + * The indexing algorithm is undefined. + */ + UNDEFINED(null); private final String value; @@ -12,10 +26,19 @@ public enum PostgreSQLVectorIndexKind { this.value = value; } + /** + * Gets the pgvector value of the index kind. + * @return the pgvector value of the index kind + */ public String getValue() { return value; } + /** + * Converts an index kind to a PostgreSQL vector index kind. + * @param indexKind the index kind + * @return the PostgreSQL vector index kind + */ public static PostgreSQLVectorIndexKind fromIndexKind(IndexKind indexKind) { switch (indexKind) { case HNSW: diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index 15f7971b..a353cdcb 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -34,6 +34,11 @@ import java.util.Map; import java.util.stream.Collectors; +/** + * The MySQL vector store query provider. + * Provides the necessary methods to interact with a MySQL + * vector store and vector store collections. + */ public class PostgreSQLVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider implements SQLVectorStoreQueryProvider { @@ -413,6 +418,9 @@ public List> search(String collectionName, } } + /** + * A builder for the PostgreSQLVectorStoreQueryProvider class. + */ public static class Builder extends JDBCVectorStoreQueryProvider.Builder { private DataSource dataSource; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java index 9550ade8..50cbbd76 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java @@ -20,6 +20,11 @@ import java.util.List; import java.util.function.BiFunction; +/** + * A mapper to convert between a record and a PostgreSQL storage model. + * + * @param the record type + */ public class PostgreSQLVectorStoreRecordMapper extends VectorStoreRecordMapper { @@ -43,6 +48,11 @@ public static Builder builder() { return new Builder<>(); } + /** + * A builder for the PostgreSQLVectorStoreRecordMapper. + * + * @param the record type + */ public static class Builder implements SemanticKernelBuilder> { private Class recordClass; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java index 115fc43b..23f9aa7c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java @@ -39,6 +39,11 @@ import java.util.Map; import java.util.stream.Collectors; +/** + * RedisHashSetVectorStoreRecordCollection is a class that represents a + * collection of records stored in Redis using the Hash Set data structure. + * @param The record type. + */ public class RedisHashSetVectorStoreRecordCollection implements VectorStoreRecordCollection, VectorizedSearch { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java index 52dc404b..071f40fc 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java @@ -13,6 +13,10 @@ import java.util.Map; import java.util.Map.Entry; +/** + * Options for a Redis hash set vector store record collection. + * @param the record type + */ public class RedisHashSetVectorStoreRecordCollectionOptions implements VectorStoreRecordCollectionOptions { private final Class recordClass; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java index 2f91ea11..25678ef2 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java @@ -30,6 +30,10 @@ import static com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollection.stringToBytes; +/** + * A mapper to convert between a record and a Redis hash set storage model. + * @param the record type + */ public class RedisHashSetVectorStoreRecordMapper extends VectorStoreRecordMapper>> { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java index d2df1766..b6eefd7a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java @@ -47,6 +47,11 @@ import redis.clients.jedis.search.Schema; import redis.clients.jedis.search.SearchResult; +/** + * Represents a Redis vector store record collection. + * + * @param The type of record in the collection. + */ public class RedisJsonVectorStoreRecordCollection implements VectorStoreRecordCollection, VectorizedSearch { @@ -72,6 +77,7 @@ public class RedisJsonVectorStoreRecordCollection * Creates a new instance of the RedisVectorRecordStore. * * @param client The Redis client. + * @param collectionName The name of the collection. * @param options The options for the store. */ @SuppressFBWarnings("EI_EXPOSE_REP2") diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java index 9702e185..cd2da3b9 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java @@ -12,6 +12,11 @@ import javax.annotation.Nullable; import java.util.Map.Entry; +/** + * Options for a Redis vector store record collection. + * + * @param the record type + */ public class RedisJsonVectorStoreRecordCollectionOptions implements VectorStoreRecordCollectionOptions { private final Class recordClass; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java index c71389cf..ecb449fb 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java @@ -17,6 +17,10 @@ import java.util.function.Function; import javax.annotation.Nullable; +/** + * A mapper to convert between a record and a Redis JSON storage model. + * @param the record type + */ public class RedisJsonVectorStoreRecordMapper extends VectorStoreRecordMapper> { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java index b9bf20b9..aa901e02 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java @@ -14,6 +14,9 @@ import reactor.core.publisher.Mono; import redis.clients.jedis.JedisPooled; +/** + * Represents a Redis vector store. + */ public class RedisVectorStore implements VectorStore { private final JedisPooled client; @@ -36,6 +39,9 @@ public RedisVectorStore(@Nonnull JedisPooled client, * Gets a collection from the vector store. * * @param collectionName The name of the collection. + * @param options The options for the collection. + * @param The type of key in the collection. + * @param The type of record in the collection. * @return The collection. */ public VectorStoreRecordCollection getCollection( @@ -82,12 +88,16 @@ public Mono> getCollectionNamesAsync() { } /** - * Builder for the Redis vector store. + * Create a builder for the Redis vector store. + * @return A new builder */ public static Builder builder() { return new Builder(); } + /** + * Builder for the Redis vector store. + */ public static class Builder implements SemanticKernelBuilder { @Nullable diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java index a9d7a329..46a4c09b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java @@ -16,6 +16,9 @@ import com.microsoft.semantickernel.exceptions.SKException; import redis.clients.jedis.search.Schema; +/** + * Maps a vector store record collection to a Redis schema. + */ public class RedisVectorStoreCollectionCreateMapping { private static final HashSet> supportedFilterableNumericTypes = new HashSet<>( Arrays.asList( @@ -72,6 +75,13 @@ private static String getRedisPath(String name, boolean withRedisJsonRoot) { return withRedisJsonRoot ? "$." + name : name; } + /** + * Maps a vector store record collection to a Redis schema. + * + * @param fields the fields + * @param storageType the Redis storage type + * @return the schema + */ public static Schema mapToSchema(List fields, RedisStorageType storageType) { Schema schema = new Schema(); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java index f9ea409b..ce2b7de0 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java @@ -16,10 +16,21 @@ import redis.clients.jedis.args.SortingOrder; import redis.clients.jedis.search.FTSearchParams; +/** + * A mapping for searching a collection of vector records in Redis. + */ public class RedisVectorStoreCollectionSearchMapping { static final String VECTOR_SCORE_FIELD = "vector_score"; + /** + * Builds a query for searching a collection of vector records in Redis. + * @param vector the vector to search for + * @param options the search options + * @param recordDefinition the record definition + * @param storageType the storage type + * @return the query and search parameters + */ public static Pair buildQuery(List vector, VectorSearchOptions options, VectorStoreRecordDefinition recordDefinition, @@ -66,6 +77,11 @@ public static Pair buildQuery(List vector, return Pair.of(knn, searchParams); } + /** + * Converts a list of floats to a byte array. + * @param embeddings the embeddings + * @return the byte array + */ public static byte[] convertListToByteArray(List embeddings) { ByteBuffer bytes = ByteBuffer.allocate(Float.BYTES * embeddings.size()); bytes.order(ByteOrder.LITTLE_ENDIAN); @@ -73,6 +89,11 @@ public static byte[] convertListToByteArray(List embeddings) { return bytes.array(); } + /** + * Converts a byte array to a list of floats. + * @param bytes the byte array + * @return the list of floats + */ public static List convertByteArrayToList(byte[] bytes) { ByteBuffer buffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN); List embeddings = new java.util.ArrayList<>(); @@ -82,6 +103,12 @@ public static List convertByteArrayToList(byte[] bytes) { return embeddings; } + /** + * Builds a filter for searching a collection of vector records in Redis. + * @param vectorSearchFilter the search filter + * @param recordDefinition the record definition + * @return the filter + */ public static String buildFilter(VectorSearchFilter vectorSearchFilter, VectorStoreRecordDefinition recordDefinition) { if (vectorSearchFilter == null diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreOptions.java index 2f09e930..338cf193 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreOptions.java @@ -6,6 +6,9 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +/** + * Options for the Redis vector store. + */ public class RedisVectorStoreOptions { @Nullable private final RedisVectorStoreRecordCollectionFactory vectorStoreRecordCollectionFactory; @@ -15,7 +18,7 @@ public class RedisVectorStoreOptions { /** * Creates a new instance of the Redis vector store options. - * + * @param storageType The storage type. * @param vectorStoreRecordCollectionFactory The vector store record collection factory. */ public RedisVectorStoreOptions( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java index 0752493c..f47fb4c8 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java @@ -17,6 +17,7 @@ public interface RedisVectorStoreRecordCollectionFactory { * @param collectionName The name of the collection. * @param recordClass The class type of the record. * @param recordDefinition The record definition. + * @param The type of the records in the collection. * @return The collection. */ VectorStoreRecordCollection createVectorStoreRecordCollection( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/filter/RedisEqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/filter/RedisEqualToFilterClause.java index aa823ef9..0cdae1f0 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/filter/RedisEqualToFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/filter/RedisEqualToFilterClause.java @@ -7,7 +7,16 @@ import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; +/** + * Represents a Redis filter clause. + */ public class RedisEqualToFilterClause extends EqualToFilterClause { + + /** + * Creates a new instance of RedisEqualToFilterClause. + * @param fieldName The name of the field to filter on. + * @param value The value to filter for. + */ public RedisEqualToFilterClause(String fieldName, Object value) { super(fieldName, value); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/sqlite/SQLiteVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/sqlite/SQLiteVectorStoreQueryProvider.java index c501c99f..8e639007 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/sqlite/SQLiteVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/sqlite/SQLiteVectorStoreQueryProvider.java @@ -21,6 +21,9 @@ import java.util.List; import java.util.stream.Collectors; +/** + * A query provider for a vector store in SQLite. + */ public class SQLiteVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider implements SQLVectorStoreQueryProvider { @@ -103,6 +106,9 @@ public void upsertRecords(String collectionName, List records, } } + /** + * A builder for {@code SQLiteVectorStoreQueryProvider}. + */ public static class Builder extends JDBCVectorStoreQueryProvider.Builder { private DataSource dataSource; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java index 0cf1044a..fd27b6d7 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java @@ -13,10 +13,18 @@ import com.microsoft.semantickernel.exceptions.SKException; import reactor.core.publisher.Mono; +/** + * Represents a volatile vector store. + * A volatile vector store is an in-memory vector store + * that does not persist data. + */ public class VolatileVectorStore implements VectorStore { private final Map> collections; + /** + * Creates a new instance of the volatile vector store. + */ public VolatileVectorStore() { this.collections = new ConcurrentHashMap<>(); } @@ -44,7 +52,7 @@ public VectorStoreRecordCollection getCollection( collections, (VolatileVectorStoreRecordCollectionOptions) options); } - + /** * Gets the names of all collections in the vector store. * diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java index 79a1326d..3a5eab6a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java @@ -13,8 +13,21 @@ import java.util.List; import java.util.stream.Collectors; +/** + * Provides methods to filter records based on a {@link VectorSearchFilter}. + */ public class VolatileVectorStoreCollectionSearchMapping { + /** + * Filters the records based on the given {@link VectorSearchFilter}. + * + * @param records The records to filter. + * @param filter The filter to apply. + * @param recordDefinition The record definition. + * @param objectMapper The object mapper. + * @param The record type. + * @return The filtered records. + */ public static List filterRecords(List records, VectorSearchFilter filter, VectorStoreRecordDefinition recordDefinition, ObjectMapper objectMapper) { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java index 3b40c16b..7d6031d4 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java @@ -32,6 +32,11 @@ import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; +/** + * Represents a volatile vector store record collection. + * + * @param The type of record in the collection. + */ public class VolatileVectorStoreRecordCollection implements VectorStoreRecordCollection { @@ -43,6 +48,12 @@ public class VolatileVectorStoreRecordCollection implements private final VectorStoreRecordDefinition recordDefinition; private final ObjectMapper objectMapper; + /** + * Creates a new instance of the volatile vector store record collection. + * + * @param collectionName The name of the collection. + * @param options The options for the collection. + */ public VolatileVectorStoreRecordCollection(String collectionName, VolatileVectorStoreRecordCollectionOptions options) { this.collectionName = collectionName; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java index 592e7653..0b5779df 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java @@ -9,6 +9,11 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +/** + * Represents the options for a volatile vector store record collection. + * + * @param the record type + */ public class VolatileVectorStoreRecordCollectionOptions implements VectorStoreRecordCollectionOptions { private final Class recordClass; @@ -22,6 +27,7 @@ public class VolatileVectorStoreRecordCollectionOptions * * @param recordClass The record class. * @param recordDefinition The record definition. + * @param objectMapper An instanc of Jackson ObjectMapper. */ @SuppressFBWarnings("EI_EXPOSE_REP2") // ObjectMapper only has package visibility public VolatileVectorStoreRecordCollectionOptions(@Nonnull Class recordClass, diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java index 1b940717..5cbfe9e7 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java @@ -1,11 +1,19 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.filter; +/** + * A filter clause that filters on any tag equal to a value. + */ public abstract class AnyTagEqualToFilterClause implements FilterClause { private final String fieldName; private final Object value; + /** + * Creates a new instance of the AnyTagEqualToFilterClause class. + * @param fieldName The field name to filter on. + * @param value The value to filter on. + */ public AnyTagEqualToFilterClause(String fieldName, Object value) { this.fieldName = fieldName; this.value = value; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java index 3590b252..b12f934d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java @@ -1,11 +1,20 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.filter; +/** + * A filter clause that filters on a field equal to a value. + */ public class EqualToFilterClause implements FilterClause { private final String fieldName; private final Object value; + /** + * Initializes a new instance of the EqualToFilterClause class. + * + * @param fieldName The field name to filter on. + * @param value The value to filter on. + */ public EqualToFilterClause(String fieldName, Object value) { this.fieldName = fieldName; this.value = value; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java index e3425be6..c8b86ca1 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java @@ -1,6 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.filter; +/** + * A filter clause for a query. + */ public interface FilterClause { /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java index 485c16c3..4f90faa8 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java @@ -17,6 +17,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +/** + * Operations for working with vectors. + */ public final class VectorOperations { /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java index cde9594f..1d0b8bf4 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java @@ -9,6 +9,9 @@ import java.util.Collections; import java.util.List; +/** + * A vector search filter. + */ public class VectorSearchFilter { /** @@ -22,6 +25,9 @@ public static VectorSearchFilter createDefault() { private final List filterClauses; + /** + * Creates a new instance of the VectorSearchFilter class. + */ public VectorSearchFilter() { this(Collections.emptyList()); } @@ -44,10 +50,18 @@ public List getFilterClauses() { return filterClauses; } + /** + * Creates a {@link Builder} for the VectorSearchFilter class. + * + * @return A new instance of the VectorSearchFilter Builder. + */ public static Builder builder() { return new Builder(); } + /** + * A builder for the VectorSearchFilter class. + */ public static class Builder { private final List filterClauses = new ArrayList<>(); @@ -74,6 +88,11 @@ public Builder withAnyTagEqualToFilterClause( return this; } + /** + * Builds the VectorSearchFilter. + * + * @return The VectorSearchFilter. + */ public VectorSearchFilter build() { return new VectorSearchFilter(filterClauses); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java index 6e508669..e1f26bee 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java @@ -1,6 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.vectorsearch; +/** + * Represents a vector search result. + * @param The type of the record. + */ public class VectorSearchResult { private final Record record; private final double score; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java index ab7c779e..00046444 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java @@ -6,6 +6,11 @@ import java.util.List; +/** + * A vectorizable text search. + * + * @param The record type. + */ public interface VectorizableTextSearch { /** * Vectorizable text search. This method searches for records that are similar to the given text. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java index 1c9274a9..59010730 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java @@ -6,6 +6,11 @@ import java.util.List; +/** + * A vectorized search. + * + * @param The record type. + */ public interface VectorizedSearch { /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java index 26e20c15..de548be4 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java @@ -16,6 +16,8 @@ public interface VectorStore { * * @param collectionName The name of the collection. * @param options The options for the collection. + * @param The key type. + * @param The record type. * @return The collection. */ VectorStoreRecordCollection getCollection( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java index 38655fbc..1eb6124d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java @@ -8,6 +8,12 @@ import java.util.List; import reactor.core.publisher.Mono; +/** + * Represents a collection of records in a vector store. + * + * @param The type of the key of the records in the collection. + * @param The type of the records in the collection. + */ public interface VectorStoreRecordCollection extends VectorizedSearch { /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java index 5782b7b0..926bd984 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java @@ -3,6 +3,12 @@ import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +/** + * Represents the options for a collection of vector store records. + * + * @param the type of the key + * @param the type of the record + */ public interface VectorStoreRecordCollectionOptions { /** * Gets the key class. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java index 8e3e7ae2..85b14d61 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java @@ -64,6 +64,7 @@ public StorageModel mapRecordToStorageModel(Record record) { * Converts a storage model to a record. * * @param storageModel the storage model to convert + * @param options the options * @return the record */ public Record mapStorageModelToRecord(StorageModel storageModel, GetRecordOptions options) { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordDataAttribute.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordDataAttribute.java index a5103105..151785a1 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordDataAttribute.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordDataAttribute.java @@ -16,16 +16,19 @@ * Storage name of the field. * This value is only used when JSON Serialization using Jackson is not supported in a VectorStore. * When Jackson is supported, @JsonProperty should be used to specify an alternate field name in the storage database. + * @return The storage name of the field. */ String storageName() default ""; /** * Whether the field is filterable. + * @return {@code true} if the field is filterable. */ boolean isFilterable() default false; /** * Whether the field is full text searchable. + * @return {@code true} if the field is full text searchable. */ boolean isFullTextSearchable() default false; -} \ No newline at end of file +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKeyAttribute.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKeyAttribute.java index 8d54d60b..27ba5fa7 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKeyAttribute.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordKeyAttribute.java @@ -14,6 +14,7 @@ public @interface VectorStoreRecordKeyAttribute { /** * Storage name of the field. + * @return The storage name of the field. */ String storageName() default ""; } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java index a72c2c69..ef22c32c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/attributes/VectorStoreRecordVectorAttribute.java @@ -19,22 +19,26 @@ /** * Number of dimensions in the vector. + * @return The number of dimensions in the vector. */ int dimensions(); /** * Storage name of the field. + * @return The storage name of the field. */ String storageName() default ""; /** * Type of index to be used for the vector. + * @return The type of index to be used for the vector. */ @Nullable IndexKind indexKind() default IndexKind.UNDEFINED; /** * Distance function to be used for to compute the distance between vectors. + * @return The distance function to be used for to compute the distance between vectors. */ @Nullable DistanceFunction distanceFunction() default DistanceFunction.UNDEFINED; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java index 8b2577a7..1512db52 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java @@ -1,6 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.vectorstorage.definition; +/** + * Distance functions for vector storage. + */ public enum DistanceFunction { /** * Cosine (angular) similarity function. @@ -29,6 +32,10 @@ public enum DistanceFunction { this.value = value; } + /** + * Gets the function name. + * @return The function name. + */ public String getValue() { return value; } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java index cda1598b..4372d651 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java @@ -1,6 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.vectorstorage.definition; +/** + * Represents the kind of index to use for a vector store. + */ public enum IndexKind { /** * Hierarchical Navigable Small World, which performs an approximate nearest neighbour (ANN) search. @@ -31,6 +34,11 @@ public enum IndexKind { this.value = value; } + /** + * Gets the string value of the index kind. + * + * @return the string value of the index kind + */ public String getValue() { return value; } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java index 76a47187..9e5aea11 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java @@ -4,10 +4,17 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +/** + * Represents a data field in a record. + */ public class VectorStoreRecordDataField extends VectorStoreRecordField { private final boolean isFilterable; private final boolean isFullTextSearchable; + /** + * Create a builder for the VectorStoreRecordDataField class. + * @return a new instance of the builder + */ public static Builder builder() { return new Builder(); } @@ -19,6 +26,7 @@ public static Builder builder() { * @param storageName the storage name of the field * @param fieldType the field type * @param isFilterable a value indicating whether the field is filterable + * @param isFullTextSearchable a value indicating whether the field is full text searchable */ public VectorStoreRecordDataField( @Nonnull String name, @@ -49,6 +57,9 @@ public boolean isFullTextSearchable() { return isFullTextSearchable; } + /** + * Builder for the VectorStoreRecordDataField class. + */ public static class Builder extends VectorStoreRecordField.Builder { private boolean isFilterable; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java index eeedfe73..876c96e9 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java @@ -30,10 +30,18 @@ public class VectorStoreRecordDefinition { private final List allFields; private final Map allFieldsMap; + /** + * Gets the key field in the record definition. + * @return VectorStoreRecordKeyField + */ public VectorStoreRecordKeyField getKeyField() { return keyField; } + /** + * Gets the data fields in the record definition. + * @return List of VectorStoreRecordDataField + */ public List getDataFields() { return dataFields; } @@ -212,6 +220,12 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) return checkFields(keyFields, dataFields, vectorFields); } + /** + * Validate that the record class contains only supported field types. + * @param fields The declared fields in the record class. + * @param supportedTypes The supported field types. + * @throws IllegalArgumentException if unsupported field types are found. + */ public static void validateSupportedTypes(List fields, Set> supportedTypes) { Set> unsupportedTypes = new HashSet<>(); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java index 9d85f2c9..0ba377af 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java @@ -19,6 +19,8 @@ public class VectorStoreRecordField { * Creates a new instance of the VectorStoreRecordField class. * * @param name the name of the field + * @param storageName the storage name of the field + * @param fieldType the field type */ public VectorStoreRecordField( @Nonnull String name, @@ -66,6 +68,11 @@ public Class getFieldType() { return fieldType; } + /** + * A builder for the VectorStoreRecordField class. + * @param the type of the field + * @param the type of the builder + */ public abstract static class Builder> implements SemanticKernelBuilder { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java index b255e561..9e2a70e7 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java @@ -5,6 +5,11 @@ * Represents the key field in a record. */ public class VectorStoreRecordKeyField extends VectorStoreRecordField { + + /** + * Create a builder for the VectorStoreRecordKeyField class. + * @return a new instance of the builder + */ public static Builder builder() { return new Builder(); } @@ -20,6 +25,9 @@ public VectorStoreRecordKeyField(String name, String storageName, Class type) super(name, storageName, type); } + /** + * A builder for the VectorStoreRecordKeyField class. + */ public static class Builder extends VectorStoreRecordField.Builder { @Override diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java index e5b94898..00b7627a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java @@ -12,6 +12,10 @@ public class VectorStoreRecordVectorField extends VectorStoreRecordField { private final IndexKind indexKind; private final DistanceFunction distanceFunction; + /** + * Create a builder for the VectorStoreRecordVectorField class. + * @return a new instance of the builder + */ public static Builder builder() { return new Builder(); } @@ -67,6 +71,9 @@ public DistanceFunction getDistanceFunction() { return distanceFunction; } + /** + * A builder for the VectorStoreRecordVectorField class. + */ public static class Builder extends VectorStoreRecordField.Builder { private int dimensions; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java index 444d4533..74d2b065 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java @@ -12,12 +12,21 @@ public class GetRecordOptions { private final boolean wildcardKeyMatching; + /** + * Creates a new instance of the GetRecordOptions class. + * @param includeVectors A value indicating whether to include vectors in a response. + */ public GetRecordOptions( boolean includeVectors) { this.includeVectors = includeVectors; this.wildcardKeyMatching = false; } + /** + * Creates a new instance of the GetRecordOptions class. + * @param includeVectors A value indicating whether to include vectors in a response. + * @param wildcardKeyMatching A value indicating whether to use wildcard key matching. + */ public GetRecordOptions( boolean includeVectors, boolean wildcardKeyMatching) { @@ -25,6 +34,10 @@ public GetRecordOptions( this.wildcardKeyMatching = wildcardKeyMatching; } + /** + * Gets whether to use wildcard key matching. + * @return {@code true} if wildcard key matching is used; otherwise, {@code false}. + */ public boolean isWildcardKeyMatching() { return wildcardKeyMatching; } @@ -38,6 +51,9 @@ public static Builder builder() { return new Builder(); } + /** + * A builder for GetRecordOptions. + */ public static class Builder implements SemanticKernelBuilder { private boolean includeVectors; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java index 28c0a6d7..35fe2f01 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java @@ -6,8 +6,14 @@ import javax.annotation.Nullable; +/** + * Options for a vector search. + */ public class VectorSearchOptions { + /** + * The default limit of the number of results to return. + */ public static final int DEFAULT_RESULT_LIMIT = 3; /** @@ -29,6 +35,14 @@ public static VectorSearchOptions createDefault(String vectorFieldName) { private final int offset; private final boolean includeVectors; + /** + * Creates a new instance of the VectorSearchOptions class. + * @param vectorSearchFilter The vector search filter. + * @param vectorFieldName The name of the vector field. + * @param limit The limit of the number of results to return. + * @param offset The offset of the results to return. + * @param includeVectors A value indicating whether to include vectors in the results. + */ public VectorSearchOptions(VectorSearchFilter vectorSearchFilter, String vectorFieldName, int limit, int offset, boolean includeVectors) { this.vectorSearchFilter = vectorSearchFilter; @@ -94,6 +108,9 @@ public static Builder builder() { return new Builder(); } + /** + * A builder for the VectorSearchOptions class. + */ public static class Builder implements SemanticKernelBuilder { private VectorSearchFilter vectorSearchFilter; private String vectorFieldName; @@ -101,32 +118,60 @@ public static class Builder implements SemanticKernelBuilder Date: Wed, 23 Oct 2024 00:58:29 -0700 Subject: [PATCH 081/277] Add Text Search API and VectorStoreTextSearch implementation --- .../JDBCVectorStoreRecordCollectionTest.java | 26 +- ...ashSetVectorStoreRecordCollectionTest.java | 18 +- ...isJsonVectorStoreRecordCollectionTest.java | 18 +- .../memory/InMemoryVolatileVectorStore.java | 34 +-- .../memory/VectorStoreWithAzureAISearch.java | 39 ++- .../memory/VectorStoreWithJDBC.java | 29 +-- .../memory/VectorStoreWithRedis.java | 30 +-- ...reAISearchVectorStoreRecordCollection.java | 18 +- .../jdbc/JDBCVectorStoreQueryProvider.java | 9 +- .../jdbc/JDBCVectorStoreRecordCollection.java | 3 +- .../jdbc/SQLVectorStoreQueryProvider.java | 3 +- .../PostgreSQLVectorStoreQueryProvider.java | 9 +- ...disHashSetVectorStoreRecordCollection.java | 7 +- .../RedisJsonVectorStoreRecordCollection.java | 12 +- ...disVectorStoreCollectionSearchMapping.java | 4 +- .../VolatileVectorStoreRecordCollection.java | 13 +- .../DefaultTextSearchResultMapper.java | 18 ++ .../DefaultTextSearchStringMapper.java | 19 ++ .../data/textsearch/KernelSearchResults.java | 70 +++++ .../data/textsearch/TextSearch.java | 36 +++ .../data/textsearch/TextSearchFilter.java | 79 ++++++ .../data/textsearch/TextSearchOptions.java | 150 +++++++++++ .../data/textsearch/TextSearchResult.java | 103 ++++++++ .../data/textsearch/TextSearchResultLink.java | 15 ++ .../textsearch/TextSearchResultMapper.java | 15 ++ .../data/textsearch/TextSearchResultName.java | 15 ++ .../textsearch/TextSearchResultValue.java | 15 ++ .../textsearch/TextSearchStringMapper.java | 15 ++ .../data/vectorsearch/VectorOperations.java | 4 +- .../vectorsearch/VectorSearchResults.java | 71 ++++++ .../vectorsearch/VectorizableTextSearch.java | 4 +- .../data/vectorsearch/VectorizedSearch.java | 4 +- .../vectorstorage/VectorStoreTextSearch.java | 239 ++++++++++++++++++ .../VectorStoreTextSearchOptions.java | 8 + .../annotations/VectorStoreRecordData.java | 2 +- .../annotations/VectorStoreRecordKey.java | 2 +- .../annotations/VectorStoreRecordVector.java | 2 +- .../options/VectorSearchOptions.java | 81 +++--- ...latileVectorStoreRecordCollectionTest.java | 15 +- 39 files changed, 1070 insertions(+), 184 deletions(-) create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreTextSearch.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreTextSearchOptions.java diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java index 8342f489..17b36756 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -416,12 +416,12 @@ public void exactSearch(QueryProvider provider, String embeddingName) { VectorSearchOptions options = VectorSearchOptions.builder() .withVectorFieldName(embeddingName) - .withLimit(3) + .withTop(3) .build(); // Embeddings similar to the third hotel List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, - options).block(); + options).block().getResults(); assertNotNull(results); assertEquals(3, results.size()); // The third hotel should be the most similar @@ -429,12 +429,12 @@ public void exactSearch(QueryProvider provider, String embeddingName) { options = VectorSearchOptions.builder() .withVectorFieldName(embeddingName) - .withOffset(1) - .withLimit(-100) + .withSkip(1) + .withTop(-100) .build(); // Skip the first result - results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, options).block(); + results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, options).block().getResults(); assertNotNull(results); assertEquals(1, results.size()); // The first hotel should be the most similar @@ -453,12 +453,12 @@ public void approximateSearch(QueryProvider provider) { VectorSearchOptions options = VectorSearchOptions.builder() .withVectorFieldName("indexedEuclidean") - .withLimit(5) + .withTop(5) .build(); // Embeddings similar to the third hotel List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, - options).block(); + options).block().getResults(); assertNotNull(results); assertEquals(5, results.size()); // The third hotel should be the most similar @@ -477,7 +477,7 @@ public void searchWithFilterEqualToFilter(QueryProvider provider, String embeddi VectorSearchOptions options = VectorSearchOptions.builder() .withVectorFieldName(embeddingName) - .withLimit(3) + .withTop(3) .withVectorSearchFilter( VectorSearchFilter.builder() .equalTo("rating", 4.0).build()) @@ -485,7 +485,7 @@ public void searchWithFilterEqualToFilter(QueryProvider provider, String embeddi // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, - options).block(); + options).block().getResults(); assertNotNull(results); assertEquals(3, results.size()); // The first hotel should be the most similar @@ -504,7 +504,7 @@ public void searchWithAnyTagEqualToFilter(QueryProvider provider, String embeddi VectorSearchOptions options = VectorSearchOptions.builder() .withVectorFieldName(embeddingName) - .withLimit(3) + .withTop(3) .withVectorSearchFilter( VectorSearchFilter.builder() .anyTagEqualTo("tags", "city").build()) @@ -512,7 +512,7 @@ public void searchWithAnyTagEqualToFilter(QueryProvider provider, String embeddi // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, - options).block(); + options).block().getResults(); assertNotNull(results); assertEquals(3, results.size()); // The first hotel should be the most similar @@ -530,7 +530,7 @@ public void postgresSearchIncludeAndNotIncludeVectors() { recordCollection.upsertBatchAsync(hotels, null).block(); List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, - null).block(); + null).block().getResults(); assertNotNull(results); assertEquals(3, results.size()); // The third hotel should be the most similar @@ -541,7 +541,7 @@ public void postgresSearchIncludeAndNotIncludeVectors() { .withIncludeVectors(true) .build(); - results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, options).block(); + results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, options).block().getResults(); assertNotNull(results); assertEquals(3, results.size()); // The third hotel should be the most similar diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java index a2b5e2c1..5256c629 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -387,9 +387,9 @@ public void search(RecordCollectionOptions options, String embeddingName) { .build(); // Embeddings similar to the third hotel - List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block().getResults(); assertNotNull(results); - assertEquals(VectorSearchOptions.DEFAULT_RESULT_LIMIT, results.size(), indexingFailureMessage); + assertEquals(VectorSearchOptions.DEFAULT_TOP, results.size(), indexingFailureMessage); // The third hotel should be the most similar assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId(), indexingFailureMessage); // Score should be different than zero @@ -412,9 +412,9 @@ public void searchWithVectors(RecordCollectionOptions options, String embeddingN .build(); // Embeddings similar to the third hotel - List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block().getResults(); assertNotNull(results); - assertEquals(VectorSearchOptions.DEFAULT_RESULT_LIMIT, results.size(), indexingFailureMessage); + assertEquals(VectorSearchOptions.DEFAULT_TOP, results.size(), indexingFailureMessage); // The third hotel should be the most similar assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId(), indexingFailureMessage); assertNotNull(results.get(0).getRecord().getEuclidean()); @@ -431,12 +431,12 @@ public void searchWithOffSet(RecordCollectionOptions options, String embeddingNa VectorSearchOptions searchOptions = VectorSearchOptions.builder() .withVectorFieldName(embeddingName) - .withOffset(1) - .withLimit(4) + .withSkip(1) + .withTop(4) .build(); // Embeddings similar to the third hotel - List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block().getResults(); assertNotNull(results); assertEquals(4, results.size(), indexingFailureMessage); // The first hotel should be the most similar @@ -454,7 +454,7 @@ public void searchWithFilterEqualToFilter(RecordCollectionOptions recordCollecti VectorSearchOptions options = VectorSearchOptions.builder() .withVectorFieldName(embeddingName) - .withLimit(3) + .withTop(3) .withVectorSearchFilter( VectorSearchFilter.builder() .equalTo("rating", 4.0).build()) @@ -462,7 +462,7 @@ public void searchWithFilterEqualToFilter(RecordCollectionOptions recordCollecti // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, - options).block(); + options).block().getResults(); assertNotNull(results); assertEquals(3, results.size()); // The first hotel should be the most similar diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java index 393690fd..7ab0bcc8 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -387,9 +387,9 @@ public void search(RecordCollectionOptions options, String embeddingName) { .build(); // Embeddings similar to the third hotel - List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block().getResults(); assertNotNull(results); - assertEquals(VectorSearchOptions.DEFAULT_RESULT_LIMIT, results.size(), indexingFailureMessage); + assertEquals(VectorSearchOptions.DEFAULT_TOP, results.size(), indexingFailureMessage); // The third hotel should be the most similar assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId(), indexingFailureMessage); // Score should be different than zero @@ -412,9 +412,9 @@ public void searchWithVectors(RecordCollectionOptions options, String embeddingN .build(); // Embeddings similar to the third hotel - List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block().getResults(); assertNotNull(results); - assertEquals(VectorSearchOptions.DEFAULT_RESULT_LIMIT, results.size(), indexingFailureMessage); + assertEquals(VectorSearchOptions.DEFAULT_TOP, results.size(), indexingFailureMessage); // The third hotel should be the most similar assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId(), indexingFailureMessage); assertNotNull(results.get(0).getRecord().getEuclidean()); @@ -431,12 +431,12 @@ public void searchWithOffSet(RecordCollectionOptions options, String embeddingNa VectorSearchOptions searchOptions = VectorSearchOptions.builder() .withVectorFieldName(embeddingName) - .withOffset(1) - .withLimit(4) + .withSkip(1) + .withTop(4) .build(); // Embeddings similar to the third hotel - List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block(); + List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block().getResults(); assertNotNull(results); assertEquals(4, results.size(), indexingFailureMessage); // The first hotel should be the most similar @@ -454,7 +454,7 @@ public void searchWithFilterEqualToFilter(RecordCollectionOptions recordCollecti VectorSearchOptions options = VectorSearchOptions.builder() .withVectorFieldName(embeddingName) - .withLimit(3) + .withTop(3) .withVectorSearchFilter( VectorSearchFilter.builder() .equalTo("rating", 4.0).build()) @@ -462,7 +462,7 @@ public void searchWithFilterEqualToFilter(RecordCollectionOptions recordCollecti // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, - options).block(); + options).block().getResults(); assertNotNull(results); assertEquals(3, results.size()); // The first hotel should be the most similar diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java index 8dc1372d..00166005 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java @@ -7,10 +7,13 @@ import com.azure.core.credential.KeyCredential; import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; +import com.microsoft.semantickernel.data.textsearch.TextSearchResultValue; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.VolatileVectorStore; import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreTextSearch; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreTextSearchOptions; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; @@ -38,12 +41,12 @@ public class InMemoryVolatileVectorStore { private static final int EMBEDDING_DIMENSIONS = 1536; static class GitHubFile { - @JsonProperty("fileId") // Set a different name for the storage field if needed - @VectorStoreRecordKey() + @VectorStoreRecordKey private final String id; - @VectorStoreRecordData() + @VectorStoreRecordData private final String description; @VectorStoreRecordData + @TextSearchResultValue private final String link; @VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List embedding; @@ -125,27 +128,24 @@ public static void inMemoryStoreAndSearch( .then(storeData(collection, embeddingGeneration, sampleData())) .block(); + // Build a vectorized search + var vectorStoreTextSearch = VectorStoreTextSearch.builder() + .withVectorizedSearch(collection) + .withTextEmbeddingGenerationService(embeddingGeneration) + .build(); + // Search for results // Volatile store executes an exhaustive search, for approximate search use Azure AI Search, Redis or JDBC with PostgreSQL - var results = search("How to get started", collection, embeddingGeneration).block(); + String query = "How to get started?"; + var results = vectorStoreTextSearch.searchAsync(query, null) + .block(); - if (results == null || results.isEmpty()) { + if (results == null || results.getTotalCount() == 0) { System.out.println("No search results found."); return; } - var searchResult = results.get(0); - System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", - searchResult.getScore(), searchResult.getRecord().link, - searchResult.getRecord().description); - } - private static Mono>> search( - String searchText, - VectorStoreRecordCollection recordCollection, - OpenAITextEmbeddingGenerationService embeddingGeneration) { - // Generate embeddings for the search text and search for the closest records - return embeddingGeneration.generateEmbeddingsAsync(Collections.singletonList(searchText)) - .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); + System.out.printf("Best result for '%s': %s%n", query, results.getResults().get(0)); } private static Mono> storeData( diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java index dca8599b..64575a02 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java @@ -15,8 +15,11 @@ import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStore; import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreOptions; import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.textsearch.TextSearchResultValue; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.VectorizedSearch; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreTextSearch; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; @@ -51,13 +54,12 @@ public class VectorStoreWithAzureAISearch { private static final int EMBEDDING_DIMENSIONS = 1536; static class GitHubFile { - - @JsonProperty("fileId") // Set a different name for the storage field if needed @VectorStoreRecordKey() private final String id; @VectorStoreRecordData() private final String description; @VectorStoreRecordData + @TextSearchResultValue private final String link; @VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_SIMILARITY) private final List embedding; @@ -111,7 +113,6 @@ public static void main(String[] args) { var searchClient = new SearchIndexClientBuilder() .endpoint(AZURE_AI_SEARCH_ENDPOINT) .credential(new AzureKeyCredential(AZURE_AISEARCH_KEY)) - .clientOptions(clientOptions()) .buildAsyncClient(); storeAndSearch(searchClient, embeddingGeneration); @@ -141,27 +142,24 @@ public static void storeAndSearch( .then(storeData(collection, embeddingGeneration, sampleData())) .block(); + // Build a vectorized search + var vectorStoreTextSearch = VectorStoreTextSearch.builder() + .withVectorizedSearch(collection) + .withTextEmbeddingGenerationService(embeddingGeneration) + .build(); + // Search for results // Might need to wait for the data to be indexed - var results = search("How to get started", collection, embeddingGeneration).block(); + String query = "How to get started?"; + var results = vectorStoreTextSearch.searchAsync(query, null) + .block(); - if (results == null || results.isEmpty()) { + if (results == null || results.getTotalCount() == 0) { System.out.println("No search results found."); return; } - var searchResult = results.get(0); - System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", - searchResult.getScore(), searchResult.getRecord().link, - searchResult.getRecord().description); - } - private static Mono>> search( - String searchText, - VectorStoreRecordCollection recordCollection, - OpenAITextEmbeddingGenerationService embeddingGeneration) { - // Generate embeddings for the search text and search for the closest records - return embeddingGeneration.generateEmbeddingAsync(searchText) - .flatMap(r -> recordCollection.searchAsync(r.getVector(), null)); + System.out.printf("Best result for '%s': %s%n", query, results.getResults().get(0)); } private static Mono> storeData( @@ -204,11 +202,4 @@ private static Map sampleData() { "README: README associated with a sample chat summary react-based webapp" }, }).collect(Collectors.toMap(element -> element[0], element -> element[1])); } - - private static ClientOptions clientOptions() { - return new ClientOptions() - .setTracingOptions(new TracingOptions()) - .setMetricsOptions(new MetricsOptions()) - .setApplicationId("Semantic-Kernel"); - } } diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java index 2099886b..1bf7ba30 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java @@ -10,8 +10,10 @@ import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.textsearch.TextSearchResultValue; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreTextSearch; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; @@ -41,12 +43,12 @@ public class VectorStoreWithJDBC { private static final int EMBEDDING_DIMENSIONS = 1536; static class GitHubFile { - @VectorStoreRecordKey() private final String id; @VectorStoreRecordData() private final String description; @VectorStoreRecordData + @TextSearchResultValue private final String link; @VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List embedding; @@ -154,26 +156,23 @@ public static void storeAndSearch(OpenAITextEmbeddingGenerationService embedding .then(storeData(collection, embeddingGeneration, sampleData())) .block(); + // Build a vectorized search + var vectorStoreTextSearch = VectorStoreTextSearch.builder() + .withVectorizedSearch(collection) + .withTextEmbeddingGenerationService(embeddingGeneration) + .build(); + // Search for results - var results = search("How to get started", collection, embeddingGeneration).block(); + String query = "How to get started?"; + var results = vectorStoreTextSearch.searchAsync(query, null) + .block(); - if (results == null || results.isEmpty()) { + if (results == null || results.getTotalCount() == 0) { System.out.println("No search results found."); return; } - var searchResult = results.get(0); - System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", - searchResult.getScore(), searchResult.getRecord().link, - searchResult.getRecord().description); - } - private static Mono>> search( - String searchText, - VectorStoreRecordCollection recordCollection, - OpenAITextEmbeddingGenerationService embeddingGeneration) { - // Generate embeddings for the search text and search for the closest records - return embeddingGeneration.generateEmbeddingsAsync(Collections.singletonList(searchText)) - .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); + System.out.printf("Best result for '%s': %s%n", query, results.getResults().get(0)); } private static Mono> storeData( diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java index 0310c9dd..63c03271 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java @@ -11,8 +11,10 @@ import com.microsoft.semantickernel.connectors.data.redis.RedisStorageType; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStore; import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreOptions; +import com.microsoft.semantickernel.data.textsearch.TextSearchResultValue; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreTextSearch; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; @@ -42,12 +44,12 @@ public class VectorStoreWithRedis { private static final int EMBEDDING_DIMENSIONS = 1536; public static class GitHubFile { - @JsonProperty("fileId") // Set a different name for the storage field if needed @VectorStoreRecordKey() private final String id; @VectorStoreRecordData() private final String description; @VectorStoreRecordData + @TextSearchResultValue private final String link; @VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, indexKind = IndexKind.HNSW, distanceFunction = DistanceFunction.COSINE_DISTANCE) private final List embedding; @@ -146,27 +148,23 @@ public static void storeAndSearch( .then(storeData(collection, embeddingGeneration, sampleData())) .block(); + // Build a vectorized search + var vectorStoreTextSearch = VectorStoreTextSearch.builder() + .withVectorizedSearch(collection) + .withTextEmbeddingGenerationService(embeddingGeneration) + .build(); + // Search for results - // Might need to wait for the data to be indexed - var results = search("How to get started", collection, embeddingGeneration).block(); + String query = "How to get started?"; + var results = vectorStoreTextSearch.searchAsync(query, null) + .block(); - if (results == null || results.isEmpty()) { + if (results == null || results.getTotalCount() == 0) { System.out.println("No search results found."); return; } - var searchResult = results.get(0); - System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", - searchResult.getScore(), searchResult.getRecord().link, - searchResult.getRecord().description); - } - private static Mono>> search( - String searchText, - VectorStoreRecordCollection recordCollection, - OpenAITextEmbeddingGenerationService embeddingGeneration) { - // Generate embeddings for the search text and search for the closest records - return embeddingGeneration.generateEmbeddingsAsync(Collections.singletonList(searchText)) - .flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null)); + System.out.printf("Best result for '%s': %s%n", query, results.getResults().get(0)); } private static Mono> storeData( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index 02971935..cbcbddfc 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -14,6 +14,7 @@ import com.azure.search.documents.models.VectorQuery; import com.azure.search.documents.models.VectorizableTextQuery; import com.azure.search.documents.models.VectorizedQuery; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorsearch.VectorizableTextSearch; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorizedSearch; @@ -287,7 +288,7 @@ public Mono deleteBatchAsync(List keys, DeleteRecordOptions option }).collect(Collectors.toList())).then(); } - private Mono>> searchAndMapAsync( + private Mono> searchAndMapAsync( List vectorQueries, VectorSearchOptions options, GetRecordOptions getRecordOptions) { @@ -296,8 +297,8 @@ private Mono>> searchAndMapAsync( SearchOptions searchOptions = new SearchOptions() .setFilter(filter) - .setTop(options.getLimit()) - .setSkip(options.getOffset()) + .setTop(options.getTop()) + .setSkip(options.getSkip()) .setScoringParameters() .setVectorSearchOptions(new com.azure.search.documents.models.VectorSearchOptions() .setQueries(vectorQueries)); @@ -323,7 +324,8 @@ record = response.getDocument(this.options.getRecordClass()); } return Mono.just(new VectorSearchResult<>(record, response.getScore())); - }).collectList(); + }).collectList().flatMap(results -> Mono.just( + new VectorSearchResults<>(results))); } /** @@ -334,7 +336,7 @@ record = response.getDocument(this.options.getRecordClass()); * @return A list of search results. */ @Override - public Mono>> searchAsync(String searchText, + public Mono> searchAsync(String searchText, VectorSearchOptions options) { if (firstVectorFieldName == null) { throw new SKException("No vector fields defined. Cannot perform vector search"); @@ -349,7 +351,7 @@ public Mono>> searchAsync(String searchText, .setFields(recordDefinition.getField(options.getVectorFieldName() != null ? options.getVectorFieldName() : firstVectorFieldName).getEffectiveStorageName()) - .setKNearestNeighborsCount(options.getLimit())); + .setKNearestNeighborsCount(options.getTop())); return searchAndMapAsync(vectorQueries, options, new GetRecordOptions(options.isIncludeVectors())); @@ -363,7 +365,7 @@ public Mono>> searchAsync(String searchText, * @return A list of search results. */ @Override - public Mono>> searchAsync(List vector, + public Mono> searchAsync(List vector, VectorSearchOptions options) { if (firstVectorFieldName == null) { throw new SKException("No vector fields defined. Cannot perform vector search"); @@ -378,7 +380,7 @@ public Mono>> searchAsync(List vector, .setFields(recordDefinition.getField(options.getVectorFieldName() != null ? options.getVectorFieldName() : firstVectorFieldName).getEffectiveStorageName()) - .setKNearestNeighborsCount(options.getLimit())); + .setKNearestNeighborsCount(options.getTop())); return searchAndMapAsync(vectorQueries, options, new GetRecordOptions(options.isIncludeVectors())); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index f33374aa..1994475c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -6,6 +6,7 @@ import com.microsoft.semantickernel.data.vectorsearch.VectorOperations; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; @@ -35,6 +36,7 @@ import javax.sql.DataSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import reactor.core.publisher.Mono; /** * A JDBC vector store query provider. @@ -526,7 +528,7 @@ protected List getRecordsWithFilter(String collectionName, * @return the search results */ @Override - public List> search(String collectionName, + public VectorSearchResults search(String collectionName, List vector, VectorSearchOptions options, VectorStoreRecordDefinition recordDefinition, VectorStoreRecordMapper mapper) { @@ -556,8 +558,9 @@ public List> search(String collectionName, ? DistanceFunction.EUCLIDEAN_DISTANCE : vectorField.getDistanceFunction(); - return VectorOperations.exactSimilaritySearch(records, vector, vectorField, - distanceFunction, options); + return new VectorSearchResults<>( + VectorOperations.exactSimilaritySearch(records, vector, vectorField, + distanceFunction, options)); } /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java index 31b25bef..28818a97 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -6,6 +6,7 @@ import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorsearch.VectorizedSearch; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; @@ -312,7 +313,7 @@ public Mono prepareAsync() { * @return A list of search results. */ @Override - public Mono>> searchAsync(List vector, + public Mono> searchAsync(List vector, VectorSearchOptions vectorSearchOptions) { return Mono.fromCallable( () -> queryProvider.search(this.collectionName, vector, vectorSearchOptions, diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java index 747e3c84..04de1949 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java @@ -4,6 +4,7 @@ import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.filter.FilterClause; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; @@ -146,7 +147,7 @@ void deleteRecords(String collectionName, List keys, * @param mapper the mapper, responsible for mapping the result set to the record type. * @return the search results */ - List> search(String collectionName, + VectorSearchResults search(String collectionName, List vector, VectorSearchOptions options, VectorStoreRecordDefinition recordDefinition, diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index 6be72981..200edafa 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -10,6 +10,7 @@ import com.microsoft.semantickernel.data.filter.EqualToFilterClause; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; @@ -334,7 +335,7 @@ public void upsertRecords(String collectionName, List records, * @return the search results */ @Override - public List> search(String collectionName, + public VectorSearchResults search(String collectionName, List vector, VectorSearchOptions options, VectorStoreRecordDefinition recordDefinition, VectorStoreRecordMapper mapper) { @@ -391,8 +392,8 @@ public List> search(String collectionName, for (Object parameter : parameters) { statement.setObject(parameterIndex++, parameter); } - statement.setInt(parameterIndex++, options.getLimit()); - statement.setInt(parameterIndex, options.getOffset()); + statement.setInt(parameterIndex++, options.getTop()); + statement.setInt(parameterIndex, options.getSkip()); List> records = new ArrayList<>(); ResultSet resultSet = statement.executeQuery(); @@ -404,7 +405,7 @@ public List> search(String collectionName, resultSet.getDouble("score"))); } - return records; + return new VectorSearchResults<>(records); } catch (SQLException | JsonProcessingException e) { throw new SKException("Failed to search records", e); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java index b65fe9f9..47c15a8e 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorsearch.VectorizedSearch; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; @@ -391,7 +392,7 @@ public Mono deleteBatchAsync(List strings, DeleteRecordOptions opt * @return A list of search results. */ @Override - public Mono>> searchAsync(List vector, + public Mono> searchAsync(List vector, VectorSearchOptions options) { if (recordDefinition.getVectorFields().isEmpty()) { return Mono @@ -406,7 +407,7 @@ public Mono>> searchAsync(List vector, SearchResult searchResult = client.ftSearch(collectionName, ftSearchParams.getLeft(), ftSearchParams.getRight()); - return searchResult.getDocuments().stream() + List> results = searchResult.getDocuments().stream() .map(doc -> { String key = removeKeyPrefixIfNeeded(doc.getId(), collectionName); double score = 0; @@ -436,6 +437,8 @@ else if (entry.getKey().equals( return new VectorSearchResult<>(record, score); }) .collect(Collectors.toList()); + + return new VectorSearchResults<>(results); }).subscribeOn(Schedulers.boundedElastic())); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java index 4af97846..2a747684 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorsearch.VectorizedSearch; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; @@ -405,10 +406,10 @@ public Mono deleteBatchAsync(List strings, DeleteRecordOptions opt * * @param vector The vector to search with. * @param options The options to use for the search. - * @return A list of search results. + * @return The search results. */ @Override - public Mono>> searchAsync(List vector, + public Mono> searchAsync(List vector, VectorSearchOptions options) { if (recordDefinition.getVectorFields().isEmpty()) { return Mono @@ -422,7 +423,7 @@ public Mono>> searchAsync(List vector, SearchResult searchResult = client.ftSearch(collectionName, ftSearchParams.getLeft(), ftSearchParams.getRight()); - return searchResult.getDocuments().stream() + List> results = searchResult.getDocuments().stream() .map(doc -> { Map properties = new HashMap<>(); for (Map.Entry entry : doc.getProperties()) { @@ -445,8 +446,9 @@ public Mono>> searchAsync(List vector, } catch (JsonProcessingException e) { throw new RuntimeException(e); } - }) - .collect(Collectors.toList()); + }).collect(Collectors.toList()); + + return new VectorSearchResults<>(results); }).subscribeOn(Schedulers.boundedElastic())); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java index de51b8e7..2064411a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java @@ -64,9 +64,9 @@ public Pair buildQuery(List vector, vectorField.getEffectiveStorageName(), VECTOR_SCORE_FIELD); FTSearchParams searchParams = new FTSearchParams() - .addParam("K", options.getLimit() + options.getOffset()) + .addParam("K", options.getTop() + options.getSkip()) .addParam("BLOB", convertListToByteArray(vector)) - .limit(options.getOffset(), options.getLimit()) + .limit(options.getSkip(), options.getTop()) .sortBy(VECTOR_SCORE_FIELD, SortingOrder.ASC) .dialect(2); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java index 7d6031d4..5be45985 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java @@ -1,14 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.microsoft.semantickernel.data.filter.FilterClause; import com.microsoft.semantickernel.data.vectorsearch.VectorOperations; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; @@ -21,7 +19,6 @@ import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -269,7 +266,7 @@ private List arrayNodeToFloatList(ArrayNode arrayNode) { * @return A list of search results. */ @Override - public Mono>> searchAsync(List vector, + public Mono> searchAsync(List vector, final VectorSearchOptions options) { if (recordDefinition.getVectorFields().isEmpty()) { throw new SKException("No vector fields defined. Cannot perform vector search"); @@ -295,8 +292,10 @@ public Mono>> searchAsync(List vector, List records = VolatileVectorStoreCollectionSearchMapping.filterRecords( new ArrayList<>(getCollection().values()), effectiveOptions.getVectorSearchFilter(), recordDefinition, objectMapper); - return VectorOperations.exactSimilaritySearch(records, vector, vectorField, - distanceFunction, effectiveOptions); + + return new VectorSearchResults<>( + VectorOperations.exactSimilaritySearch(records, vector, vectorField, + distanceFunction, effectiveOptions)); }).subscribeOn(Schedulers.boundedElastic()); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java new file mode 100644 index 00000000..652d2474 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.textsearch; + +/** + * Default implementation of {@link TextSearchResultMapper}. + */ +public class DefaultTextSearchResultMapper implements TextSearchResultMapper { + /** + * Maps a search result to a {@link TextSearchResult}. + * + * @param result The search result. + * @return The {@link TextSearchResult}. + */ + @Override + public TextSearchResult fromResultToTextSearchResult(Object result) { + return TextSearchResult.fromRecord(result); + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java new file mode 100644 index 00000000..5d9c0545 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.textsearch; + +/** + * Default implementation of {@link TextSearchStringMapper}. + */ +public class DefaultTextSearchStringMapper implements TextSearchStringMapper { + /** + * Maps a search result to a string. + * + * @param result The search result. + * @return The string. + */ + @Override + public String fromResultToString(Object result) { + TextSearchResult textSearchResult = TextSearchResult.fromRecord(result); + return textSearchResult.getValue(); + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java new file mode 100644 index 00000000..23343977 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.textsearch; + +import javax.annotation.Nullable; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * The search results. + * + * @param The type of the search results. + */ +public class KernelSearchResults { + private final List results; + private final long totalCount; + private final Map metadata; + + /** + * Creates a new instance of the KernelSearchResults class. + * + * @param results The search results. + */ + public KernelSearchResults(List results) { + this(results, results.size(), Collections.emptyMap()); + } + + /** + * Creates a new instance of the KernelSearchResults class. + * + * @param results The search results. + * @param totalCount The total count of search results. + * @param metadata The metadata. + */ + public KernelSearchResults(List results, long totalCount, + Map metadata) { + this.results = Collections.unmodifiableList(results); + this.totalCount = totalCount; + this.metadata = Collections.unmodifiableMap(metadata); + } + + /** + * Gets the total count of search results. + * This value represents the total number of results that are available for the current query and not the number of results being returned. + * + * @return The total count of search results. + */ + public long getTotalCount() { + return totalCount; + } + + /** + * Gets the search results. + * + * @return The search results. + */ + public List getResults() { + return results; + } + + /** + * Gets the metadata associated with the search results. + * + * @return The metadata. + */ + public Map getMetadata() { + return metadata; + } + +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java new file mode 100644 index 00000000..04ca24ad --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.textsearch; + +import reactor.core.publisher.Mono; + +public interface TextSearch { + + /** + * Perform a search for content related to the specified query and return String values representing the search results. + * + * @param query The text to search for. + * @param options The search options. + * @return The search results. + */ + Mono> searchAsync(String query, TextSearchOptions options); + + /** + * Perform a search for content related to the specified query and return TextSearchResult values representing the search results. + * + * @param query The text to search for. + * @param options The search options. + * @return The search results. + */ + Mono> getTextSearchResultsAsync(String query, + TextSearchOptions options); + + /** + * Perform a search for content related to the specified query and return Object values representing the search results. + * + * @param query The text to search for. + * @param options The search options. + * @return The search results. + */ + Mono> getSearchResultsAsync(String query, + TextSearchOptions options); +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java new file mode 100644 index 00000000..8e448448 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.textsearch; + +import com.microsoft.semantickernel.data.filter.EqualToFilterClause; +import com.microsoft.semantickernel.data.filter.FilterClause; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Represents a text search filter. + */ +public class TextSearchFilter { + + private final List filterClauses; + + /** + * Creates a new instance of the TextSearchFilter class. + */ + public TextSearchFilter() { + this(Collections.emptyList()); + } + + /** + * Creates a new instance of the TextSearchFilter class. + * + * @param filterClauses The filter clauses. + */ + public TextSearchFilter(List filterClauses) { + this.filterClauses = Collections.unmodifiableList(filterClauses); + } + + /** + * Gets the filter clauses. + * + * @return The filter clauses. + */ + public List getFilterClauses() { + return filterClauses; + } + + /** + * Creates a new instance of the {@link Builder} class. + * + * @return The builder. + */ + public static Builder builder() { + return new Builder(); + } + + /** + * The builder for the {@link TextSearchFilter} class. + */ + public static class Builder { + private final List filterClauses = new ArrayList<>(); + + /** + * Adds an EqualToFilterClause to the filter. + * + * @param fieldName The field name to filter on. + * @param value The value. + * @return The builder. + */ + public Builder equalTo(String fieldName, Object value) { + filterClauses.add(new EqualToFilterClause(fieldName, value)); + return this; + } + + /** + * Builds the filter. + * + * @return The filter. + */ + public TextSearchFilter build() { + return new TextSearchFilter(filterClauses); + } + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java new file mode 100644 index 00000000..81688581 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java @@ -0,0 +1,150 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.textsearch; + +/** + * Represents the options for a text search. + */ +public class TextSearchOptions { + + /** + * The default number of search results to return. + */ + public static final int DEFAULT_TOP = 3; + private final boolean includeTotalCount; + private final int top; + private final int skip; + private final TextSearchFilter filter; + + /** + * Creates a new instance of the TextSearchOptions class with default values. + * + * @return A new instance of the TextSearchOptions class with default values. + */ + public static TextSearchOptions createDefault() { + return new TextSearchOptions(false, DEFAULT_TOP, 0, null); + } + + /** + * Creates a new instance of the TextSearchOptions class. + * + * @param includeTotalCount A value indicating whether to include the total count of search results. + * @param top The limit of the number of results to return. + * @param skip The offset of the results to return. + * @param filter The search filter. + */ + TextSearchOptions(boolean includeTotalCount, int top, int skip, TextSearchFilter filter) { + this.includeTotalCount = includeTotalCount; + this.top = top; + this.skip = skip; + this.filter = filter; + } + + /** + * Gets a value indicating whether to include the total count of search results. + * + * @return A value indicating whether to include the total count of search results. + */ + public boolean isIncludeTotalCount() { + return includeTotalCount; + } + + /** + * Gets the limit of the number of results to return. + * + * @return The limit of the number of results to return. + */ + public int getTop() { + return top; + } + + /** + * Gets the offset of the results to return. + * + * @return The offset of the results to return. + */ + public int getSkip() { + return skip; + } + + /** + * Gets the search filter. + * + * @return The search filter. + */ + public TextSearchFilter getFilter() { + return filter; + } + + /** + * Creates a new instance of the {@link Builder} class. + * + * @return The builder. + */ + public static Builder builder() { + return new Builder(); + } + + /** + * The builder for the {@link TextSearchOptions} class. + */ + public static class Builder { + private boolean includeTotalCount = false; + private int top = DEFAULT_TOP; + private int skip = 0; + private TextSearchFilter filter; + + /** + * Sets a value indicating whether to include the total count of search results. + * + * @param includeTotalCount A value indicating whether to include the total count of search results. + * @return The builder. + */ + public Builder withIncludeTotalCount(boolean includeTotalCount) { + this.includeTotalCount = includeTotalCount; + return this; + } + + /** + * Sets the limit of the number of results to return. + * + * @param top The limit of the number of results to return. + * @return The builder. + */ + public Builder withTop(int top) { + this.top = top; + return this; + } + + /** + * Sets the offset of the results to return. + * + * @param skip The offset of the results to return. + * @return The builder. + */ + public Builder withSkip(int skip) { + this.skip = skip; + return this; + } + + /** + * Sets the search filter. + * + * @param filter The search filter. + * @return The builder. + */ + public Builder withFilter(TextSearchFilter filter) { + this.filter = filter; + return this; + } + + /** + * Builds a new instance of the {@link TextSearchOptions} class. + * + * @return A new instance of the TextSearchOptions class. + */ + public TextSearchOptions build() { + return new TextSearchOptions(includeTotalCount, top, skip, filter); + } + } + +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java new file mode 100644 index 00000000..2a6221f3 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.textsearch; + +import com.microsoft.semantickernel.exceptions.SKException; + +import java.lang.reflect.Field; + +/** + * Represents a text search result. + */ +public class TextSearchResult { + private final String name; + private final String value; + private final String link; + + /** + * Creates a new instance of the TextSearchResult class. + * + * @param name The name of the search result. + * @param value The value of the search result. + * @param link The link of the search result. + */ + TextSearchResult(String name, String value, String link) { + this.name = name; + this.value = value; + this.link = link; + } + + /** + * Gets the name of the search result. + * + * @return The name of the search result. + */ + public String getName() { + return name; + } + + /** + * Gets the value of the search result. + * + * @return The value of the search result. + */ + public String getValue() { + return value; + } + + /** + * Gets the link of the search result. + * + * @return The link of the search result. + */ + public String getLink() { + return link; + } + + /** + * Creates a new instance of the {@link TextSearchResult} class from a record. + * The record should have fields annotated with {@link TextSearchResultName}, {@link TextSearchResultValue}, and {@link TextSearchResultLink}. + * + * @param record The record. + * @return The TextSearchResult. + */ + public static TextSearchResult fromRecord(Object record) { + String name = null, value = null, link = null; + + try { + for (Field field : record.getClass().getDeclaredFields()) { + if (field.isAnnotationPresent(TextSearchResultName.class)) { + if (name != null) { + throw new SKException("Multiple fields with @TextSearchResultName found"); + } + + field.setAccessible(true); + name = (String) field.get(record); + } + if (field.isAnnotationPresent(TextSearchResultValue.class)) { + if (value != null) { + throw new SKException("Multiple fields with @TextSearchResultValue found"); + } + + field.setAccessible(true); + value = (String) field.get(record); + } + if (field.isAnnotationPresent(TextSearchResultLink.class)) { + if (link != null) { + throw new SKException("Multiple fields with @TextSearchResultLink found"); + } + + field.setAccessible(true); + link = (String) field.get(record); + } + } + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + + if (value == null) { + throw new SKException("No field with @TextSearchResultValue found"); + } + + return new TextSearchResult(name, value, link); + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java new file mode 100644 index 00000000..fab7624e --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.textsearch; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation to mark a property on a record class as the link to the source data. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.FIELD) +public @interface TextSearchResultLink { +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java new file mode 100644 index 00000000..2fcbe37e --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.textsearch; + +/** + * Maps a search result to a {@link TextSearchResult}. + */ +public interface TextSearchResultMapper { + /** + * Maps a search result to a {@link TextSearchResult}. + * + * @param result The search result. + * @return The {@link TextSearchResult}. + */ + TextSearchResult fromResultToTextSearchResult(Object result); +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java new file mode 100644 index 00000000..77a3751e --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.textsearch; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation to mark a property on a record class as the name of the source data. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.FIELD) +public @interface TextSearchResultName { +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java new file mode 100644 index 00000000..6b3dd568 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.textsearch; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation to mark a property on a record class as the value of the source data. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.FIELD) +public @interface TextSearchResultValue { +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java new file mode 100644 index 00000000..c3afa430 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.textsearch; + +/** + * Maps a search result to a string. + */ +public interface TextSearchStringMapper { + /** + * Maps a search result to a string. + * + * @param result The search result. + * @return The string. + */ + String fromResultToString(Object result); +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java index 4f90faa8..cbc1e314 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java @@ -234,8 +234,8 @@ public static List> exactSimilaritySearch( return results.stream() .sorted(comparator) - .skip(options.getOffset()) - .limit(options.getLimit()) + .skip(options.getSkip()) + .limit(options.getTop()) .collect(Collectors.toList()); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java new file mode 100644 index 00000000..a74dbf23 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorsearch; + +import javax.annotation.Nullable; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * The search results. + * + * @param The type of the search results. + */ +public class VectorSearchResults { + + private final long totalCount; + private final List> results; + private final Map metadata; + + /** + * Creates a new instance of the VectorSearchResults class. + * + * @param results The search results. + */ + public VectorSearchResults(List> results) { + this(results, results.size(), Collections.emptyMap()); + } + + /** + * Creates a new instance of the VectorSearchResults class. + * + * @param results The search results. + * @param totalCount The total count of search results. + * @param metadata The metadata. + */ + public VectorSearchResults(List> results, long totalCount, + Map metadata) { + this.results = Collections.unmodifiableList(results); + this.totalCount = totalCount; + this.metadata = Collections.unmodifiableMap(metadata); + } + + /** + * Gets the total count of search results. + * This value represents the total number of results that are available for the current query and not the number of results being returned. + * + * @return The total count of search results. + */ + public long getTotalCount() { + return totalCount; + } + + /** + * Gets the search results. + * + * @return The search results. + */ + public List> getResults() { + return results; + } + + /** + * Gets the metadata associated with the search results. + * + * @return The metadata. + */ + public Map getMetadata() { + return metadata; + } + +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java index 00046444..93aa883d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java @@ -17,8 +17,8 @@ public interface VectorizableTextSearch { * * @param searchText The text to search with. * @param options The options to use for the search. - * @return A list of search results. + * @return VectorSearchResults. */ - Mono>> searchAsync(String searchText, + Mono> searchAsync(String searchText, VectorSearchOptions options); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java index 59010730..a8b61b1a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java @@ -18,8 +18,8 @@ public interface VectorizedSearch { * * @param vector The vector to search with. * @param options The options to use for the search. - * @return A list of search results. + * @return Vector search results. */ - Mono>> searchAsync(List vector, + Mono> searchAsync(List vector, VectorSearchOptions options); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreTextSearch.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreTextSearch.java new file mode 100644 index 00000000..46684f5a --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreTextSearch.java @@ -0,0 +1,239 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorstorage; + +import com.microsoft.semantickernel.data.textsearch.DefaultTextSearchResultMapper; +import com.microsoft.semantickernel.data.textsearch.DefaultTextSearchStringMapper; +import com.microsoft.semantickernel.data.textsearch.KernelSearchResults; +import com.microsoft.semantickernel.data.textsearch.TextSearch; +import com.microsoft.semantickernel.data.textsearch.TextSearchOptions; +import com.microsoft.semantickernel.data.textsearch.TextSearchResult; +import com.microsoft.semantickernel.data.textsearch.TextSearchResultMapper; +import com.microsoft.semantickernel.data.textsearch.TextSearchStringMapper; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; +import com.microsoft.semantickernel.data.vectorsearch.VectorizedSearch; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import com.microsoft.semantickernel.exceptions.SKException; +import com.microsoft.semantickernel.services.textembedding.TextEmbeddingGenerationService; +import reactor.core.publisher.Mono; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.stream.Collectors; + +/** + * A text search implementation that uses a vector record collection to perform the search. + * + * @param The record type. + */ +public class VectorStoreTextSearch implements TextSearch { + + private final VectorizedSearch vectorizedSearch; + private final TextEmbeddingGenerationService textEmbeddingGenerationService; + private final TextSearchStringMapper stringMapper; + private final TextSearchResultMapper resultMapper; + private final VectorStoreTextSearchOptions options; + + /** + * Create a new instance of VectorStoreTextSearch with {@link VectorizedSearch} to perform vectorized search and + * {@link TextEmbeddingGenerationService} to generate text embeddings. + * + * @param vectorizedSearch The vectorized search. Usually a vector record collection. + * @param textEmbeddingGenerationService The text embedding generation service. + * @param stringMapper The string mapper. + * @param resultMapper The result mapper. + * @param options The options. + */ + public VectorStoreTextSearch( + @Nonnull VectorizedSearch vectorizedSearch, + @Nonnull TextEmbeddingGenerationService textEmbeddingGenerationService, + @Nullable TextSearchStringMapper stringMapper, + @Nullable TextSearchResultMapper resultMapper, + @Nullable VectorStoreTextSearchOptions options) { + this.vectorizedSearch = vectorizedSearch; + this.textEmbeddingGenerationService = textEmbeddingGenerationService; + this.stringMapper = stringMapper == null ? new DefaultTextSearchStringMapper() + : stringMapper; + this.resultMapper = resultMapper == null ? new DefaultTextSearchResultMapper() + : resultMapper; + this.options = options == null ? new VectorStoreTextSearchOptions() : options; + } + + private Mono> executeSearchAsync(String query, + TextSearchOptions options) { + if (options == null) { + options = TextSearchOptions.createDefault(); + } + + VectorSearchOptions vectorSearchOptions = VectorSearchOptions.builder() + .withVectorSearchFilter(options.getFilter() != null + ? new VectorSearchFilter(options.getFilter().getFilterClauses()) + : null) + .withTop(options.getTop()) + .withSkip(options.getSkip()) + .withIncludeTotalCount(options.isIncludeTotalCount()) + .build(); + + return textEmbeddingGenerationService.generateEmbeddingAsync(query) + .flatMap(embedding -> vectorizedSearch.searchAsync(embedding.getVector(), + vectorSearchOptions)); + } + + /** + * Perform a search for content related to the specified query and return String values representing the search results. + * + * @param query The text to search for. + * @param options The search options. + * @return The search results. + */ + @Override + public Mono> searchAsync(String query, TextSearchOptions options) { + return executeSearchAsync(query, options) + .map(results -> new KernelSearchResults<>( + results.getResults().stream() + .map(r -> stringMapper.fromResultToString(r.getRecord())) + .collect(Collectors.toList()), + results.getTotalCount(), + results.getMetadata())); + } + + /** + * Perform a search for content related to the specified query and return TextSearchResult values representing the search results. + * + * @param query The text to search for. + * @param options The search options. + * @return The search results. + */ + @Override + public Mono> getTextSearchResultsAsync(String query, + TextSearchOptions options) { + return executeSearchAsync(query, options) + .map(results -> new KernelSearchResults<>( + results.getResults().stream() + .map(r -> resultMapper.fromResultToTextSearchResult(r.getRecord())) + .collect(Collectors.toList()), + results.getTotalCount(), + results.getMetadata())); + } + + /** + * Perform a search for content related to the specified query and return Object values representing the search results. + * + * @param query The text to search for. + * @param options The search options. + * @return The search results. + */ + @Override + public Mono> getSearchResultsAsync(String query, + TextSearchOptions options) { + return executeSearchAsync(query, options) + .map(results -> new KernelSearchResults<>( + results.getResults().stream() + .map(r -> resultMapper.fromResultToTextSearchResult(r.getRecord())) + .collect(Collectors.toList()), + results.getTotalCount(), + results.getMetadata())); + } + + /** + * Create a new instance of {@link Builder}. + * + * @param The record type. + * @return The builder. + */ + public static Builder builder() { + return new Builder<>(); + } + + /** + * A builder for the {@link VectorStoreTextSearch} class. + * + * @param The record type. + */ + public static class Builder { + @Nullable + private VectorizedSearch vectorizedSearch; + @Nullable + private TextEmbeddingGenerationService textEmbeddingGenerationService; + @Nullable + private TextSearchStringMapper stringMapper; + @Nullable + private TextSearchResultMapper resultMapper; + @Nullable + private VectorStoreTextSearchOptions options; + + /** + * Sets the vectorized search. + * + * @param vectorizedSearch The vectorized search. + * @return The builder. + */ + public Builder withVectorizedSearch(VectorizedSearch vectorizedSearch) { + this.vectorizedSearch = vectorizedSearch; + return this; + } + + /** + * Sets the text embedding generation service. + * + * @param textEmbeddingGenerationService The text embedding generation service. + * @return The builder. + */ + public Builder withTextEmbeddingGenerationService( + TextEmbeddingGenerationService textEmbeddingGenerationService) { + this.textEmbeddingGenerationService = textEmbeddingGenerationService; + return this; + } + + /** + * Sets the string mapper. + * + * @param stringMapper The string mapper. + * @return The builder. + */ + public Builder withStringMapper(TextSearchStringMapper stringMapper) { + this.stringMapper = stringMapper; + return this; + } + + /** + * Sets the result mapper. + * + * @param resultMapper The result mapper. + * @return The builder. + */ + public Builder withResultMapper(TextSearchResultMapper resultMapper) { + this.resultMapper = resultMapper; + return this; + } + + /** + * Sets the options. + * + * @param options The options. + * @return The builder. + */ + public Builder withOptions(VectorStoreTextSearchOptions options) { + this.options = options; + return this; + } + + /** + * Builds the {@link VectorStoreTextSearch} instance. + * + * @return The {@link VectorStoreTextSearch} instance. + */ + public VectorStoreTextSearch build() { + if (vectorizedSearch == null) { + throw new SKException("Vectorized search is required"); + } + if (textEmbeddingGenerationService == null) { + throw new SKException("Text embedding generation service is required"); + } + + return new VectorStoreTextSearch<>(vectorizedSearch, textEmbeddingGenerationService, + stringMapper, resultMapper, options); + } + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreTextSearchOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreTextSearchOptions.java new file mode 100644 index 00000000..2f0a73e9 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreTextSearchOptions.java @@ -0,0 +1,8 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.vectorstorage; + +/** + * Options for vector store text search. + */ +public class VectorStoreTextSearchOptions { +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java index 532cffdb..4043af26 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java @@ -7,7 +7,7 @@ import java.lang.annotation.Target; /** - * Represents a data attribute in a record. + * Represents a data field in a record. */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.FIELD) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java index 8f8c98a7..108b6a0c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java @@ -7,7 +7,7 @@ import java.lang.annotation.Target; /** - * Represents the key attribute in a record. + * Represents the key field in a record. */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.FIELD) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java index 90128b41..dfce1352 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java @@ -11,7 +11,7 @@ import java.lang.annotation.Target; /** - * Represents a vector attribute in a record. + * Represents a vector field in a record. */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.FIELD) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java index 35fe2f01..241f069f 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java @@ -14,7 +14,7 @@ public class VectorSearchOptions { /** * The default limit of the number of results to return. */ - public static final int DEFAULT_RESULT_LIMIT = 3; + public static final int DEFAULT_TOP = 3; /** * Creates a new instance of the VectorSearchOptions class with default values. @@ -31,25 +31,28 @@ public static VectorSearchOptions createDefault(String vectorFieldName) { private final VectorSearchFilter vectorSearchFilter; @Nullable private final String vectorFieldName; - private final int limit; - private final int offset; + private final int top; + private final int skip; private final boolean includeVectors; + private final boolean includeTotalCount; /** * Creates a new instance of the VectorSearchOptions class. * @param vectorSearchFilter The vector search filter. * @param vectorFieldName The name of the vector field. - * @param limit The limit of the number of results to return. - * @param offset The offset of the results to return. + * @param top The limit of the number of results to return. + * @param skip The offset of the results to return. * @param includeVectors A value indicating whether to include vectors in the results. */ public VectorSearchOptions(VectorSearchFilter vectorSearchFilter, - String vectorFieldName, int limit, int offset, boolean includeVectors) { + String vectorFieldName, int top, int skip, boolean includeVectors, + boolean includeTotalCount) { this.vectorSearchFilter = vectorSearchFilter; this.vectorFieldName = vectorFieldName; - this.limit = Math.max(1, limit); - this.offset = Math.max(0, offset); + this.top = Math.max(1, top); + this.skip = Math.max(0, skip); this.includeVectors = includeVectors; + this.includeTotalCount = includeTotalCount; } /** @@ -77,8 +80,8 @@ public String getVectorFieldName() { * * @return The limit of the number of results to return. */ - public int getLimit() { - return limit; + public int getTop() { + return top; } /** @@ -86,8 +89,8 @@ public int getLimit() { * * @return The offset of the results to return. */ - public int getOffset() { - return offset; + public int getSkip() { + return skip; } /** @@ -99,6 +102,15 @@ public boolean isIncludeVectors() { return includeVectors; } + /** + * Gets a value indicating whether to include the total count of the results. + * + * @return A value indicating whether to include the total count of the results. + */ + public boolean isIncludeTotalCount() { + return includeTotalCount; + } + /** * Creates a new instance of the Builder class. * @@ -114,17 +126,10 @@ public static Builder builder() { public static class Builder implements SemanticKernelBuilder { private VectorSearchFilter vectorSearchFilter; private String vectorFieldName; - private int limit; - private int offset; - private boolean includeVectors; - - /** - * Creates a new instance of the Builder class with default values. - */ - public Builder() { - this.limit = DEFAULT_RESULT_LIMIT; - this.includeVectors = false; - } + private int top = DEFAULT_TOP; + private int skip = 0; + private boolean includeVectors = false; + private boolean includeTotalCount = false; /** * Sets the vector search filter. @@ -149,21 +154,21 @@ public Builder withVectorFieldName(String vectorFieldName) { /** * Sets the limit of the number of results to return. - * @param limit the limit of the number of results to return + * @param top the limit of the number of results to return * @return {@code this} builder */ - public Builder withLimit(int limit) { - this.limit = limit; + public Builder withTop(int top) { + this.top = top; return this; } /** * Sets the offset of the results to return. - * @param offset the offset of the results to return + * @param skip the offset of the results to return * @return {@code this} builder */ - public Builder withOffset(int offset) { - this.offset = offset; + public Builder withSkip(int skip) { + this.skip = skip; return this; } @@ -177,10 +182,24 @@ public Builder withIncludeVectors(boolean includeVectors) { return this; } + /** + * Sets a value indicating whether to include the total count of the results. + * @param includeTotalCount a value indicating whether to include the total count of the results + * @return {@code this} builder + */ + public Builder withIncludeTotalCount(boolean includeTotalCount) { + this.includeTotalCount = includeTotalCount; + return this; + } + + /** + * Builds a new instance of the VectorSearchOptions class. + * @return a new instance of the VectorSearchOptions class + */ @Override public VectorSearchOptions build() { - return new VectorSearchOptions(vectorSearchFilter, vectorFieldName, limit, offset, - includeVectors); + return new VectorSearchOptions(vectorSearchFilter, vectorFieldName, top, skip, + includeVectors, includeTotalCount); } } } diff --git a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java index 23b1ac78..bd2c633d 100644 --- a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java +++ b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java @@ -10,7 +10,6 @@ import java.util.List; import java.util.stream.Collectors; -import com.microsoft.semantickernel.data.filter.EqualToFilterClause; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; @@ -158,12 +157,12 @@ public void exactSearch(DistanceFunction distanceFunction) { VectorSearchOptions options = VectorSearchOptions.builder() .withVectorFieldName(distanceFunction.getValue()) - .withLimit(3) + .withTop(3) .build(); // Embeddings similar to the third hotel List> results = recordCollection - .searchAsync(SEARCH_EMBEDDINGS, options).block(); + .searchAsync(SEARCH_EMBEDDINGS, options).block().getResults(); assertNotNull(results); assertEquals(3, results.size()); // The third hotel should be the most similar @@ -171,12 +170,12 @@ public void exactSearch(DistanceFunction distanceFunction) { options = VectorSearchOptions.builder() .withVectorFieldName(distanceFunction.getValue()) - .withOffset(1) - .withLimit(-100) + .withSkip(1) + .withTop(-100) .build(); // Skip the first result - results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, options).block(); + results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, options).block().getResults(); assertNotNull(results); assertEquals(1, results.size()); // The first hotel should be the most similar @@ -191,7 +190,7 @@ public void searchWithFilter(DistanceFunction distanceFunction) { VectorSearchOptions options = VectorSearchOptions.builder() .withVectorFieldName(distanceFunction.getValue()) - .withLimit(3) + .withTop(3) .withVectorSearchFilter( VectorSearchFilter.builder() .equalTo("rating", 4.0).build()) @@ -199,7 +198,7 @@ public void searchWithFilter(DistanceFunction distanceFunction) { // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned List> results = recordCollection - .searchAsync(SEARCH_EMBEDDINGS, options).block(); + .searchAsync(SEARCH_EMBEDDINGS, options).block().getResults(); assertNotNull(results); assertEquals(3, results.size()); // The first hotel should be the most similar From 2beb1654843d35cee1cd61eeb8c21272c58fc0d2 Mon Sep 17 00:00:00 2001 From: Martijn Verburg Date: Thu, 24 Oct 2024 12:28:44 +1300 Subject: [PATCH 082/277] update properties and plugins, mvn clean install site runs OK --- .gitignore | 1 + .mvn/wrapper/maven-wrapper.properties | 5 +- mvnw | 435 ++++++++++++-------------- mvnw.cmd | 304 ++++++++---------- pom.xml | 62 ++-- semantickernel-api/pom.xml | 1 - semantickernel-bom/pom.xml | 13 +- semantickernel-experimental/pom.xml | 1 - 8 files changed, 360 insertions(+), 462 deletions(-) diff --git a/.gitignore b/.gitignore index e6f26223..44c24fc0 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,4 @@ __pycache__/ __pypackages__/ .pdm.toml global.json +.java-version diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties index 6d3a5665..d58dfb70 100644 --- a/.mvn/wrapper/maven-wrapper.properties +++ b/.mvn/wrapper/maven-wrapper.properties @@ -14,5 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.3/apache-maven-3.9.3-bin.zip -wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar +wrapperVersion=3.3.2 +distributionType=only-script +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip diff --git a/mvnw b/mvnw index 8d937f4c..19529ddf 100755 --- a/mvnw +++ b/mvnw @@ -19,290 +19,241 @@ # ---------------------------------------------------------------------------- # ---------------------------------------------------------------------------- -# Apache Maven Wrapper startup batch script, version 3.2.0 -# -# Required ENV vars: -# ------------------ -# JAVA_HOME - location of a JDK home dir +# Apache Maven Wrapper startup batch script, version 3.3.2 # # Optional ENV vars # ----------------- -# MAVEN_OPTS - parameters passed to the Java VM when running Maven -# e.g. to debug Maven itself, use -# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 -# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# JAVA_HOME - location of a JDK home dir, required when download maven via java source +# MVNW_REPOURL - repo url base for downloading maven distribution +# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output # ---------------------------------------------------------------------------- -if [ -z "$MAVEN_SKIP_RC" ] ; then - - if [ -f /usr/local/etc/mavenrc ] ; then - . /usr/local/etc/mavenrc - fi - - if [ -f /etc/mavenrc ] ; then - . /etc/mavenrc - fi - - if [ -f "$HOME/.mavenrc" ] ; then - . "$HOME/.mavenrc" - fi - -fi +set -euf +[ "${MVNW_VERBOSE-}" != debug ] || set -x -# OS specific support. $var _must_ be set to either true or false. -cygwin=false; -darwin=false; -mingw=false +# OS specific support. +native_path() { printf %s\\n "$1"; } case "$(uname)" in - CYGWIN*) cygwin=true ;; - MINGW*) mingw=true;; - Darwin*) darwin=true - # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home - # See https://developer.apple.com/library/mac/qa/qa1170/_index.html - if [ -z "$JAVA_HOME" ]; then - if [ -x "/usr/libexec/java_home" ]; then - JAVA_HOME="$(/usr/libexec/java_home)"; export JAVA_HOME - else - JAVA_HOME="/Library/Java/Home"; export JAVA_HOME - fi - fi - ;; +CYGWIN* | MINGW*) + [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")" + native_path() { cygpath --path --windows "$1"; } + ;; esac -if [ -z "$JAVA_HOME" ] ; then - if [ -r /etc/gentoo-release ] ; then - JAVA_HOME=$(java-config --jre-home) - fi -fi - -# For Cygwin, ensure paths are in UNIX format before anything is touched -if $cygwin ; then - [ -n "$JAVA_HOME" ] && - JAVA_HOME=$(cygpath --unix "$JAVA_HOME") - [ -n "$CLASSPATH" ] && - CLASSPATH=$(cygpath --path --unix "$CLASSPATH") -fi - -# For Mingw, ensure paths are in UNIX format before anything is touched -if $mingw ; then - [ -n "$JAVA_HOME" ] && [ -d "$JAVA_HOME" ] && - JAVA_HOME="$(cd "$JAVA_HOME" || (echo "cannot cd into $JAVA_HOME."; exit 1); pwd)" -fi - -if [ -z "$JAVA_HOME" ]; then - javaExecutable="$(which javac)" - if [ -n "$javaExecutable" ] && ! [ "$(expr "\"$javaExecutable\"" : '\([^ ]*\)')" = "no" ]; then - # readlink(1) is not available as standard on Solaris 10. - readLink=$(which readlink) - if [ ! "$(expr "$readLink" : '\([^ ]*\)')" = "no" ]; then - if $darwin ; then - javaHome="$(dirname "\"$javaExecutable\"")" - javaExecutable="$(cd "\"$javaHome\"" && pwd -P)/javac" - else - javaExecutable="$(readlink -f "\"$javaExecutable\"")" - fi - javaHome="$(dirname "\"$javaExecutable\"")" - javaHome=$(expr "$javaHome" : '\(.*\)/bin') - JAVA_HOME="$javaHome" - export JAVA_HOME - fi - fi -fi - -if [ -z "$JAVACMD" ] ; then - if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then +# set JAVACMD and JAVACCMD +set_java_home() { + # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched + if [ -n "${JAVA_HOME-}" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then # IBM's JDK on AIX uses strange locations for the executables JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACCMD="$JAVA_HOME/jre/sh/javac" else JAVACMD="$JAVA_HOME/bin/java" + JAVACCMD="$JAVA_HOME/bin/javac" + + if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then + echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2 + echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2 + return 1 + fi fi else - JAVACMD="$(\unset -f command 2>/dev/null; \command -v java)" - fi -fi - -if [ ! -x "$JAVACMD" ] ; then - echo "Error: JAVA_HOME is not defined correctly." >&2 - echo " We cannot execute $JAVACMD" >&2 - exit 1 -fi + JAVACMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v java + )" || : + JAVACCMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v javac + )" || : -if [ -z "$JAVA_HOME" ] ; then - echo "Warning: JAVA_HOME environment variable is not set." -fi - -# traverses directory structure from process work directory to filesystem root -# first directory with .mvn subdirectory is considered project base directory -find_maven_basedir() { - if [ -z "$1" ] - then - echo "Path not specified to find_maven_basedir" - return 1 + if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then + echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2 + return 1 + fi fi +} - basedir="$1" - wdir="$1" - while [ "$wdir" != '/' ] ; do - if [ -d "$wdir"/.mvn ] ; then - basedir=$wdir - break - fi - # workaround for JBEAP-8937 (on Solaris 10/Sparc) - if [ -d "${wdir}" ]; then - wdir=$(cd "$wdir/.." || exit 1; pwd) - fi - # end of workaround +# hash string like Java String::hashCode +hash_string() { + str="${1:-}" h=0 + while [ -n "$str" ]; do + char="${str%"${str#?}"}" + h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296)) + str="${str#?}" done - printf '%s' "$(cd "$basedir" || exit 1; pwd)" + printf %x\\n $h } -# concatenates all lines of a file -concat_lines() { - if [ -f "$1" ]; then - # Remove \r in case we run on Windows within Git Bash - # and check out the repository with auto CRLF management - # enabled. Otherwise, we may read lines that are delimited with - # \r\n and produce $'-Xarg\r' rather than -Xarg due to word - # splitting rules. - tr -s '\r\n' ' ' < "$1" - fi +verbose() { :; } +[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; } + +die() { + printf %s\\n "$1" >&2 + exit 1 } -log() { - if [ "$MVNW_VERBOSE" = true ]; then - printf '%s\n' "$1" - fi +trim() { + # MWRAPPER-139: + # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds. + # Needed for removing poorly interpreted newline sequences when running in more + # exotic environments such as mingw bash on Windows. + printf "%s" "${1}" | tr -d '[:space:]' +} + +# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties +while IFS="=" read -r key value; do + case "${key-}" in + distributionUrl) distributionUrl=$(trim "${value-}") ;; + distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;; + esac +done <"${0%/*}/.mvn/wrapper/maven-wrapper.properties" +[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in ${0%/*}/.mvn/wrapper/maven-wrapper.properties" + +case "${distributionUrl##*/}" in +maven-mvnd-*bin.*) + MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ + case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in + *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;; + :Darwin*x86_64) distributionPlatform=darwin-amd64 ;; + :Darwin*arm64) distributionPlatform=darwin-aarch64 ;; + :Linux*x86_64*) distributionPlatform=linux-amd64 ;; + *) + echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2 + distributionPlatform=linux-amd64 + ;; + esac + distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip" + ;; +maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;; +*) MVN_CMD="mvn${0##*/mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;; +esac + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}" +distributionUrlName="${distributionUrl##*/}" +distributionUrlNameMain="${distributionUrlName%.*}" +distributionUrlNameMain="${distributionUrlNameMain%-bin}" +MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}" +MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")" + +exec_maven() { + unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || : + exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD" } -BASE_DIR=$(find_maven_basedir "$(dirname "$0")") -if [ -z "$BASE_DIR" ]; then - exit 1; +if [ -d "$MAVEN_HOME" ]; then + verbose "found existing MAVEN_HOME at $MAVEN_HOME" + exec_maven "$@" fi -MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"}; export MAVEN_PROJECTBASEDIR -log "$MAVEN_PROJECTBASEDIR" +case "${distributionUrl-}" in +*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;; +*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;; +esac -########################################################################################## -# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central -# This allows using the maven wrapper in projects that prohibit checking in binary data. -########################################################################################## -wrapperJarPath="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" -if [ -r "$wrapperJarPath" ]; then - log "Found $wrapperJarPath" +# prepare tmp dir +if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then + clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; } + trap clean HUP INT TERM EXIT else - log "Couldn't find $wrapperJarPath, downloading it ..." + die "cannot create temp dir" +fi - if [ -n "$MVNW_REPOURL" ]; then - wrapperUrl="$MVNW_REPOURL/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" - else - wrapperUrl="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" - fi - while IFS="=" read -r key value; do - # Remove '\r' from value to allow usage on windows as IFS does not consider '\r' as a separator ( considers space, tab, new line ('\n'), and custom '=' ) - safeValue=$(echo "$value" | tr -d '\r') - case "$key" in (wrapperUrl) wrapperUrl="$safeValue"; break ;; - esac - done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" - log "Downloading from: $wrapperUrl" +mkdir -p -- "${MAVEN_HOME%/*}" - if $cygwin; then - wrapperJarPath=$(cygpath --path --windows "$wrapperJarPath") - fi +# Download and Install Apache Maven +verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +verbose "Downloading from: $distributionUrl" +verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" - if command -v wget > /dev/null; then - log "Found wget ... using wget" - [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--quiet" - if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then - wget $QUIET "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" - else - wget $QUIET --http-user="$MVNW_USERNAME" --http-password="$MVNW_PASSWORD" "$wrapperUrl" -O "$wrapperJarPath" || rm -f "$wrapperJarPath" - fi - elif command -v curl > /dev/null; then - log "Found curl ... using curl" - [ "$MVNW_VERBOSE" = true ] && QUIET="" || QUIET="--silent" - if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then - curl $QUIET -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" - else - curl $QUIET --user "$MVNW_USERNAME:$MVNW_PASSWORD" -o "$wrapperJarPath" "$wrapperUrl" -f -L || rm -f "$wrapperJarPath" - fi - else - log "Falling back to using Java to download" - javaSource="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.java" - javaClass="$MAVEN_PROJECTBASEDIR/.mvn/wrapper/MavenWrapperDownloader.class" - # For Cygwin, switch paths to Windows format before running javac - if $cygwin; then - javaSource=$(cygpath --path --windows "$javaSource") - javaClass=$(cygpath --path --windows "$javaClass") - fi - if [ -e "$javaSource" ]; then - if [ ! -e "$javaClass" ]; then - log " - Compiling MavenWrapperDownloader.java ..." - ("$JAVA_HOME/bin/javac" "$javaSource") - fi - if [ -e "$javaClass" ]; then - log " - Running MavenWrapperDownloader.java ..." - ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$wrapperUrl" "$wrapperJarPath") || rm -f "$wrapperJarPath" - fi - fi - fi +# select .zip or .tar.gz +if ! command -v unzip >/dev/null; then + distributionUrl="${distributionUrl%.zip}.tar.gz" + distributionUrlName="${distributionUrl##*/}" fi -########################################################################################## -# End of extension -########################################################################################## -# If specified, validate the SHA-256 sum of the Maven wrapper jar file -wrapperSha256Sum="" -while IFS="=" read -r key value; do - case "$key" in (wrapperSha256Sum) wrapperSha256Sum=$value; break ;; - esac -done < "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.properties" -if [ -n "$wrapperSha256Sum" ]; then - wrapperSha256Result=false - if command -v sha256sum > /dev/null; then - if echo "$wrapperSha256Sum $wrapperJarPath" | sha256sum -c > /dev/null 2>&1; then - wrapperSha256Result=true +# verbose opt +__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR='' +[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v + +# normalize http auth +case "${MVNW_PASSWORD:+has-password}" in +'') MVNW_USERNAME='' MVNW_PASSWORD='' ;; +has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;; +esac + +if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then + verbose "Found wget ... using wget" + wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl" +elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then + verbose "Found curl ... using curl" + curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl" +elif set_java_home; then + verbose "Falling back to use Java to download" + javaSource="$TMP_DOWNLOAD_DIR/Downloader.java" + targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName" + cat >"$javaSource" <<-END + public class Downloader extends java.net.Authenticator + { + protected java.net.PasswordAuthentication getPasswordAuthentication() + { + return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() ); + } + public static void main( String[] args ) throws Exception + { + setDefault( new Downloader() ); + java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() ); + } + } + END + # For Cygwin/MinGW, switch paths to Windows format before running javac and java + verbose " - Compiling Downloader.java ..." + "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java" + verbose " - Running Downloader.java ..." + "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")" +fi + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +if [ -n "${distributionSha256Sum-}" ]; then + distributionSha256Result=false + if [ "$MVN_CMD" = mvnd.sh ]; then + echo "Checksum validation is not supported for maven-mvnd." >&2 + echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + elif command -v sha256sum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c >/dev/null 2>&1; then + distributionSha256Result=true fi - elif command -v shasum > /dev/null; then - if echo "$wrapperSha256Sum $wrapperJarPath" | shasum -a 256 -c > /dev/null 2>&1; then - wrapperSha256Result=true + elif command -v shasum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then + distributionSha256Result=true fi else - echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." - echo "Please install either command, or disable validation by removing 'wrapperSha256Sum' from your maven-wrapper.properties." + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2 + echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 exit 1 fi - if [ $wrapperSha256Result = false ]; then - echo "Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised." >&2 - echo "Investigate or delete $wrapperJarPath to attempt a clean download." >&2 - echo "If you updated your Maven version, you need to update the specified wrapperSha256Sum property." >&2 + if [ $distributionSha256Result = false ]; then + echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2 + echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2 exit 1 fi fi -MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" - -# For Cygwin, switch paths to Windows format before running java -if $cygwin; then - [ -n "$JAVA_HOME" ] && - JAVA_HOME=$(cygpath --path --windows "$JAVA_HOME") - [ -n "$CLASSPATH" ] && - CLASSPATH=$(cygpath --path --windows "$CLASSPATH") - [ -n "$MAVEN_PROJECTBASEDIR" ] && - MAVEN_PROJECTBASEDIR=$(cygpath --path --windows "$MAVEN_PROJECTBASEDIR") +# unzip and move +if command -v unzip >/dev/null; then + unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip" +else + tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar" fi +printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/mvnw.url" +mv -- "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME" -# Provide a "standardized" way to retrieve the CLI args that will -# work with both Windows and non-Windows executions. -MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $*" -export MAVEN_CMD_LINE_ARGS - -WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain - -# shellcheck disable=SC2086 # safe args -exec "$JAVACMD" \ - $MAVEN_OPTS \ - $MAVEN_DEBUG_OPTS \ - -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ - "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ - ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" +clean || : +exec_maven "$@" diff --git a/mvnw.cmd b/mvnw.cmd index c4586b56..249bdf38 100644 --- a/mvnw.cmd +++ b/mvnw.cmd @@ -1,3 +1,4 @@ +<# : batch portion @REM ---------------------------------------------------------------------------- @REM Licensed to the Apache Software Foundation (ASF) under one @REM or more contributor license agreements. See the NOTICE file @@ -18,188 +19,131 @@ @REM ---------------------------------------------------------------------------- @REM ---------------------------------------------------------------------------- -@REM Apache Maven Wrapper startup batch script, version 3.2.0 -@REM -@REM Required ENV vars: -@REM JAVA_HOME - location of a JDK home dir +@REM Apache Maven Wrapper startup batch script, version 3.3.2 @REM @REM Optional ENV vars -@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands -@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending -@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven -@REM e.g. to debug Maven itself, use -@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 -@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM MVNW_REPOURL - repo url base for downloading maven distribution +@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output @REM ---------------------------------------------------------------------------- -@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' -@echo off -@REM set title of command window -title %0 -@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' -@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% - -@REM set %HOME% to equivalent of $HOME -if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") - -@REM Execute a user defined script before this one -if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre -@REM check for pre script, once with legacy .bat ending and once with .cmd ending -if exist "%USERPROFILE%\mavenrc_pre.bat" call "%USERPROFILE%\mavenrc_pre.bat" %* -if exist "%USERPROFILE%\mavenrc_pre.cmd" call "%USERPROFILE%\mavenrc_pre.cmd" %* -:skipRcPre - -@setlocal - -set ERROR_CODE=0 - -@REM To isolate internal variables from possible post scripts, we use another setlocal -@setlocal - -@REM ==== START VALIDATION ==== -if not "%JAVA_HOME%" == "" goto OkJHome - -echo. -echo Error: JAVA_HOME not found in your environment. >&2 -echo Please set the JAVA_HOME variable in your environment to match the >&2 -echo location of your Java installation. >&2 -echo. -goto error - -:OkJHome -if exist "%JAVA_HOME%\bin\java.exe" goto init - -echo. -echo Error: JAVA_HOME is set to an invalid directory. >&2 -echo JAVA_HOME = "%JAVA_HOME%" >&2 -echo Please set the JAVA_HOME variable in your environment to match the >&2 -echo location of your Java installation. >&2 -echo. -goto error - -@REM ==== END VALIDATION ==== - -:init - -@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". -@REM Fallback to current working directory if not found. - -set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% -IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir - -set EXEC_DIR=%CD% -set WDIR=%EXEC_DIR% -:findBaseDir -IF EXIST "%WDIR%"\.mvn goto baseDirFound -cd .. -IF "%WDIR%"=="%CD%" goto baseDirNotFound -set WDIR=%CD% -goto findBaseDir - -:baseDirFound -set MAVEN_PROJECTBASEDIR=%WDIR% -cd "%EXEC_DIR%" -goto endDetectBaseDir - -:baseDirNotFound -set MAVEN_PROJECTBASEDIR=%EXEC_DIR% -cd "%EXEC_DIR%" - -:endDetectBaseDir - -IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig - -@setlocal EnableExtensions EnableDelayedExpansion -for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a -@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% - -:endReadAdditionalConfig - -SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" -set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" -set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain - -set WRAPPER_URL="https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" - -FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( - IF "%%A"=="wrapperUrl" SET WRAPPER_URL=%%B -) - -@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central -@REM This allows using the maven wrapper in projects that prohibit checking in binary data. -if exist %WRAPPER_JAR% ( - if "%MVNW_VERBOSE%" == "true" ( - echo Found %WRAPPER_JAR% - ) -) else ( - if not "%MVNW_REPOURL%" == "" ( - SET WRAPPER_URL="%MVNW_REPOURL%/org/apache/maven/wrapper/maven-wrapper/3.2.0/maven-wrapper-3.2.0.jar" - ) - if "%MVNW_VERBOSE%" == "true" ( - echo Couldn't find %WRAPPER_JAR%, downloading it ... - echo Downloading from: %WRAPPER_URL% - ) - - powershell -Command "&{"^ - "$webclient = new-object System.Net.WebClient;"^ - "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ - "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ - "}"^ - "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%WRAPPER_URL%', '%WRAPPER_JAR%')"^ - "}" - if "%MVNW_VERBOSE%" == "true" ( - echo Finished downloading %WRAPPER_JAR% - ) -) -@REM End of extension - -@REM If specified, validate the SHA-256 sum of the Maven wrapper jar file -SET WRAPPER_SHA_256_SUM="" -FOR /F "usebackq tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( - IF "%%A"=="wrapperSha256Sum" SET WRAPPER_SHA_256_SUM=%%B +@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0) +@SET __MVNW_CMD__= +@SET __MVNW_ERROR__= +@SET __MVNW_PSMODULEP_SAVE=%PSModulePath% +@SET PSModulePath= +@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @( + IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B) ) -IF NOT %WRAPPER_SHA_256_SUM%=="" ( - powershell -Command "&{"^ - "$hash = (Get-FileHash \"%WRAPPER_JAR%\" -Algorithm SHA256).Hash.ToLower();"^ - "If('%WRAPPER_SHA_256_SUM%' -ne $hash){"^ - " Write-Output 'Error: Failed to validate Maven wrapper SHA-256, your Maven wrapper might be compromised.';"^ - " Write-Output 'Investigate or delete %WRAPPER_JAR% to attempt a clean download.';"^ - " Write-Output 'If you updated your Maven version, you need to update the specified wrapperSha256Sum property.';"^ - " exit 1;"^ - "}"^ - "}" - if ERRORLEVEL 1 goto error -) - -@REM Provide a "standardized" way to retrieve the CLI args that will -@REM work with both Windows and non-Windows executions. -set MAVEN_CMD_LINE_ARGS=%* - -%MAVEN_JAVA_EXE% ^ - %JVM_CONFIG_MAVEN_PROPS% ^ - %MAVEN_OPTS% ^ - %MAVEN_DEBUG_OPTS% ^ - -classpath %WRAPPER_JAR% ^ - "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" ^ - %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* -if ERRORLEVEL 1 goto error -goto end - -:error -set ERROR_CODE=1 - -:end -@endlocal & set ERROR_CODE=%ERROR_CODE% - -if not "%MAVEN_SKIP_RC%"=="" goto skipRcPost -@REM check for post script, once with legacy .bat ending and once with .cmd ending -if exist "%USERPROFILE%\mavenrc_post.bat" call "%USERPROFILE%\mavenrc_post.bat" -if exist "%USERPROFILE%\mavenrc_post.cmd" call "%USERPROFILE%\mavenrc_post.cmd" -:skipRcPost - -@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' -if "%MAVEN_BATCH_PAUSE%"=="on" pause - -if "%MAVEN_TERMINATE_CMD%"=="on" exit %ERROR_CODE% - -cmd /C exit /B %ERROR_CODE% +@SET PSModulePath=%__MVNW_PSMODULEP_SAVE% +@SET __MVNW_PSMODULEP_SAVE= +@SET __MVNW_ARG0_NAME__= +@SET MVNW_USERNAME= +@SET MVNW_PASSWORD= +@IF NOT "%__MVNW_CMD__%"=="" (%__MVNW_CMD__% %*) +@echo Cannot start maven from wrapper >&2 && exit /b 1 +@GOTO :EOF +: end batch / begin powershell #> + +$ErrorActionPreference = "Stop" +if ($env:MVNW_VERBOSE -eq "true") { + $VerbosePreference = "Continue" +} + +# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties +$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl +if (!$distributionUrl) { + Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" +} + +switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) { + "maven-mvnd-*" { + $USE_MVND = $true + $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip" + $MVN_CMD = "mvnd.cmd" + break + } + default { + $USE_MVND = $false + $MVN_CMD = $script -replace '^mvnw','mvn' + break + } +} + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +if ($env:MVNW_REPOURL) { + $MVNW_REPO_PATTERN = if ($USE_MVND) { "/org/apache/maven/" } else { "/maven/mvnd/" } + $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace '^.*'+$MVNW_REPO_PATTERN,'')" +} +$distributionUrlName = $distributionUrl -replace '^.*/','' +$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$','' +$MAVEN_HOME_PARENT = "$HOME/.m2/wrapper/dists/$distributionUrlNameMain" +if ($env:MAVEN_USER_HOME) { + $MAVEN_HOME_PARENT = "$env:MAVEN_USER_HOME/wrapper/dists/$distributionUrlNameMain" +} +$MAVEN_HOME_NAME = ([System.Security.Cryptography.MD5]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join '' +$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME" + +if (Test-Path -Path "$MAVEN_HOME" -PathType Container) { + Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME" + Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" + exit $? +} + +if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) { + Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl" +} + +# prepare tmp dir +$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile +$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir" +$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null +trap { + if ($TMP_DOWNLOAD_DIR.Exists) { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } + } +} + +New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null + +# Download and Install Apache Maven +Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +Write-Verbose "Downloading from: $distributionUrl" +Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +$webclient = New-Object System.Net.WebClient +if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) { + $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD) +} +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum +if ($distributionSha256Sum) { + if ($USE_MVND) { + Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." + } + Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash + if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) { + Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property." + } +} + +# unzip and move +Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null +Rename-Item -Path "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" -NewName $MAVEN_HOME_NAME | Out-Null +try { + Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null +} catch { + if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) { + Write-Error "fail to move MAVEN_HOME" + } +} finally { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } +} + +Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" diff --git a/pom.xml b/pom.xml index 991f9187..9a886b34 100644 --- a/pom.xml +++ b/pom.xml @@ -14,64 +14,64 @@ Parent pom for the Semantic Kernel Project + 1.0.0-beta.11 - 10.13.0 + 10.18.2 0.10.21 + false 2.19.1 1.17.0 1.6.0 - 5.10.2 - 2.22.1 + 5.11.3 + 2.24.1 3.1.0 2.12.1 - 3.3.1 - 3.3.2 - 3.12.1 + 3.5.0 + 3.4.0 + 3.13.0 8 8 8 - 3.6.1 - 3.1.1 + 3.8.0 + 3.1.3 1.0 - 3.4.1 - 3.1.1 - 3.1.1 - 0.8.11 - 3.3.0 - 3.6.3 - 3.3.2 + 3.5.0 + 3.5.0 + 3.1.3 + 0.8.12 + 3.4.2 + 3.10.1 + 3.5.0 2.4.0 + 3.21.2 - 3.5.0 + 3.8.0 0.16.1 - 3.0.1 + 3.1.1 3.3.1 - 4.0.0-M13 - 3.5.1 - 3.3.0 - 4.8.3.1 + 4.0.0-M16 + 3.6.0 + 3.3.1 + 4.8.6.5 2.38.0 - 3.2.5 - 3.9.3 - 2.16.2 - 3.7.0 - 5.10.0 + 3.5.1 + 3.9.9 + 2.17.1 + 5.14.2 0.9.1 + 6.55.0 UTF-8 microsoft/semantic-kernel git@github.com:${project.github.repository}.git - 4.8.3 - false + 4.8.6 semantickernel-bom - semantickernel-api semantickernel-experimental - aiservices/openai aiservices/google aiservices/huggingface @@ -391,7 +391,7 @@ org.codehaus.mojo animal-sniffer-maven-plugin - 1.23 + 1.24 android diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index 6f63e152..82c7d252 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -93,7 +93,6 @@ org.apache.maven.plugins maven-surefire-plugin - 3.2.5 false 1 diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index dd284fe5..ffb61468 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -21,7 +21,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.3.0 + 3.5.0 enforce-maven @@ -31,14 +31,20 @@ - 3.9.3 + 3.9.9 + + org.apache.maven.plugins + maven-project-info-reports-plugin + 3.8.0 + + @@ -175,7 +181,6 @@ provided - com.github.spotbugs spotbugs-annotations @@ -193,8 +198,6 @@ 1.1.0 - - com.github.victools jsonschema-generator diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index c7b890ad..951d557f 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -125,7 +125,6 @@ org.apache.maven.plugins maven-surefire-plugin - 3.2.5 false 1 From b5e8fbddd2138762c2b0ff5b322fa30d459b1c86 Mon Sep 17 00:00:00 2001 From: Martijn Verburg Date: Thu, 24 Oct 2024 14:44:54 +1300 Subject: [PATCH 083/277] Updated dependnecies all tests pass --- aiservices/google/pom.xml | 2 +- pom.xml | 4 ++-- .../KernelFunctionYamlTest.java | 1 + semantickernel-bom/pom.xml | 22 ++++++++++--------- semantickernel-experimental/pom.xml | 4 ++-- .../options/VectorSearchOptions.java | 1 + 6 files changed, 19 insertions(+), 15 deletions(-) diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index 67c44fd2..e0c334d1 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -17,7 +17,7 @@ com.google.cloud libraries-bom - 26.37.0 + 26.49.0 pom import diff --git a/pom.xml b/pom.xml index 9a886b34..1c179fdf 100644 --- a/pom.xml +++ b/pom.xml @@ -152,13 +152,13 @@ org.wiremock wiremock - 3.9.1 + 3.9.2 test org.mockito mockito-junit-jupiter - 5.12.0 + 5.14.2 test diff --git a/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionYamlTest.java b/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionYamlTest.java index 7f89d4db..eb71ffec 100644 --- a/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionYamlTest.java +++ b/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionYamlTest.java @@ -97,6 +97,7 @@ public static void assertEqualsJson(Object a, Object b) { /** * Test of fromYaml method, of class KernelFunctionYaml. + * @throws Exception if parsing the YAML throws an Exception */ @Test @Disabled diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index ffb61468..1dce5d04 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -13,7 +13,7 @@ https://www.github.com/microsoft/semantic-kernel - 2.17.1 + 2.18.0 @@ -52,7 +52,7 @@ io.opentelemetry opentelemetry-bom - 1.39.0 + 1.43.0 pom import @@ -100,22 +100,23 @@ com.azure azure-ai-openai + 1.0.0-beta.11 com.azure azure-identity - 1.12.2 + 1.14.0 com.azure azure-core - 1.49.1 + 1.53.0 com.azure azure-search-documents - 11.7.0-beta.3 + 11.8.0-beta.1 com.azure @@ -126,7 +127,7 @@ redis.clients jedis - 5.2.0-beta5 + 5.2.0 @@ -152,6 +153,7 @@ ${com.fasterxml.jackson.core.version} runtime + com.github.jknack handlebars @@ -165,7 +167,7 @@ org.slf4j slf4j-api - 2.0.12 + 2.0.16 com.google.code.findbugs @@ -184,18 +186,18 @@ com.github.spotbugs spotbugs-annotations - 4.8.3 + 4.8.6 org.apache.commons commons-text - 1.11.0 + 1.12.0 com.google.cloud google-cloud-vertexai - 1.1.0 + 1.12.0 diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 951d557f..4c93f575 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -110,12 +110,12 @@ org.postgresql postgresql - 42.7.3 + 42.7.4 org.xerial sqlite-jdbc - 3.46.1.0 + 3.47.0.0 diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java index 241f069f..53ef59dd 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java @@ -43,6 +43,7 @@ public static VectorSearchOptions createDefault(String vectorFieldName) { * @param top The limit of the number of results to return. * @param skip The offset of the results to return. * @param includeVectors A value indicating whether to include vectors in the results. + * @param includeTotalCount A value indicating whether to include the total count of the results. */ public VectorSearchOptions(VectorSearchFilter vectorSearchFilter, String vectorFieldName, int top, int skip, boolean includeVectors, From 599caa8919de920a2e88d3dfcbe1e825fe7a99bc Mon Sep 17 00:00:00 2001 From: Martijn Verburg Date: Thu, 24 Oct 2024 14:54:26 +1300 Subject: [PATCH 084/277] Excliude spotbugs false positive --- .../semantickernel/services/textembedding/Embedding.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java index dcdce1c6..e98941f6 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.services.textembedding; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.annotation.Nonnull; import java.util.ArrayList; import java.util.Collections; @@ -8,6 +9,7 @@ import java.util.Objects; /** Represents a strongly typed vector of numeric data. */ +@SuppressFBWarnings("SING_SINGLETON_HAS_NONPRIVATE_CONSTRUCTOR") // This class is not a singleton public class Embedding { // vector is immutable! From e6e3ac9a9f01695d1fb38e03257595a9fbc45d44 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Fri, 25 Oct 2024 01:07:16 -0700 Subject: [PATCH 085/277] Move vector stores out of experimental Remove old connectors --- api-test/integration-tests/pom.xml | 7 +- .../memory => data}/jdbc/Hotel.java | 2 +- .../JDBCVectorStoreRecordCollectionTest.java | 16 +- .../jdbc/JDBCVectorStoreTest.java | 26 +- .../memory => data}/redis/Hotel.java | 2 +- ...ashSetVectorStoreRecordCollectionTest.java | 6 +- ...isJsonVectorStoreRecordCollectionTest.java | 6 +- .../redis/RedisVectorStoreTest.java | 14 +- .../memory/redis/JsonMemoryRecord.java | 163 ----- .../memory/redis/RedisException.java | 69 -- .../memory/redis/RedisIndexSchemaParams.java | 9 - .../memory/redis/RedisMemoryRecord.java | 106 --- .../memory/redis/RedisMemoryStore.java | 658 ------------------ .../redis/RedisVectorDistanceMetric.java | 18 - .../memory/redis/RedisVectorType.java | 7 - .../pom.xml | 35 - .../semantickernel-data-azureaisearch/pom.xml | 39 ++ .../AzureAISearchVectorStore.java | 2 +- ...rchVectorStoreCollectionCreateMapping.java | 2 +- ...rchVectorStoreCollectionSearchMapping.java | 2 +- .../AzureAISearchVectorStoreOptions.java | 2 +- ...reAISearchVectorStoreRecordCollection.java | 2 +- ...rchVectorStoreRecordCollectionFactory.java | 2 +- ...rchVectorStoreRecordCollectionOptions.java | 2 +- data/semantickernel-data-jdbc/pom.xml | 69 ++ .../data/jdbc/JDBCVectorStore.java | 2 +- .../data/jdbc/JDBCVectorStoreOptions.java | 2 +- .../jdbc/JDBCVectorStoreQueryProvider.java | 4 +- .../jdbc/JDBCVectorStoreRecordCollection.java | 10 +- ...DBCVectorStoreRecordCollectionFactory.java | 2 +- ...DBCVectorStoreRecordCollectionOptions.java | 13 +- .../jdbc/JDBCVectorStoreRecordMapper.java | 3 +- .../data/jdbc/SQLVectorStore.java | 2 +- .../SQLVectorStoreFilterQueryProvider.java | 2 +- .../jdbc/SQLVectorStoreQueryProvider.java | 4 +- .../jdbc/SQLVectorStoreRecordCollection.java | 2 +- .../HSQLDBVectorStoreQueryProvider.java | 7 +- .../mysql/MySQLVectorStoreQueryProvider.java | 6 +- .../PostgreSQLVectorDistanceFunction.java | 2 +- .../postgres/PostgreSQLVectorIndexKind.java | 2 +- .../PostgreSQLVectorStoreQueryProvider.java | 6 +- .../PostgreSQLVectorStoreRecordMapper.java | 2 +- .../SQLiteVectorStoreQueryProvider.java | 7 +- .../semantickernel-data-redis}/pom.xml | 59 +- ...disHashSetVectorStoreRecordCollection.java | 2 +- ...SetVectorStoreRecordCollectionOptions.java | 2 +- .../RedisHashSetVectorStoreRecordMapper.java | 6 +- .../RedisJsonVectorStoreRecordCollection.java | 3 +- ...sonVectorStoreRecordCollectionOptions.java | 2 +- .../RedisJsonVectorStoreRecordMapper.java | 2 +- .../data/redis/RedisStorageType.java | 2 +- .../data/redis/RedisVectorStore.java | 2 +- ...disVectorStoreCollectionCreateMapping.java | 2 +- ...disVectorStoreCollectionSearchMapping.java | 2 +- .../data/redis/RedisVectorStoreOptions.java | 2 +- ...disVectorStoreRecordCollectionFactory.java | 2 +- pom.xml | 3 + .../semantickernel-syntax-examples/pom.xml | 14 + .../memory/InMemoryVolatileVectorStore.java | 5 +- .../memory/VectorStoreWithAzureAISearch.java | 13 +- .../memory/VectorStoreWithJDBC.java | 11 +- .../memory/VectorStoreWithRedis.java | 12 +- .../data}/VectorStoreTextSearch.java | 3 +- .../data}/VectorStoreTextSearchOptions.java | 2 +- .../data/VolatileVectorStore.java | 0 ...ileVectorStoreCollectionSearchMapping.java | 0 .../VolatileVectorStoreRecordCollection.java | 1 - ...ileVectorStoreRecordCollectionOptions.java | 0 .../filter/AnyTagEqualToFilterClause.java | 0 .../data/filter/EqualToFilterClause.java | 0 .../data/filter/FilterClause.java | 0 .../data/filter/FilterMapping.java | 0 .../DefaultTextSearchResultMapper.java | 0 .../DefaultTextSearchStringMapper.java | 0 .../data/textsearch/KernelSearchResults.java | 0 .../data/textsearch/TextSearch.java | 0 .../data/textsearch/TextSearchFilter.java | 0 .../data/textsearch/TextSearchOptions.java | 0 .../data/textsearch/TextSearchResult.java | 0 .../data/textsearch/TextSearchResultLink.java | 0 .../textsearch/TextSearchResultMapper.java | 0 .../data/textsearch/TextSearchResultName.java | 0 .../textsearch/TextSearchResultValue.java | 0 .../textsearch/TextSearchStringMapper.java | 0 .../data/vectorsearch/VectorOperations.java | 0 .../data/vectorsearch/VectorSearchFilter.java | 0 .../data/vectorsearch/VectorSearchResult.java | 0 .../vectorsearch/VectorSearchResults.java | 1 - .../vectorsearch/VectorizableTextSearch.java | 2 - .../data/vectorsearch/VectorizedSearch.java | 0 .../data/vectorstorage/VectorStore.java | 0 .../VectorStoreRecordCollection.java | 0 .../VectorStoreRecordCollectionOptions.java | 0 .../VectorStoreRecordMapper.java | 0 .../annotations/VectorStoreRecordData.java | 0 .../annotations/VectorStoreRecordKey.java | 0 .../annotations/VectorStoreRecordVector.java | 0 .../definition/DistanceFunction.java | 0 .../vectorstorage/definition/IndexKind.java | 0 .../VectorStoreRecordDataField.java | 0 .../VectorStoreRecordDefinition.java | 0 .../definition/VectorStoreRecordField.java | 0 .../definition/VectorStoreRecordKeyField.java | 0 .../VectorStoreRecordVectorField.java | 0 .../options/DeleteRecordOptions.java | 0 .../options/GetRecordOptions.java | 0 .../options/UpsertRecordOptions.java | 0 .../options/VectorSearchOptions.java | 0 .../microsoft/semantickernel/data/Hotel.java | 0 ...latileVectorStoreRecordCollectionTest.java | 0 .../data/VolatileVectorStoreTest.java | 0 semantickernel-bom/pom.xml | 15 + 112 files changed, 286 insertions(+), 1228 deletions(-) rename api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/{connectors/memory => data}/jdbc/Hotel.java (98%) rename api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/{connectors/memory => data}/jdbc/JDBCVectorStoreRecordCollectionTest.java (96%) rename api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/{connectors/memory => data}/jdbc/JDBCVectorStoreTest.java (83%) rename api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/{connectors/memory => data}/redis/Hotel.java (97%) rename api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/{connectors/memory => data}/redis/RedisHashSetVectorStoreRecordCollectionTest.java (98%) rename api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/{connectors/memory => data}/redis/RedisJsonVectorStoreRecordCollectionTest.java (98%) rename api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/{connectors/memory => data}/redis/RedisVectorStoreTest.java (82%) delete mode 100644 connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/JsonMemoryRecord.java delete mode 100644 connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisException.java delete mode 100644 connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisIndexSchemaParams.java delete mode 100644 connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisMemoryRecord.java delete mode 100644 connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisMemoryStore.java delete mode 100644 connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisVectorDistanceMetric.java delete mode 100644 connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisVectorType.java delete mode 100644 connectors/semantickernel-connectors-web-bing/pom.xml create mode 100644 data/semantickernel-data-azureaisearch/pom.xml rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel}/data/azureaisearch/AzureAISearchVectorStore.java (98%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel}/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java (99%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel}/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java (98%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel}/data/azureaisearch/AzureAISearchVectorStoreOptions.java (97%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel}/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java (99%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel}/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java (93%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel}/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java (98%) create mode 100644 data/semantickernel-data-jdbc/pom.xml rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel}/data/jdbc/JDBCVectorStore.java (99%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel}/data/jdbc/JDBCVectorStoreOptions.java (98%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel}/data/jdbc/JDBCVectorStoreQueryProvider.java (99%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel}/data/jdbc/JDBCVectorStoreRecordCollection.java (96%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel}/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java (94%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel}/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java (91%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel}/data/jdbc/JDBCVectorStoreRecordMapper.java (98%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel}/data/jdbc/SQLVectorStore.java (88%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel}/data/jdbc/SQLVectorStoreFilterQueryProvider.java (91%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel}/data/jdbc/SQLVectorStoreQueryProvider.java (96%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel}/data/jdbc/SQLVectorStoreRecordCollection.java (91%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data => data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc}/hsqldb/HSQLDBVectorStoreQueryProvider.java (97%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data => data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc}/mysql/MySQLVectorStoreQueryProvider.java (96%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data => data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc}/postgres/PostgreSQLVectorDistanceFunction.java (96%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data => data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc}/postgres/PostgreSQLVectorIndexKind.java (95%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data => data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc}/postgres/PostgreSQLVectorStoreQueryProvider.java (98%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data => data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc}/postgres/PostgreSQLVectorStoreRecordMapper.java (99%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data => data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc}/sqlite/SQLiteVectorStoreQueryProvider.java (96%) rename {connectors/semantickernel-connectors-memory-redis => data/semantickernel-data-redis}/pom.xml (55%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel}/data/redis/RedisHashSetVectorStoreRecordCollection.java (99%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel}/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java (99%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel}/data/redis/RedisHashSetVectorStoreRecordMapper.java (95%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel}/data/redis/RedisJsonVectorStoreRecordCollection.java (99%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel}/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java (99%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel}/data/redis/RedisJsonVectorStoreRecordMapper.java (98%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel}/data/redis/RedisStorageType.java (81%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel}/data/redis/RedisVectorStore.java (98%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel}/data/redis/RedisVectorStoreCollectionCreateMapping.java (99%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel}/data/redis/RedisVectorStoreCollectionSearchMapping.java (99%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel}/data/redis/RedisVectorStoreOptions.java (98%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors => data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel}/data/redis/RedisVectorStoreRecordCollectionFactory.java (94%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage => semantickernel-api/src/main/java/com/microsoft/semantickernel/data}/VectorStoreTextSearch.java (98%) rename {semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage => semantickernel-api/src/main/java/com/microsoft/semantickernel/data}/VectorStoreTextSearchOptions.java (71%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java (99%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java (98%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java (96%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/test/java/com/microsoft/semantickernel/data/Hotel.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java (100%) rename {semantickernel-experimental => semantickernel-api}/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreTest.java (100%) diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index ff618703..1f2a87f8 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -58,7 +58,12 @@ com.microsoft.semantic-kernel - semantickernel-experimental + semantickernel-data-jdbc + test + + + com.microsoft.semantic-kernel + semantickernel-data-redis test diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/jdbc/Hotel.java similarity index 98% rename from api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java rename to api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/jdbc/Hotel.java index d78a333d..587fa3a8 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/jdbc/Hotel.java @@ -1,4 +1,4 @@ -package com.microsoft.semantickernel.tests.connectors.memory.jdbc; +package com.microsoft.semantickernel.tests.data.jdbc; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/jdbc/JDBCVectorStoreRecordCollectionTest.java similarity index 96% rename from api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java rename to api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/jdbc/JDBCVectorStoreRecordCollectionTest.java index 17b36756..6bc99f25 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -1,16 +1,16 @@ -package com.microsoft.semantickernel.tests.connectors.memory.jdbc; +package com.microsoft.semantickernel.tests.data.jdbc; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; -import com.microsoft.semantickernel.connectors.data.hsqldb.HSQLDBVectorStoreQueryProvider; -import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollection; -import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.connectors.data.sqlite.SQLiteVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.hsqldb.HSQLDBVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollection; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.jdbc.SQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.mysql.MySQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.sqlite.SQLiteVectorStoreQueryProvider; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/jdbc/JDBCVectorStoreTest.java similarity index 83% rename from api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java rename to api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/jdbc/JDBCVectorStoreTest.java index f3a8744d..ce62c70d 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/jdbc/JDBCVectorStoreTest.java @@ -1,13 +1,13 @@ -package com.microsoft.semantickernel.tests.connectors.memory.jdbc; - -import com.microsoft.semantickernel.connectors.data.hsqldb.HSQLDBVectorStoreQueryProvider; -import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStore; -import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions; -import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.connectors.data.sqlite.SQLiteVectorStoreQueryProvider; +package com.microsoft.semantickernel.tests.data.jdbc; + +import com.microsoft.semantickernel.data.jdbc.hsqldb.HSQLDBVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.SQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.jdbc.mysql.MySQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.sqlite.SQLiteVectorStoreQueryProvider; import com.mysql.cj.jdbc.MysqlDataSource; import org.hsqldb.jdbc.JDBCDataSourceFactory; import org.junit.jupiter.params.ParameterizedTest; @@ -21,17 +21,15 @@ import org.testcontainers.utility.DockerImageName; import javax.sql.DataSource; -import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Properties; -import com.microsoft.semantickernel.tests.connectors.memory.jdbc.JDBCVectorStoreRecordCollectionTest.QueryProvider; +import com.microsoft.semantickernel.tests.data.jdbc.JDBCVectorStoreRecordCollectionTest.QueryProvider; -import static com.microsoft.semantickernel.tests.connectors.memory.jdbc.JDBCVectorStoreRecordCollectionTest.createTempDbFile; +import static com.microsoft.semantickernel.tests.data.jdbc.JDBCVectorStoreRecordCollectionTest.createTempDbFile; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/Hotel.java similarity index 97% rename from api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java rename to api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/Hotel.java index da64efb0..416e06b6 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/Hotel.java @@ -1,4 +1,4 @@ -package com.microsoft.semantickernel.tests.connectors.memory.redis; +package com.microsoft.semantickernel.tests.data.redis; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisHashSetVectorStoreRecordCollectionTest.java similarity index 98% rename from api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java rename to api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisHashSetVectorStoreRecordCollectionTest.java index 5256c629..c5b6a186 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -1,7 +1,7 @@ -package com.microsoft.semantickernel.tests.connectors.memory.redis; +package com.microsoft.semantickernel.tests.data.redis; -import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollection; -import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.redis.RedisHashSetVectorStoreRecordCollection; +import com.microsoft.semantickernel.data.redis.RedisHashSetVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisJsonVectorStoreRecordCollectionTest.java similarity index 98% rename from api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java rename to api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisJsonVectorStoreRecordCollectionTest.java index 7ab0bcc8..1b4c30e8 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -1,7 +1,7 @@ -package com.microsoft.semantickernel.tests.connectors.memory.redis; +package com.microsoft.semantickernel.tests.data.redis; -import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollection; -import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.redis.RedisJsonVectorStoreRecordCollection; +import com.microsoft.semantickernel.data.redis.RedisJsonVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisVectorStoreTest.java similarity index 82% rename from api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java rename to api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisVectorStoreTest.java index ea48a155..cfff9ea0 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/redis/RedisVectorStoreTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisVectorStoreTest.java @@ -1,12 +1,12 @@ -package com.microsoft.semantickernel.tests.connectors.memory.redis; +package com.microsoft.semantickernel.tests.data.redis; -import com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.connectors.data.redis.RedisStorageType; -import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStore; -import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreOptions; +import com.microsoft.semantickernel.data.redis.RedisHashSetVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.redis.RedisJsonVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.redis.RedisStorageType; +import com.microsoft.semantickernel.data.redis.RedisVectorStore; +import com.microsoft.semantickernel.data.redis.RedisVectorStoreOptions; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.tests.connectors.memory.jdbc.Hotel; +import com.microsoft.semantickernel.tests.data.jdbc.Hotel; import com.redis.testcontainers.RedisContainer; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; diff --git a/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/JsonMemoryRecord.java b/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/JsonMemoryRecord.java deleted file mode 100644 index 68ca9d40..00000000 --- a/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/JsonMemoryRecord.java +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.memory.redis; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.microsoft.semantickernel.ai.embeddings.Embedding; -import com.microsoft.semantickernel.memory.MemoryRecord; -import com.microsoft.semantickernel.memory.MemoryRecordMetadata; -import java.nio.charset.StandardCharsets; -import java.util.*; - -/** Represents an entry in the Semantic Kernel Memory Table. */ -public class JsonMemoryRecord { - - static final String ID = "Id"; - static final String TEXT = "Text"; - static final String DESCRIPTION = "Description"; - static final String ADDITIONAL_METADATA = "AdditionalMetadata"; - static final String EMBEDDING = "Embedding"; - static final String EXTERNAL_SOURCE_NAME = "ExternalSourceName"; - static final String IS_REFERENCE = "Reference"; - static final String TIME = "Timestamp"; - - private final String id; - private final String text; - private final String description; - private final String additionalMetadata; - private final List embedding; - private final String externalSourceName; - private final boolean isReference; - - @JsonCreator - public JsonMemoryRecord( - @JsonProperty(ID) String id, - @JsonProperty(TEXT) String text, - @JsonProperty(DESCRIPTION) String description, - @JsonProperty(ADDITIONAL_METADATA) String additionalMetadata, - @JsonProperty(EMBEDDING) List embedding, - @JsonProperty(EXTERNAL_SOURCE_NAME) String externalSourceName, - @JsonProperty(IS_REFERENCE) boolean isReference) { - this.id = id; - this.text = text; - this.description = description; - this.additionalMetadata = additionalMetadata; - this.embedding = embedding != null ? embedding : Collections.emptyList(); - this.externalSourceName = externalSourceName; - this.isReference = isReference; - } - - /** - * Record ID. The record is not filterable to save quota, also SK uses only semantic search. - * - * @return Record ID. - */ - public String getId() { - return id; - } - - /** - * Content is stored here. - * - * @return Content is stored here, {@code null} if not set. - */ - public String getText() { - return text; - } - - /** - * Optional description of the content, e.g. a title. This can be useful when indexing external - * data without pulling in the entire content. - * - * @return Optional description of the content, {@code null} if not set. - */ - public String getDescription() { - return description; - } - - /** - * Additional metadata. Currently, this is a String where you could store serialized data as - * JSON. In future the design might change to allow storing named values and leverage filters. - * - * @return Additional metadata, {@code null} if not set. - */ - public String getAdditionalMetadata() { - return additionalMetadata; - } - - /** - * Embedding vector. - * - * @return Embedding vector. - */ - public List getEmbedding() { - return Collections.unmodifiableList(embedding); - } - - /** - * Name of the external source, in cases where the content and the Id are referenced to external - * information. - * - * @return Name of the external source, in cases where the content and the Id are referenced to - * external information, {@code null} if not set. - */ - public String getExternalSourceName() { - return externalSourceName; - } - - /** - * Whether the record references external information. - * - * @return {@code true} if the record references external information, {@code false} otherwise. - */ - public boolean isReference() { - return isReference; - } - - public MemoryRecordMetadata toMemoryRecordMetadata() { - return new MemoryRecordMetadata( - this.isReference, - decodeId(this.id), - this.text, - this.description, - this.externalSourceName, - this.additionalMetadata); - } - - public static JsonMemoryRecord fromMemoryRecord(MemoryRecord record) { - return new JsonMemoryRecord( - record.getMetadata().getId(), - record.getMetadata().getText(), - record.getMetadata().getDescription(), - record.getMetadata().getAdditionalMetadata(), - record.getEmbedding().getVector(), - record.getMetadata().getExternalSourceName(), - record.getMetadata().isReference()); - } - - public MemoryRecord toMemoryRecord() { - return toMemoryRecord(true); - } - - public MemoryRecord toMemoryRecord(boolean withEmbeddings) { - return new MemoryRecord( - this.toMemoryRecordMetadata(), - new Embedding(withEmbeddings ? this.embedding : Collections.emptyList()), - this.id, - null); - } - - // ACS keys can contain only letters, digits, underscore, dash, equal sign, recommending - // to encode values with a URL-safe algorithm. - // Original Id - // Encoded id - static String encodeId(String realId) { - byte[] bytes = Base64.getUrlEncoder().encode(realId.getBytes(StandardCharsets.UTF_8)); - return new String(bytes, StandardCharsets.UTF_8); - } - - static String decodeId(String encodedId) { - byte[] bytes = Base64.getUrlDecoder().decode(encodedId.getBytes(StandardCharsets.UTF_8)); - return new String(bytes, StandardCharsets.UTF_8); - } -} diff --git a/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisException.java b/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisException.java deleted file mode 100644 index 2060938a..00000000 --- a/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisException.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.memory.redis; - -import com.microsoft.semantickernel.SKException; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -/** Exception thrown by the SQL connector. */ -public class RedisException extends SKException { - - private final ErrorCodes errorCode; - - /** - * Create an exception with a message - * - * @param errorCode The error code - */ - public RedisException(@Nonnull ErrorCodes errorCode) { - - this(errorCode, null, null); - } - - /** - * Create an exception with a message - * - * @param errorCode The error code - * @param message a description of the cause of the exception - */ - public RedisException(@Nonnull ErrorCodes errorCode, @Nullable String message) { - - this(errorCode, message, null); - } - - /** - * Create an exception with a message and a cause - * - * @param errorCode the error code - * @param message a description of the cause of the exception - * @param cause the cause of the exception - */ - public RedisException( - @Nonnull ErrorCodes errorCode, @Nullable String message, @Nullable Throwable cause) { - super(message, cause); - this.errorCode = errorCode; - } - - public ErrorCodes getErrorCode() { - return errorCode; - } - - public enum ErrorCodes { - REDIS_ERROR("Redis error"), - UNKNOWN_ERROR("Unknown error"), - INVALID_EMBEDDING_SIZE("Invalid embedding size"), - INVALID_INDEX_NAME("Invalid index name"), - READ_FAILURE("Read failure"), - WRITE_FAILURE("Write failure"); - - final String message; - - ErrorCodes(String message) { - this.message = message; - } - - public String getMessage() { - return message; - } - } -} diff --git a/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisIndexSchemaParams.java b/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisIndexSchemaParams.java deleted file mode 100644 index e3f4eb9d..00000000 --- a/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisIndexSchemaParams.java +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.memory.redis; - -public class RedisIndexSchemaParams { - - public static final String TYPE = "TYPE"; - public static final String DIM = "DIM"; - public static final String DIST = "DISTANCE_METRIC"; -} diff --git a/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisMemoryRecord.java b/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisMemoryRecord.java deleted file mode 100644 index db687045..00000000 --- a/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisMemoryRecord.java +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.memory.redis; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.microsoft.semantickernel.ai.embeddings.Embedding; -import com.microsoft.semantickernel.memory.MemoryRecord; -import java.time.ZonedDateTime; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; -import redis.clients.jedis.search.Document; - -/** Represents an entry in the Semantic Kernel Memory Table. */ -public class RedisMemoryRecord extends JsonMemoryRecord { - - public RedisMemoryRecord( - String id, - String text, - String description, - String additionalMetadata, - List embedding, - String externalSourceName, - boolean isReference) { - super( - id, - text, - description, - additionalMetadata, - embedding, - externalSourceName, - isReference); - } - - /** - * Converts a RedisSearch Document to a HashMap. - * - * @param document A Redis Search Document - * @return A HashMap containing the iterator's entries - */ - protected Map documentToMap(Document document) { - return StreamSupport.stream(document.getProperties().spliterator(), false) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - /** - * Gets the embedding information associated with the entry. - * - * @param map A hashmap with keys representing a record - * @param withEmbedding A boolean flag indicating that the embedding should be returned with the - * metadata - * @return A MemoryRecord object. - */ - public static MemoryRecord mapToRecord(Map map, boolean withEmbedding) { - Embedding embedding = Embedding.empty(); - ZonedDateTime time = null; - ObjectMapper mapper = new ObjectMapper(); - try { - if (map.get(TIME) != null) { - time = - mapper.readValue( - map.getOrDefault(TIME, ZonedDateTime.now().toString()), - ZonedDateTime.class); - } - if (withEmbedding) { - List embeddings = mapper.readValue(map.get(EMBEDDING), List.class); - embedding = new Embedding(embeddings); - } - } catch (JsonProcessingException e) { - throw new RedisException( - RedisException.ErrorCodes.REDIS_ERROR, "Error deserializing Redis entry", e); - } - - return MemoryRecord.localRecord( - map.get(ID), - map.get(TEXT), - map.get(DESCRIPTION), - embedding, - map.get(ADDITIONAL_METADATA), - map.get(EXTERNAL_SOURCE_NAME), - time); - } - - /** - * Gets the embedding information associated with the entry. - * - * @param record A MemoryRecord object - * @return A hashmap containing the entries of the record. - */ - public static Map recordToMap(MemoryRecord record) { - - Map map = new HashMap(); - - map.put(ID, record.getMetadata().getId()); - map.put(EXTERNAL_SOURCE_NAME, record.getMetadata().getExternalSourceName()); - map.put(TEXT, record.getMetadata().getText()); - map.put(DESCRIPTION, record.getMetadata().getDescription()); - map.put(EMBEDDING, record.getEmbedding().getVector()); - map.put(ADDITIONAL_METADATA, record.getMetadata().getAdditionalMetadata()); - map.put(IS_REFERENCE, record.getMetadata().isReference()); - - return map; - } -} diff --git a/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisMemoryStore.java b/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisMemoryStore.java deleted file mode 100644 index 6a40d3d5..00000000 --- a/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisMemoryStore.java +++ /dev/null @@ -1,658 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.memory.redis; - -import com.microsoft.semantickernel.SKException; -import com.microsoft.semantickernel.ai.embeddings.Embedding; -import com.microsoft.semantickernel.memory.MemoryException; -import com.microsoft.semantickernel.memory.MemoryException.ErrorCodes; -import com.microsoft.semantickernel.memory.MemoryRecord; -import com.microsoft.semantickernel.memory.MemoryStore; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.text.MessageFormat; -import java.time.Instant; -import java.time.OffsetDateTime; -import java.time.ZoneId; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; -import javax.annotation.Nonnull; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; -import reactor.util.function.Tuple2; -import reactor.util.function.Tuples; -import redis.clients.jedis.JedisPooled; -import redis.clients.jedis.exceptions.JedisDataException; -import redis.clients.jedis.resps.ScanResult; -import redis.clients.jedis.search.Document; -import redis.clients.jedis.search.IndexDefinition; -import redis.clients.jedis.search.IndexOptions; -import redis.clients.jedis.search.Query; -import redis.clients.jedis.search.Schema; -import redis.clients.jedis.search.SearchResult; -import redis.clients.jedis.search.schemafields.VectorField.VectorAlgorithm; - -/** - * Semantic Memory implementation using Redis Vector Search. For more information about Redis Vector - * Search {@see https://redis.com/solutions/use-cases/vector-database/} - */ -public class RedisMemoryStore implements MemoryStore { - - private static final VectorAlgorithm DefaultIndexAlgorithm = - VectorAlgorithm.HNSW; // FLAT or HNSW - private static final String DefaultVectorType = - RedisVectorType.FLOAT32; // prefer better accuracy - private static final String DefaultDistanceMetric = RedisVectorDistanceMetric.L2; - private static final Integer DefaultQueryDialect = - 2; // this has to be >= 2 for vector functionality - private static final Integer DefaultVectorSize = 1536; // framework model dependent - ada-002 - private static final String SUFFIX = "sk"; - private static final String INDEX = "idx"; - private static final String OK = "OK"; - - private final JedisPooled client; - private final String vectorDistanceMetric; - - @SuppressFBWarnings("SS_SHOULD_BE_STATIC") - private final String vectorType = DefaultVectorType; - - private final Integer queryDialect; - - public static void isXinRange(Integer x, Integer lower, Integer upper, String message) { - if (lower > x || x > upper) { - throw new IllegalArgumentException(MessageFormat.format(message, x)); - } - } - - /** - * Create a new instance of semantic memory using Redis. - * - * @param client A Redis Database client connection. - * @param vectorSize Embedding vector size, defaults to 1536 - framework model dependent - - * ada-002. - * @param vectorDistanceMetric Metric for measuring vector distances, defaults to "COSINE" - * @param queryDialect Query dialect, must be 2 or greater for vector similarity searching, - * defaults to 2 - */ - @SuppressFBWarnings("EI_EXPOSE_REP2") - public RedisMemoryStore( - JedisPooled client, - Integer vectorSize, - String vectorDistanceMetric, - Integer queryDialect) { - - isXinRange( - vectorSize, - 0, - 8192, - "Invalid vector size: {x}. Vector size must be in the range 0-8192."); - isXinRange( - queryDialect, - 2, - 4, - "Invalid query dialect: {x}. Query dialect must be in the range 2-4."); - - this.client = client; - this.vectorDistanceMetric = vectorDistanceMetric; - this.queryDialect = queryDialect; - } - - /** - * Create a new instance of semantic memory using Redis. - * - * @param connectionString Provide connection URL to a Redis instance. - * @param vectorSize Embedding vector size, defaults to 1536. - * @param vectorIndexAlgorithm Indexing algorithm for vectors, defaults to "HNSW" - * @param vectorDistanceMetric Metric for measuring vector distances, defaults to "COSINE" - * @param queryDialect Query dialect, must be 2 or greater for vector similarity searching, - * defaults to 2 - */ - public RedisMemoryStore( - String connectionString, - Integer vectorSize, - VectorAlgorithm vectorIndexAlgorithm, - String vectorDistanceMetric, - Integer queryDialect) { - - isXinRange( - vectorSize, - 0, - 8192, - "Invalid vector size: {x}. Vector size must be in the range 0-8192."); - isXinRange( - queryDialect, - 2, - 4, - "Invalid query dialect: {x}. Query dialect must be in the range 2-4."); - - this.client = new JedisPooled(connectionString); - this.vectorDistanceMetric = vectorDistanceMetric; - this.queryDialect = queryDialect; - } - - /** - * Returns a Redis key - * - * @param collectionName The name associated with a collection of embeddings. - * @param key The unique id associated with the memory record to get. - * @return A Redis key that identifies a particular Hash of Json object. - */ - private static String getRedisKey(String collectionName, String key) { - if (key == null) { - throw new SKException("cannot create a key with a null value"); - } - - if (collectionName == null) { - throw new SKException("cannot form a redis key with a null collection name"); - } - - String name = collectionName.toLowerCase(Locale.ROOT); - String id = JsonMemoryRecord.encodeId(key); - return String.format("%s:%s", name, id); - } - - /** - * Returns a long timestamp given a OffsetDateTime - * - * @param timestamp An OffsetDateTime. - * @return A long timestamp or -1 in the case of an error. - */ - private static long toTimestampLong(OffsetDateTime timestamp) { - if (timestamp != null) { - return timestamp.toInstant().toEpochMilli(); - } - return -1; - } - - /** - * Returns an OffsetDateTime given a valid timestamp - * - * @param timestamp A long timestamp value. - * @return An OffsetDateTime if the timestamp is valid, else null. - */ - private static OffsetDateTime toDateTime(long timestamp) { - if (timestamp > 0) { - Instant instant = Instant.ofEpochMilli(timestamp); - return OffsetDateTime.ofInstant(instant, ZoneId.systemDefault()); - } - return null; - } - - @Override - public Mono> getCollectionsAsync() { - return getIndexesAsync() - .map( - names -> - names.stream() - .map( - name -> - name.substring( - 0, - name.length() - "-sk-idx".length())) - .collect(Collectors.toList())); - } - - /** - * Returns a List of Index(Collection) Names - * - * @return A list of names of the embedding indexes, including suffix, in the database - */ - private Mono> getIndexesAsync() { - return Mono.just( - this.client.ftList().stream() - .filter(name -> name.endsWith("-sk-idx")) - .collect(Collectors.toList())); - } - - @Override - public Mono createCollectionAsync(@Nonnull String collectionName) { - // Indexes are created when sending a record; creation requires the size of the embedding - // vector - return Mono.empty(); - } - - public Mono doesIndexExistAsync(@Nonnull String collectionName) { - - try { - Map info = this.client.ftInfo(collectionName); - return Mono.just(info != null && !info.isEmpty()); - } catch (Exception e) { - if (!(e instanceof JedisDataException)) { - throw e; - } - return Mono.just(false); - } - // return Mono.just(this.client.ftInfo(createIndexName(collectionName)) != null); - } - - @Override - public Mono doesCollectionExistAsync(@Nonnull String collectionName) { - - Objects.requireNonNull(collectionName); - String normalizedIndexName = createIndexName(collectionName); - return getIndexesAsync() - .map( - list -> - list.stream() - .anyMatch( - name -> - name.equalsIgnoreCase(collectionName) - || name.equalsIgnoreCase( - normalizedIndexName))); - } - - @Override - public Mono deleteCollectionAsync(@Nonnull String collectionName) { - Objects.requireNonNull(collectionName); - try { - this.client.ftDropIndex(createIndexName(collectionName)); - return Mono.empty(); - } catch (Exception e) { - return Mono.error(e); - } - } - - @Override - public Mono getAsync( - @Nonnull String collectionName, @Nonnull String key, boolean withEmbedding) { - - Map map = - this.client.hgetAll( - String.format("%s:%s", collectionName, key).toLowerCase(Locale.ROOT)); - if (map == null || map.isEmpty()) { - return Mono.empty(); - } - MemoryRecord record = RedisMemoryRecord.mapToRecord(map, withEmbedding); - return Mono.just(record); - } - - public Mono getInternalAsync( - @Nonnull String collectionName, @Nonnull String key, boolean withEmbedding) { - - Objects.requireNonNull(collectionName); - Objects.requireNonNull(key); - - Map entry = - this.client.hgetAll(String.format("%s:%s", collectionName, key)); - - if (entry == null) return Mono.empty(); - - return Mono.just(RedisMemoryRecord.mapToRecord(entry, withEmbedding)); - } - - /** - * Converts a RedisSearch Document to a HashMap. - * - * @param document A Redis Search Document - * @return A HashMap containing the iterator's entries - */ - protected Map documentToMap(Document document) { - - return iterableToMap(document.getProperties()); - } - - /** - * Converts an Iterable> to a HashMap. - * - * @param entryIterator An Iterable> - * @return A HashMap containing the iterator's entries - */ - protected Map iterableToMap(Iterable> entryIterator) { - - Map entryMap = new HashMap<>(); - entryIterator.forEach((e) -> entryMap.put(e.getKey(), e.getValue().toString())); - - return entryMap; - } - - protected MemoryRecord documentToMemoryRecord(Document document, boolean withEmbedding) { - - Map map = documentToMap(document); - return RedisMemoryRecord.mapToRecord(map, withEmbedding); - } - - @Override - public Mono> getBatchAsync( - @Nonnull String collectionName, - @Nonnull Collection keys, - boolean withEmbeddings) { - Objects.requireNonNull(collectionName); - // ACS issues one query per key; redis has several possible calls, fastest will be hkeys - // followed by hvals - - return Flux.fromIterable(keys) - .flatMap(key -> getAsync(collectionName, key, withEmbeddings)) - .collectList() - .map(records -> records); - } - - @Override - public Mono upsertAsync(@Nonnull String collectionName, @Nonnull MemoryRecord record) { - return upsertRecordAsync(collectionName, record); - } - - /** - * Returns the key of the upserted object (wraps the record in a list and calls the batch - * function). - * - * @param collectionName The name associated with a collection of embeddings. - * @param record The memory record to upsert. - * @return The key of the newly created record. - */ - public Mono upsertRecordAsync( - @Nonnull String collectionName, @Nonnull MemoryRecord record) { - return upsertBatchAsync(collectionName, Collections.singletonList(record)) - .map(Collection::iterator) - .map(Iterator::next); - } - - @Override - public Mono> upsertBatchAsync( - @Nonnull String collectionName, @Nonnull Collection records) { - - Objects.requireNonNull(collectionName); - Objects.requireNonNull(records); - - if (records.isEmpty()) { - return Mono.just(Collections.emptyList()); - } - - doesCollectionExistAsync(collectionName) - .map( - exists -> { - if (!exists) { - int embeddingSize = - records.stream() - .map(record -> record.getEmbedding().getVector()) - .map(List::size) - .max(Integer::compareTo) - .orElse(0); - createIndexAsync(collectionName, embeddingSize); - } - return true; - }); - - return Mono.just( - records.stream() - .map( - record -> { - Map map = RedisMemoryRecord.recordToMap(record); - // we have to change the embeddings to bytes - it's easiest to - // modify the map - List embedding = record.getEmbedding().getVector(); - map.replace( - RedisMemoryRecord.EMBEDDING, - embedding, - embeddingToBytes(embedding)); - String key = - getRedisKey( - collectionName, record.getMetadata().getId()); - this.client.hsetObject(key, map); - return key; - }) - .collect(Collectors.toList())); - } - - @Override - public Mono removeAsync(@Nonnull String collectionName, @Nonnull String key) { - return Mono.fromRunnable( - () -> this.client.hdel(String.format("%s:%s", collectionName, key))); - } - - @Override - public Mono removeBatchAsync( - @Nonnull String collectionName, @Nonnull Collection keys) { - return Mono.fromRunnable( - () -> { - // ScanResult results = this.client.scan(collectionName); - // results.getResult().forEach(key -> this.client.hdel(key)); - keys.forEach( - key -> this.client.hdel(String.format("%s:%s", collectionName, key))); - }); - } - - /** - * Create a new search index. - * - * @param collectionName Index name - * @param embeddingSize Size of the embedding vector - * @return A Mono that completes when the index is created - */ - private Mono createIndexAsync(@Nonnull String collectionName, int embeddingSize) { - if (embeddingSize < 1) { - throw new RedisException( - RedisException.ErrorCodes.INVALID_EMBEDDING_SIZE, - "the value must be greater than zero"); - } - - Map attributes = new HashMap<>(); - - attributes.put(RedisIndexSchemaParams.TYPE, vectorType); - attributes.put(RedisIndexSchemaParams.DIM, embeddingSize); - attributes.put(RedisIndexSchemaParams.DIST, vectorDistanceMetric); - - Schema schema = - new Schema() - .addVectorField("Embedding", Schema.VectorField.VectorAlgo.FLAT, attributes) - .as("Embedding") - .addTextField("Id", 1.0) - .as("Id") - .addTextField("Text", 1.0) - .as("Text") - .addTextField("Description", 1.0) - .as("Description") - .addTextField("AdditionalMetadata", 1.0) - .as("AdditionalMetadata") - .addTextField("ExternalSourceName", 1.0) - .as("ExternalSourceName"); - - IndexDefinition rule = - new IndexDefinition(IndexDefinition.Type.HASH) - .setPrefixes(collectionName.toLowerCase() + ":"); - // customarily redis indexes are named -idx; we're going to suffix them - // (-sk-idx) so we can find them - String normalizedIndexName = createIndexName(collectionName); - String result = - this.client.ftCreate( - normalizedIndexName, - IndexOptions.defaultOptions().setDefinition(rule), - schema); - - if (result.equals(OK)) { - return Mono.just(true); - } - return null; - } - - @Override - public Mono>> getNearestMatchesAsync( - @Nonnull String collectionName, - @Nonnull Embedding embedding, - int limit, - float minRelevanceScore, - boolean withEmbedding) { - - Objects.requireNonNull(collectionName); - Objects.requireNonNull(embedding); - - if (limit <= 0) { - return Mono.just(Collections.emptyList()); - } - Query query = - new Query("*=>[KNN $k @Embedding $vec AS vector_score]") - .returnFields("Id", "Text", "Description", "vector_score") - .setSortBy("vector_score", true) - .addParam("k", limit) - .addParam("vec", embeddingToBytes(embedding.getVector())) - .limit(0, limit) - .dialect(queryDialect); - String indexName = createIndexName(collectionName); - SearchResult results = this.client.ftSearch(indexName, query); - - // TODO convert to Collection> - Collection> searchTuples = - results.getDocuments().stream() - .map( - document -> { - MemoryRecord memoryRecord = - documentToMemoryRecord(document, withEmbedding); - return Tuples.of( - memoryRecord, document.getScore().floatValue()); - }) - .collect(Collectors.toList()); - return Mono.just(searchTuples); - } - - @Override - public Mono> getNearestMatchAsync( - @Nonnull String collectionName, - @Nonnull Embedding embedding, - float minRelevanceScore, - boolean withEmbedding) { - return getNearestMatchesAsync( - collectionName, embedding, 1, minRelevanceScore, withEmbedding) - .flatMap( - nearestMatches -> { - if (nearestMatches.isEmpty()) { - return Mono.empty(); - } - return Mono.just(nearestMatches.iterator().next()); - }); - } - - /** - * Convert the Azure List embedding structure to byte[] for storage in Redis - * - * @param embeddings the list of vectors returned by the model - * @return byte[] - */ - private byte[] embeddingToBytes(List embeddings) { - ByteBuffer bytes = ByteBuffer.allocate(Float.BYTES * embeddings.size()); - bytes.order(ByteOrder.LITTLE_ENDIAN); - embeddings.iterator().forEachRemaining(bytes::putFloat); - return bytes.array(); - } - - private static OffsetDateTime ParseTimestamp(Long timestamp) { - if (timestamp != null && timestamp > 0) { - return OffsetDateTime.parse(timestamp.toString()); - } - return null; - } - - /** - * Normalize an index name - * - * @param indexName Index name - * @return A string to be used as the root of the index name; minus '-idx' - */ - private static String normalizeCollectionName(String indexName) { - if (indexName.length() > 128) { - throw new IllegalArgumentException("The collection name cannot exceed 128 chars"); - } - return indexName.toLowerCase(Locale.ROOT); - } - - private static String createIndexName(String collectionName) { - - if (collectionName.length() > 128) { - throw new IllegalArgumentException("The indexName name cannot exceed 128 chars"); - } - return String.format( - "%s-%s-%s", - normalizeCollectionName(collectionName), RedisMemoryStore.SUFFIX, INDEX); - } - - protected Map getCollection(@Nonnull String collectionName) { - Objects.requireNonNull(collectionName); - ScanResult results = this.client.scan(collectionName); - - if (results == null) - throw new MemoryException( - ErrorCodes.ATTEMPTED_TO_ACCESS_NONEXISTENT_COLLECTION, collectionName); - - return Collections.unmodifiableMap( - (Map) - results.getResult().stream() - .map( - key -> - RedisMemoryRecord.mapToRecord( - this.client.hgetAll( - String.format( - "%s:%s", - collectionName, key)), - false))); - } - - public static class Builder implements MemoryStore.Builder { - private String connectionString; - private int vectorSize; // in theory determined by the model but can be overridden - private VectorAlgorithm vectorIndexAlgorithm; - private String vectorDistanceMetric; - private int queryDialect = 0; // must best at least two for vector index - - public Builder connectionString(String connectionString) { - this.connectionString = connectionString; - return this; - } - - public Builder vectorSize(int vectorSize) { - this.vectorSize = vectorSize; - return this; - } - - public Builder vectorIndexAlgorithm(VectorAlgorithm vectorIndexAlgorithm) { - this.vectorIndexAlgorithm = vectorIndexAlgorithm; - return this; - } - - public Builder vectorDistanceMetric(String vectorDistanceMetric) { - this.vectorDistanceMetric = vectorDistanceMetric; - return this; - } - - public Builder queryDialect(int queryDialect) { - this.queryDialect = queryDialect; - return this; - } - - @Override - /* - * Create a RedisMemoryStore using assigned values. - * - * @return A RedisMemoryStore object configured with the specified values - */ - public RedisMemoryStore build() { - - // make sure everything we need has been set - return new RedisMemoryStore( - connectionString, - vectorSize, - vectorIndexAlgorithm, - vectorDistanceMetric, - queryDialect); - } - - /** - * Create a RedisMemoryStore object using default settings; requires a valid connection - * string. - * - * @return A RedisMemoryStore object configured with default values - */ - public RedisMemoryStore buildDefault() { - return new RedisMemoryStore( - connectionString, - DefaultVectorSize, - RedisMemoryStore.DefaultIndexAlgorithm, - RedisMemoryStore.DefaultDistanceMetric, - RedisMemoryStore.DefaultQueryDialect); - } - } -} diff --git a/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisVectorDistanceMetric.java b/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisVectorDistanceMetric.java deleted file mode 100644 index 06d17643..00000000 --- a/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisVectorDistanceMetric.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.memory.redis; - -/* - Supported distance metrics are {L2, IP, COSINE}. The default value is "COSINE". - - */ -public class RedisVectorDistanceMetric { - - /* Euclidean distance between two vectors */ - public static final String L2 = "L2"; - - /* Inner product of two vectors */ - public static final String IP = "IP"; - - /* Cosine distance of two vectors */ - public static final String COS = "COSINE"; -} diff --git a/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisVectorType.java b/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisVectorType.java deleted file mode 100644 index 92014681..00000000 --- a/connectors/semantickernel-connectors-memory-redis/src/main/java/com/microsoft/semantickernel/connectors/memory/redis/RedisVectorType.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.memory.redis; - -public class RedisVectorType { - public static final String FLOAT32 = "FLOAT32"; - public static final String FLOAT64 = "FLOAT64"; -} diff --git a/connectors/semantickernel-connectors-web-bing/pom.xml b/connectors/semantickernel-connectors-web-bing/pom.xml deleted file mode 100644 index 93eaf28a..00000000 --- a/connectors/semantickernel-connectors-web-bing/pom.xml +++ /dev/null @@ -1,35 +0,0 @@ - - - - 4.0.0 - - - com.microsoft.semantic-kernel - semantickernel-parent - 1.0.0-rc2-SNAPSHOT - ../../pom.xml - - - semantickernel-connectors-web-bing - - Semantic Kernel Bing Connector - Defines Bing connectors to be used with the Semantic Kernel - - - - com.microsoft.semantic-kernel - semantickernel-api - - - com.fasterxml.jackson.core - jackson-databind - compile - - - com.fasterxml.jackson.core - jackson-core - compile - - - - diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml new file mode 100644 index 00000000..7aeca60d --- /dev/null +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -0,0 +1,39 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.3.1-SNAPSHOT + ../../pom.xml + + + semantickernel-data-azureaisearch + Semantic Kernel Azure AI Search connector + Azure AI Search connector for Semantic Kernel + + + + com.azure + azure-ai-openai + + + com.microsoft.semantic-kernel + semantickernel-api + + + + com.azure + azure-search-documents + + + com.azure + azure-core-serializer-json-jackson + + + + + + \ No newline at end of file diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java b/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStore.java similarity index 98% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java rename to data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStore.java index e1e64048..66dbb3c6 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java +++ b/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStore.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.azureaisearch; +package com.microsoft.semantickernel.data.azureaisearch; import com.azure.search.documents.indexes.SearchIndexAsyncClient; import com.azure.search.documents.indexes.models.SearchIndex; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java b/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java similarity index 99% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java rename to data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java index 996790f3..1b12b0ad 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java +++ b/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.azureaisearch; +package com.microsoft.semantickernel.data.azureaisearch; import com.azure.search.documents.indexes.models.ExhaustiveKnnAlgorithmConfiguration; import com.azure.search.documents.indexes.models.ExhaustiveKnnParameters; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java b/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java similarity index 98% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java rename to data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java index 4abbe1fb..c31af301 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java +++ b/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.azureaisearch; +package com.microsoft.semantickernel.data.azureaisearch; import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; import com.microsoft.semantickernel.data.filter.EqualToFilterClause; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreOptions.java b/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreOptions.java similarity index 97% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreOptions.java rename to data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreOptions.java index 2a5d71b8..23e907f2 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreOptions.java +++ b/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreOptions.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.azureaisearch; +package com.microsoft.semantickernel.data.azureaisearch; import javax.annotation.Nullable; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java similarity index 99% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java rename to data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index cbcbddfc..c4f44c42 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.azureaisearch; +package com.microsoft.semantickernel.data.azureaisearch; import com.azure.search.documents.SearchAsyncClient; import com.azure.search.documents.SearchDocument; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java b/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java similarity index 93% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java rename to data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java index 6856b3c1..ada0f70b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java +++ b/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.azureaisearch; +package com.microsoft.semantickernel.data.azureaisearch; import com.azure.search.documents.indexes.SearchIndexAsyncClient; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java b/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java similarity index 98% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java rename to data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java index e62f1020..2cd14360 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java +++ b/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.azureaisearch; +package com.microsoft.semantickernel.data.azureaisearch; import com.azure.search.documents.SearchDocument; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml new file mode 100644 index 00000000..e2a7c699 --- /dev/null +++ b/data/semantickernel-data-jdbc/pom.xml @@ -0,0 +1,69 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.3.1-SNAPSHOT + ../../pom.xml + + + semantickernel-data-jdbc + Semantic Kernel JDBC connector + Provides a JDBC connector for the Semantic Kernel + + + + com.microsoft.semantic-kernel + semantickernel-api + + + + org.slf4j + slf4j-api + + + com.fasterxml.jackson.core + jackson-databind + compile + + + com.fasterxml.jackson.core + jackson-core + compile + + + com.github.jknack + handlebars + + + com.google.code.findbugs + jsr305 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + compile + + + com.github.spotbugs + spotbugs-annotations + + + org.apache.commons + commons-text + + + org.postgresql + postgresql + 42.7.4 + + + org.xerial + sqlite-jdbc + 3.47.0.0 + + + \ No newline at end of file diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStore.java similarity index 99% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java rename to data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStore.java index 3d045e58..ddc2f750 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStore.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.jdbc; +package com.microsoft.semantickernel.data.jdbc; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreOptions.java similarity index 98% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java rename to data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreOptions.java index db2e6c8d..729ebbbd 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreOptions.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.jdbc; +package com.microsoft.semantickernel.data.jdbc; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java similarity index 99% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java rename to data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java index 1994475c..c1cc6273 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -1,11 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.jdbc; +package com.microsoft.semantickernel.data.jdbc; import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; import com.microsoft.semantickernel.data.filter.EqualToFilterClause; import com.microsoft.semantickernel.data.vectorsearch.VectorOperations; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; -import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; @@ -36,7 +35,6 @@ import javax.sql.DataSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import reactor.core.publisher.Mono; /** * A JDBC vector store query provider. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java similarity index 96% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java rename to data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java index 28818a97..fdcdd6d3 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -1,13 +1,11 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.jdbc; +package com.microsoft.semantickernel.data.jdbc; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreRecordMapper; -import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.jdbc.mysql.MySQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; -import com.microsoft.semantickernel.data.vectorsearch.VectorizedSearch; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java similarity index 94% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java rename to data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java index 50063627..796f0720 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.jdbc; +package com.microsoft.semantickernel.data.jdbc; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java similarity index 91% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java rename to data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java index 5d2e439c..596d0f29 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.jdbc; +package com.microsoft.semantickernel.data.jdbc; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; @@ -9,9 +9,8 @@ import java.sql.ResultSet; -import static com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider.validateSQLidentifier; -import static com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider.DEFAULT_COLLECTIONS_TABLE; -import static com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider.DEFAULT_PREFIX_FOR_COLLECTION_TABLES; +import static com.microsoft.semantickernel.data.jdbc.SQLVectorStoreQueryProvider.DEFAULT_COLLECTIONS_TABLE; +import static com.microsoft.semantickernel.data.jdbc.SQLVectorStoreQueryProvider.DEFAULT_PREFIX_FOR_COLLECTION_TABLES; /** * Options for a JDBC vector store record collection. @@ -169,7 +168,8 @@ public Builder withQueryProvider(SQLVectorStoreQueryProvider queryProvid * @return the builder */ public Builder withCollectionsTableName(String collectionsTableName) { - this.collectionsTableName = validateSQLidentifier(collectionsTableName); + this.collectionsTableName = JDBCVectorStoreQueryProvider + .validateSQLidentifier(collectionsTableName); return this; } @@ -179,7 +179,8 @@ public Builder withCollectionsTableName(String collectionsTableName) { * @return the builder */ public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { - this.prefixForCollectionTables = validateSQLidentifier(prefixForCollectionTables); + this.prefixForCollectionTables = JDBCVectorStoreQueryProvider + .validateSQLidentifier(prefixForCollectionTables); return this; } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordMapper.java similarity index 98% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java rename to data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordMapper.java index 880d93df..b52071cf 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordMapper.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.jdbc; +package com.microsoft.semantickernel.data.jdbc; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -15,7 +15,6 @@ import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.sql.ResultSetMetaData; import java.util.List; import java.sql.ResultSet; import java.sql.SQLException; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStore.java similarity index 88% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java rename to data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStore.java index 5fc68a64..08d167b8 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStore.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.jdbc; +package com.microsoft.semantickernel.data.jdbc; import com.microsoft.semantickernel.data.vectorstorage.VectorStore; import reactor.core.publisher.Mono; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreFilterQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreFilterQueryProvider.java similarity index 91% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreFilterQueryProvider.java rename to data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreFilterQueryProvider.java index 330df60d..4feb272f 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreFilterQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreFilterQueryProvider.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.jdbc; +package com.microsoft.semantickernel.data.jdbc; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.filter.FilterMapping; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java similarity index 96% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java rename to data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java index 04de1949..18806c5d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java @@ -1,9 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.jdbc; +package com.microsoft.semantickernel.data.jdbc; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.data.filter.FilterClause; -import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreRecordCollection.java similarity index 91% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java rename to data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreRecordCollection.java index 120d0273..03fe3e07 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreRecordCollection.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.jdbc; +package com.microsoft.semantickernel.data.jdbc; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import reactor.core.publisher.Mono; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/hsqldb/HSQLDBVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java similarity index 97% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/hsqldb/HSQLDBVectorStoreQueryProvider.java rename to data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java index 730458a5..48431316 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/hsqldb/HSQLDBVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java @@ -1,19 +1,18 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.hsqldb; +package com.microsoft.semantickernel.data.jdbc.hsqldb; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreQueryProvider; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.lang.reflect.Field; + import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/mysql/MySQLVectorStoreQueryProvider.java similarity index 96% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java rename to data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/mysql/MySQLVectorStoreQueryProvider.java index 89e8f857..feb6dc4f 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/mysql/MySQLVectorStoreQueryProvider.java @@ -1,11 +1,11 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.mysql; +package com.microsoft.semantickernel.data.jdbc.mysql; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; -import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.SQLVectorStoreQueryProvider; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorDistanceFunction.java similarity index 96% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java rename to data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorDistanceFunction.java index 669f3652..4ad2bd3b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorDistanceFunction.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorDistanceFunction.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.postgres; +package com.microsoft.semantickernel.data.jdbc.postgres; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorIndexKind.java similarity index 95% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java rename to data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorIndexKind.java index 6681ef83..cd0681b8 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorIndexKind.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorIndexKind.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.postgres; +package com.microsoft.semantickernel.data.jdbc.postgres; import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java similarity index 98% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java rename to data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java index 200edafa..4734f484 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -1,11 +1,11 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.postgres; +package com.microsoft.semantickernel.data.jdbc.postgres; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; -import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.SQLVectorStoreQueryProvider; import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; import com.microsoft.semantickernel.data.filter.EqualToFilterClause; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreRecordMapper.java similarity index 99% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java rename to data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreRecordMapper.java index 4b784a0a..11ced0e1 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreRecordMapper.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.postgres; +package com.microsoft.semantickernel.data.jdbc.postgres; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/sqlite/SQLiteVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java similarity index 96% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/sqlite/SQLiteVectorStoreQueryProvider.java rename to data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java index b9776f19..3ef126e6 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/sqlite/SQLiteVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java @@ -1,11 +1,11 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.sqlite; +package com.microsoft.semantickernel.data.jdbc.sqlite; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; -import com.microsoft.semantickernel.connectors.data.jdbc.SQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.SQLVectorStoreQueryProvider; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; @@ -20,7 +20,6 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.List; -import java.util.stream.Collectors; /** * A query provider for a vector store in SQLite. diff --git a/connectors/semantickernel-connectors-memory-redis/pom.xml b/data/semantickernel-data-redis/pom.xml similarity index 55% rename from connectors/semantickernel-connectors-memory-redis/pom.xml rename to data/semantickernel-data-redis/pom.xml index 463e076f..3666ca23 100644 --- a/connectors/semantickernel-connectors-memory-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -1,23 +1,28 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 0.2.14-alpha-SNAPSHOT + 1.3.1-SNAPSHOT ../../pom.xml - semantickernel-connectors-memory-redis - - Semantic Kernel Redis Memory Connector - Defines connector for Redis to be used with the Semantic Kernel + semantickernel-data-redis + Semantic Kernel Redis connector + Provides a Redis connector for the Semantic Kernel - com.fasterxml.jackson.core - jackson-core - compile + com.microsoft.semantic-kernel + semantickernel-api + + + + org.slf4j + slf4j-api com.fasterxml.jackson.core @@ -25,34 +30,36 @@ compile - com.microsoft.semantic-kernel - semantickernel-api - - - org.mockito - mockito-core - test + com.fasterxml.jackson.core + jackson-core + compile - org.junit.jupiter - junit-jupiter - test + com.github.jknack + handlebars - redis.clients - jedis - 5.2.0-beta5 - provided + com.google.code.findbugs + jsr305 - com.microsoft.semantic-kernel - semantickernel-connectors-memory-jdbc - ${project.version} + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml compile com.github.spotbugs spotbugs-annotations + + org.apache.commons + commons-text + + + + redis.clients + jedis + + \ No newline at end of file diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisHashSetVectorStoreRecordCollection.java similarity index 99% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java rename to data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisHashSetVectorStoreRecordCollection.java index 47c15a8e..74dd1213 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollection.java +++ b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisHashSetVectorStoreRecordCollection.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.redis; +package com.microsoft.semantickernel.data.redis; import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java similarity index 99% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java rename to data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java index 071f40fc..c4730d2a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java +++ b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisHashSetVectorStoreRecordCollectionOptions.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.redis; +package com.microsoft.semantickernel.data.redis; import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisHashSetVectorStoreRecordMapper.java similarity index 95% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java rename to data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisHashSetVectorStoreRecordMapper.java index 25678ef2..5ba97dfe 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisHashSetVectorStoreRecordMapper.java +++ b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisHashSetVectorStoreRecordMapper.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.redis; +package com.microsoft.semantickernel.data.redis; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -10,8 +10,6 @@ import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; @@ -28,7 +26,7 @@ import java.util.function.BiFunction; import java.util.function.Function; -import static com.microsoft.semantickernel.connectors.data.redis.RedisHashSetVectorStoreRecordCollection.stringToBytes; +import static com.microsoft.semantickernel.data.redis.RedisHashSetVectorStoreRecordCollection.stringToBytes; /** * A mapper to convert between a record and a Redis hash set storage model. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisJsonVectorStoreRecordCollection.java similarity index 99% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java rename to data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisJsonVectorStoreRecordCollection.java index 2a747684..50db6bd9 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollection.java +++ b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisJsonVectorStoreRecordCollection.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.redis; +package com.microsoft.semantickernel.data.redis; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -44,7 +44,6 @@ import redis.clients.jedis.search.FTSearchParams; import redis.clients.jedis.search.IndexDefinition; import redis.clients.jedis.search.IndexOptions; -import redis.clients.jedis.search.Query; import redis.clients.jedis.search.Schema; import redis.clients.jedis.search.SearchResult; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java similarity index 99% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java rename to data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java index cd2da3b9..0ba8b901 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java +++ b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisJsonVectorStoreRecordCollectionOptions.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.redis; +package com.microsoft.semantickernel.data.redis; import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisJsonVectorStoreRecordMapper.java similarity index 98% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java rename to data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisJsonVectorStoreRecordMapper.java index ecb449fb..f0de10bb 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisJsonVectorStoreRecordMapper.java +++ b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisJsonVectorStoreRecordMapper.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.redis; +package com.microsoft.semantickernel.data.redis; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisStorageType.java b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisStorageType.java similarity index 81% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisStorageType.java rename to data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisStorageType.java index c2f7d753..066e6524 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisStorageType.java +++ b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisStorageType.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.redis; +package com.microsoft.semantickernel.data.redis; /** * The storage type for the Redis vector store. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisVectorStore.java similarity index 98% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java rename to data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisVectorStore.java index aa901e02..6a0f1cee 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java +++ b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisVectorStore.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.redis; +package com.microsoft.semantickernel.data.redis; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.vectorstorage.VectorStore; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisVectorStoreCollectionCreateMapping.java similarity index 99% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java rename to data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisVectorStoreCollectionCreateMapping.java index 46a4c09b..05222e25 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionCreateMapping.java +++ b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisVectorStoreCollectionCreateMapping.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.redis; +package com.microsoft.semantickernel.data.redis; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisVectorStoreCollectionSearchMapping.java similarity index 99% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java rename to data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisVectorStoreCollectionSearchMapping.java index 2064411a..f029f678 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreCollectionSearchMapping.java +++ b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisVectorStoreCollectionSearchMapping.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.redis; +package com.microsoft.semantickernel.data.redis; import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; import com.microsoft.semantickernel.data.filter.EqualToFilterClause; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreOptions.java b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisVectorStoreOptions.java similarity index 98% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreOptions.java rename to data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisVectorStoreOptions.java index 338cf193..73093027 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreOptions.java +++ b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisVectorStoreOptions.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.redis; +package com.microsoft.semantickernel.data.redis; import com.microsoft.semantickernel.exceptions.SKException; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisVectorStoreRecordCollectionFactory.java similarity index 94% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java rename to data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisVectorStoreRecordCollectionFactory.java index f47fb4c8..bb1ab9a4 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java +++ b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisVectorStoreRecordCollectionFactory.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.redis; +package com.microsoft.semantickernel.data.redis; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; diff --git a/pom.xml b/pom.xml index 1c179fdf..87a03ac9 100644 --- a/pom.xml +++ b/pom.xml @@ -75,6 +75,9 @@ aiservices/openai aiservices/google aiservices/huggingface + data/semantickernel-data-azureaisearch + data/semantickernel-data-jdbc + data/semantickernel-data-redis diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 43266da4..29049770 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -30,6 +30,20 @@ com.microsoft.semantic-kernel semantickernel-api + + + com.microsoft.semantic-kernel + semantickernel-data-azureaisearch + + + com.microsoft.semantic-kernel + semantickernel-data-jdbc + + + com.microsoft.semantic-kernel + semantickernel-data-redis + + com.microsoft.semantic-kernel semantickernel-experimental diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java index 00166005..513ab72f 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemoryVolatileVectorStore.java @@ -5,15 +5,12 @@ import com.azure.ai.openai.OpenAIClientBuilder; import com.azure.core.credential.AzureKeyCredential; import com.azure.core.credential.KeyCredential; -import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; import com.microsoft.semantickernel.data.textsearch.TextSearchResultValue; -import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.VolatileVectorStore; import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.vectorstorage.VectorStoreTextSearch; -import com.microsoft.semantickernel.data.vectorstorage.VectorStoreTextSearchOptions; +import com.microsoft.semantickernel.data.VectorStoreTextSearch; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java index 64575a02..9b2e70ac 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithAzureAISearch.java @@ -5,21 +5,16 @@ import com.azure.ai.openai.OpenAIClientBuilder; import com.azure.core.credential.AzureKeyCredential; import com.azure.core.credential.KeyCredential; -import com.azure.core.util.ClientOptions; -import com.azure.core.util.MetricsOptions; -import com.azure.core.util.TracingOptions; import com.azure.search.documents.indexes.SearchIndexAsyncClient; import com.azure.search.documents.indexes.SearchIndexClientBuilder; import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; -import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStore; -import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreOptions; -import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.azureaisearch.AzureAISearchVectorStore; +import com.microsoft.semantickernel.data.azureaisearch.AzureAISearchVectorStoreOptions; +import com.microsoft.semantickernel.data.azureaisearch.AzureAISearchVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.textsearch.TextSearchResultValue; -import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; -import com.microsoft.semantickernel.data.vectorsearch.VectorizedSearch; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.vectorstorage.VectorStoreTextSearch; +import com.microsoft.semantickernel.data.VectorStoreTextSearch; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java index 1bf7ba30..3a393ad2 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithJDBC.java @@ -6,14 +6,13 @@ import com.azure.core.credential.AzureKeyCredential; import com.azure.core.credential.KeyCredential; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; -import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStore; -import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions; -import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.jdbc.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.data.textsearch.TextSearchResultValue; -import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.vectorstorage.VectorStoreTextSearch; +import com.microsoft.semantickernel.data.VectorStoreTextSearch; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java index 63c03271..b037f6f7 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithRedis.java @@ -5,16 +5,14 @@ import com.azure.ai.openai.OpenAIClientBuilder; import com.azure.core.credential.AzureKeyCredential; import com.azure.core.credential.KeyCredential; -import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; -import com.microsoft.semantickernel.connectors.data.redis.RedisJsonVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.connectors.data.redis.RedisStorageType; -import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStore; -import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreOptions; +import com.microsoft.semantickernel.data.redis.RedisJsonVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.redis.RedisStorageType; +import com.microsoft.semantickernel.data.redis.RedisVectorStore; +import com.microsoft.semantickernel.data.redis.RedisVectorStoreOptions; import com.microsoft.semantickernel.data.textsearch.TextSearchResultValue; -import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.vectorstorage.VectorStoreTextSearch; +import com.microsoft.semantickernel.data.VectorStoreTextSearch; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreTextSearch.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearch.java similarity index 98% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreTextSearch.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearch.java index 46684f5a..7ae38111 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreTextSearch.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearch.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorstorage; +package com.microsoft.semantickernel.data; import com.microsoft.semantickernel.data.textsearch.DefaultTextSearchResultMapper; import com.microsoft.semantickernel.data.textsearch.DefaultTextSearchStringMapper; @@ -10,7 +10,6 @@ import com.microsoft.semantickernel.data.textsearch.TextSearchResultMapper; import com.microsoft.semantickernel.data.textsearch.TextSearchStringMapper; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; -import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorsearch.VectorizedSearch; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreTextSearchOptions.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearchOptions.java similarity index 71% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreTextSearchOptions.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearchOptions.java index 2f0a73e9..80a468ce 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreTextSearchOptions.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearchOptions.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.data.vectorstorage; +package com.microsoft.semantickernel.data; /** * Options for vector store text search. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java similarity index 99% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java index 5be45985..a8ef8a65 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java @@ -5,7 +5,6 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.data.vectorsearch.VectorOperations; -import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java similarity index 98% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java index a74dbf23..ad926e8f 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.vectorsearch; -import javax.annotation.Nullable; import java.util.Collections; import java.util.List; import java.util.Map; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java similarity index 96% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java index 93aa883d..df4d38a8 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java @@ -4,8 +4,6 @@ import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import reactor.core.publisher.Mono; -import java.util.List; - /** * A vectorizable text search. * diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java similarity index 100% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java rename to semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java diff --git a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java b/semantickernel-api/src/test/java/com/microsoft/semantickernel/data/Hotel.java similarity index 100% rename from semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/Hotel.java rename to semantickernel-api/src/test/java/com/microsoft/semantickernel/data/Hotel.java diff --git a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java b/semantickernel-api/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java similarity index 100% rename from semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java rename to semantickernel-api/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java diff --git a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreTest.java b/semantickernel-api/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreTest.java similarity index 100% rename from semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreTest.java rename to semantickernel-api/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreTest.java diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 1dce5d04..922338b5 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -96,6 +96,21 @@ semantickernel-aiservices-google ${project.version} + + com.microsoft.semantic-kernel + semantickernel-data-azureaisearch + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-data-jdbc + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-data-redis + ${project.version} + com.azure From 818917af0cc661e3e7b38b53661ef50535310743 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Fri, 25 Oct 2024 12:51:04 +0100 Subject: [PATCH 086/277] Bump azure-ai-openai --- CHANGELOG.md | 4 ++++ pom.xml | 3 +-- semantickernel-bom/pom.xml | 3 +-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8859c23f..93186239 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 1.3.1 + +- Upgrade to azure-ai-openai 1.0.0-beta.12 + # 1.3.0 - Added support for Json Schema to Open AI Chat Completions diff --git a/pom.xml b/pom.xml index 1c179fdf..ebacfbf5 100644 --- a/pom.xml +++ b/pom.xml @@ -14,8 +14,7 @@ Parent pom for the Semantic Kernel Project - - 1.0.0-beta.11 + 1.0.0-beta.12 10.18.2 0.10.21 false diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 1dce5d04..875caab5 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -100,8 +100,7 @@ com.azure azure-ai-openai - - 1.0.0-beta.11 + 1.0.0-beta.12 com.azure From 6a6dece25f5ae2b660e9b2e00c851e351ee40c59 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Fri, 25 Oct 2024 13:36:47 +0100 Subject: [PATCH 087/277] Fix tests --- .../chatcompletion/BinaryDataUtils.java | 16 ++++++++++ .../chatcompletion/OpenAIChatCompletion.java | 29 ++++++++++++++----- .../chatcompletion/OpenAiXMLPromptParser.java | 14 ++++++--- .../semantickernel/tests/RenderingTest.java | 8 ++--- .../tests/ToolCallBehaviourTest.java | 4 +-- ...-12b6744e-443f-4fe7-82e2-55cc41195ff1.json | 2 +- ...-71b07779-49a8-44e5-a60b-ee5b0a3ad697.json | 2 +- ...-0c8046c5-74ad-4836-8aa9-09da60f367a2.json | 2 +- ...-0215b128-4822-4368-ac3d-2f580a221f00.json | 2 +- 9 files changed, 57 insertions(+), 22 deletions(-) create mode 100644 aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/BinaryDataUtils.java diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/BinaryDataUtils.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/BinaryDataUtils.java new file mode 100644 index 00000000..9a3189a7 --- /dev/null +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/BinaryDataUtils.java @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.aiservices.openai.chatcompletion; + +import com.azure.core.util.BinaryData; +import javax.annotation.Nullable; + +public class BinaryDataUtils { + + @Nullable + public static String toString(@Nullable BinaryData b) { + if (b == null) { + return null; + } + return b.toString(); + } +} diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java index 919ff995..6d822687 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java @@ -6,6 +6,7 @@ import com.azure.ai.openai.models.ChatCompletions; import com.azure.ai.openai.models.ChatCompletionsFunctionToolCall; import com.azure.ai.openai.models.ChatCompletionsFunctionToolDefinition; +import com.azure.ai.openai.models.ChatCompletionsFunctionToolDefinitionFunction; import com.azure.ai.openai.models.ChatCompletionsJsonResponseFormat; import com.azure.ai.openai.models.ChatCompletionsNamedToolSelection; import com.azure.ai.openai.models.ChatCompletionsOptions; @@ -26,6 +27,7 @@ import com.azure.ai.openai.models.ChatResponseMessage; import com.azure.ai.openai.models.CompletionsUsage; import com.azure.ai.openai.models.FunctionCall; +import com.azure.ai.openai.models.FunctionDefinition; import com.azure.json.JsonOptions; import com.azure.json.implementation.DefaultJsonReader; import com.fasterxml.jackson.core.JsonProcessingException; @@ -734,7 +736,8 @@ private List> toOpenAIChatMessageContent( if (message instanceof ChatRequestUserMessage) { return new OpenAIChatMessageContent<>( AuthorRole.USER, - ((ChatRequestUserMessage) message).getContent().toString(), + BinaryDataUtils + .toString(((ChatRequestUserMessage) message).getContent()), null, null, null, @@ -743,7 +746,8 @@ private List> toOpenAIChatMessageContent( } else if (message instanceof ChatRequestSystemMessage) { return new OpenAIChatMessageContent<>( AuthorRole.SYSTEM, - ((ChatRequestSystemMessage) message).getContent(), + BinaryDataUtils + .toString(((ChatRequestSystemMessage) message).getContent()), null, null, null, @@ -755,7 +759,8 @@ private List> toOpenAIChatMessageContent( ((ChatRequestAssistantMessage) message).getToolCalls()); return new OpenAIChatMessageContent<>( AuthorRole.ASSISTANT, - ((ChatRequestAssistantMessage) message).getContent(), + BinaryDataUtils + .toString(((ChatRequestAssistantMessage) message).getContent()), null, null, null, @@ -767,7 +772,8 @@ private List> toOpenAIChatMessageContent( } else if (message instanceof ChatRequestToolMessage) { return new OpenAIChatMessageContent<>( AuthorRole.TOOL, - ((ChatRequestToolMessage) message).getContent(), + BinaryDataUtils + .toString(((ChatRequestToolMessage) message).getContent()), null, null, null, @@ -966,10 +972,14 @@ private static void configureToolCallBehaviorOptions( List toolDefinitions = new ArrayList<>(); + FunctionDefinition function = OpenAIFunction.toFunctionDefinition( + toolChoice.getMetadata(), + toolChoice.getPluginName()); + toolDefinitions.add(new ChatCompletionsFunctionToolDefinition( - OpenAIFunction.toFunctionDefinition( - toolChoice.getMetadata(), - toolChoice.getPluginName()))); + new ChatCompletionsFunctionToolDefinitionFunction(function.getName()) + .setDescription(function.getDescription()) + .setParameters(function.getParameters()))); options.setTools(toolDefinitions); try { @@ -1003,7 +1013,10 @@ private static void configureToolCallBehaviorOptions( function.getName()); }) .map(OpenAIFunction::getFunctionDefinition) - .map(ChatCompletionsFunctionToolDefinition::new) + .map(it -> new ChatCompletionsFunctionToolDefinitionFunction(it.getName()) + .setDescription(it.getDescription()) + .setParameters(it.getParameters())) + .map(it -> new ChatCompletionsFunctionToolDefinition(it)) .collect(Collectors.toList()); if (toolDefinitions.isEmpty()) { diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAiXMLPromptParser.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAiXMLPromptParser.java index 4088f72f..4f5fd99c 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAiXMLPromptParser.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAiXMLPromptParser.java @@ -118,19 +118,23 @@ public static ChatRequestMessage unescapeRequest(ChatRequestMessage message) { if (message instanceof ChatRequestUserMessage) { ChatRequestUserMessage chatRequestMessage = (ChatRequestUserMessage) message; String content = StringEscapeUtils.unescapeXml( - chatRequestMessage.getContent().toString()); + BinaryDataUtils.toString(chatRequestMessage.getContent())); return new ChatRequestUserMessage(content) .setName(chatRequestMessage.getName()); } else if (message instanceof ChatRequestSystemMessage) { ChatRequestSystemMessage chatRequestMessage = (ChatRequestSystemMessage) message; - String content = StringEscapeUtils.unescapeXml(chatRequestMessage.getContent()); + String content = StringEscapeUtils + .unescapeXml( + BinaryDataUtils.toString(chatRequestMessage.getContent())); return new ChatRequestSystemMessage(content) .setName(chatRequestMessage.getName()); } else if (message instanceof ChatRequestAssistantMessage) { ChatRequestAssistantMessage chatRequestMessage = (ChatRequestAssistantMessage) message; - String content = StringEscapeUtils.unescapeXml(chatRequestMessage.getContent()); + String content = StringEscapeUtils + .unescapeXml( + BinaryDataUtils.toString(chatRequestMessage.getContent())); return new ChatRequestAssistantMessage(content) .setToolCalls(chatRequestMessage.getToolCalls()) @@ -145,7 +149,9 @@ public static ChatRequestMessage unescapeRequest(ChatRequestMessage message) { content); } else if (message instanceof ChatRequestToolMessage) { ChatRequestToolMessage chatRequestMessage = (ChatRequestToolMessage) message; - String content = StringEscapeUtils.unescapeXml(chatRequestMessage.getContent()); + String content = StringEscapeUtils + .unescapeXml( + BinaryDataUtils.toString(chatRequestMessage.getContent())); return new ChatRequestToolMessage( content, diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/RenderingTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/RenderingTest.java index 4c24eb92..3f09b50b 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/RenderingTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/RenderingTest.java @@ -240,10 +240,10 @@ public void canHandleIterableReturnFromFunction() { .block(); String requestBody = wm.getAllServeEvents().get(0).getRequest().getBodyAsString(); - Assertions.assertTrue(requestBody.contains("{\"content\":\"[]")); - Assertions.assertTrue(requestBody.contains("{\"content\":\"[1, 2, 3]")); - Assertions.assertTrue(requestBody.contains("{\"content\":\"[4, 5, 6]")); - Assertions.assertTrue(requestBody.contains("{\"content\":\"[7, 8, 9]")); + Assertions.assertTrue(requestBody.contains("\"content\":\"[]")); + Assertions.assertTrue(requestBody.contains("\"content\":\"[1, 2, 3]")); + Assertions.assertTrue(requestBody.contains("\"content\":\"[4, 5, 6]")); + Assertions.assertTrue(requestBody.contains("\"content\":\"[7, 8, 9]")); } private Kernel buildTextKernel() { diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ToolCallBehaviourTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ToolCallBehaviourTest.java index 4f96f782..09f88df0 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ToolCallBehaviourTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ToolCallBehaviourTest.java @@ -282,7 +282,7 @@ private ChatCompletionService getChatCompletionService() { public static MappingBuilder buildTextResponse(String bodyMatcher, String responseBody) { return post(urlEqualTo( - "//openai/deployments/gpt-35-turbo-2/completions?api-version=2024-07-01-preview")) + "//openai/deployments/gpt-35-turbo-2/completions?api-version=2024-08-01-preview")) .withRequestBody(new ContainsPattern(bodyMatcher)) .willReturn( aResponse() @@ -293,7 +293,7 @@ public static MappingBuilder buildTextResponse(String bodyMatcher, String respon public static MappingBuilder buildResponse(String bodyMatcher, String responseBody) { return post(urlEqualTo( - "//openai/deployments/gpt-35-turbo-2/chat/completions?api-version=2024-07-01-preview")) + "//openai/deployments/gpt-35-turbo-2/chat/completions?api-version=2024-08-01-preview")) .withRequestBody(new ContainsPattern(bodyMatcher)) .willReturn( aResponse() diff --git a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-12b6744e-443f-4fe7-82e2-55cc41195ff1.json b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-12b6744e-443f-4fe7-82e2-55cc41195ff1.json index 9697bca8..cdc9cbd0 100644 --- a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-12b6744e-443f-4fe7-82e2-55cc41195ff1.json +++ b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-12b6744e-443f-4fe7-82e2-55cc41195ff1.json @@ -2,7 +2,7 @@ "priority": 1, "request": { "method": "POST", - "url": "//openai/deployments/gpt-35-turbo-2/chat/completions?api-version=2024-07-01-preview", + "url": "//openai/deployments/gpt-35-turbo-2/chat/completions?api-version=2024-08-01-preview", "bodyPatterns": [ { "contains": "That is all" diff --git a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-71b07779-49a8-44e5-a60b-ee5b0a3ad697.json b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-71b07779-49a8-44e5-a60b-ee5b0a3ad697.json index c99d0145..0d71a457 100644 --- a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-71b07779-49a8-44e5-a60b-ee5b0a3ad697.json +++ b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-71b07779-49a8-44e5-a60b-ee5b0a3ad697.json @@ -1,7 +1,7 @@ { "request": { "method": "POST", - "url": "//openai/deployments/gpt-35-turbo-2/chat/completions?api-version=2024-07-01-preview" + "url": "//openai/deployments/gpt-35-turbo-2/chat/completions?api-version=2024-08-01-preview" }, "response": { "body": "{\"id\":\"chatcmpl-xxx\",\"object\":\"chat.completion\",\"created\":1707253061,\"model\":\"gpt-35-turbo\",\"prompt_filter_results\":[{\"prompt_index\":0,\"content_filter_results\":{\"hate\":{\"filtered\":false,\"severity\":\"safe\"},\"self_harm\":{\"filtered\":false,\"severity\":\"safe\"},\"sexual\":{\"filtered\":false,\"severity\":\"safe\"},\"violence\":{\"filtered\":false,\"severity\":\"safe\"}}}],\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"1979b4ce-5463-4cfb-8ec8-1d05c4b44ccf\"},\"content_filter_results\":{\"hate\":{\"filtered\":false,\"severity\":\"safe\"},\"self_harm\":{\"filtered\":false,\"severity\":\"safe\"},\"sexual\":{\"filtered\":false,\"severity\":\"safe\"},\"violence\":{\"filtered\":false,\"severity\":\"safe\"}}}],\"usage\":{\"prompt_tokens\":17,\"completion_tokens\":67,\"total_tokens\":84}}", diff --git a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo_chat_completions-0c8046c5-74ad-4836-8aa9-09da60f367a2.json b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo_chat_completions-0c8046c5-74ad-4836-8aa9-09da60f367a2.json index d55d6bae..53becbdf 100644 --- a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo_chat_completions-0c8046c5-74ad-4836-8aa9-09da60f367a2.json +++ b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo_chat_completions-0c8046c5-74ad-4836-8aa9-09da60f367a2.json @@ -1,7 +1,7 @@ { "request": { "method": "POST", - "url": "//openai/deployments/gpt-35-turbo/chat/completions?api-version=2024-07-01-preview" + "url": "//openai/deployments/gpt-35-turbo/chat/completions?api-version=2024-08-01-preview" }, "response": { "body": "{\"id\":\"chatcmpl-xxx\",\"object\":\"chat.completion\",\"created\":1707253039,\"model\":\"gpt-35-turbo\",\"prompt_filter_results\":[{\"prompt_index\":0,\"content_filter_results\":{\"hate\":{\"filtered\":false,\"severity\":\"safe\"},\"self_harm\":{\"filtered\":false,\"severity\":\"safe\"},\"sexual\":{\"filtered\":false,\"severity\":\"safe\"},\"violence\":{\"filtered\":false,\"severity\":\"safe\"}}}],\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"ac9817bc-7e1a-48e4-b06c-0ff7618b88c6\"},\"content_filter_results\":{\"hate\":{\"filtered\":false,\"severity\":\"safe\"},\"self_harm\":{\"filtered\":false,\"severity\":\"safe\"},\"sexual\":{\"filtered\":false,\"severity\":\"safe\"},\"violence\":{\"filtered\":false,\"severity\":\"safe\"}}}],\"usage\":{\"prompt_tokens\":26,\"completion_tokens\":131,\"total_tokens\":157}}", diff --git a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_text-davinci-003_completions-0215b128-4822-4368-ac3d-2f580a221f00.json b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_text-davinci-003_completions-0215b128-4822-4368-ac3d-2f580a221f00.json index 15adeb40..7ac0518d 100644 --- a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_text-davinci-003_completions-0215b128-4822-4368-ac3d-2f580a221f00.json +++ b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_text-davinci-003_completions-0215b128-4822-4368-ac3d-2f580a221f00.json @@ -1,7 +1,7 @@ { "request": { "method": "POST", - "url": "//openai/deployments/text-davinci-003/completions?api-version=2024-07-01-preview" + "url": "//openai/deployments/text-davinci-003/completions?api-version=2024-08-01-preview" }, "response": { "body": "{\"id\":\"cmpl-xxx\",\"object\":\"text_completion\",\"created\":1707253062,\"model\":\"text-davinci-003\",\"prompt_filter_results\":[{\"prompt_index\":0,\"content_filter_results\":{\"hate\":{\"filtered\":false,\"severity\":\"safe\"},\"self_harm\":{\"filtered\":false,\"severity\":\"safe\"},\"sexual\":{\"filtered\":false,\"severity\":\"safe\"},\"violence\":{\"filtered\":false,\"severity\":\"safe\"}}}],\"choices\":[{\"text\":\"7949e981-c632-422f-9b76-335a2379cd83\",\"index\":0,\"finish_reason\":\"stop\",\"logprobs\":null,\"content_filter_results\":{\"hate\":{\"filtered\":false,\"severity\":\"safe\"},\"self_harm\":{\"filtered\":false,\"severity\":\"safe\"},\"sexual\":{\"filtered\":false,\"severity\":\"safe\"},\"violence\":{\"filtered\":false,\"severity\":\"safe\"}}}],\"usage\":{\"prompt_tokens\":10,\"completion_tokens\":26,\"total_tokens\":36}}", From 7d9e3f177059ff3b3d9e0b26da7fe6de9e1bb257 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Mon, 28 Oct 2024 00:58:23 -0700 Subject: [PATCH 088/277] Remove default dimensions for models that don't support it --- .../OpenAITextEmbeddingGenerationService.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textembedding/OpenAITextEmbeddingGenerationService.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textembedding/OpenAITextEmbeddingGenerationService.java index 89b52831..355f2ab9 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textembedding/OpenAITextEmbeddingGenerationService.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textembedding/OpenAITextEmbeddingGenerationService.java @@ -26,7 +26,6 @@ public class OpenAITextEmbeddingGenerationService extends OpenAiService> generateEmbeddingsAsync(List data) { protected Mono> internalGenerateTextEmbeddingsAsync(List data) { EmbeddingsOptions options = new EmbeddingsOptions(data) .setModel(getModelId()) - .setDimensions(dimensions) .setInputType("string"); + if (dimensions > 0) { + options.setDimensions(dimensions); + } return getClient() .getEmbeddings(getModelId(), options) @@ -118,7 +119,7 @@ protected Mono> internalGenerateTextEmbeddingsAsync(List public static class Builder extends OpenAiServiceBuilder { - private int dimensions = DEFAULT_DIMENSIONS; + private int dimensions = -1; /** * Sets the dimensions for the embeddings. From d7e2d142ea94024e7be01696fb0f592a67e70672 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 29 Oct 2024 09:02:19 -0700 Subject: [PATCH 089/277] Update version and CHANGELOG.md --- CHANGELOG.md | 3 ++- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- data/semantickernel-data-azureaisearch/pom.xml | 2 +- data/semantickernel-data-jdbc/pom.xml | 2 +- data/semantickernel-data-redis/pom.xml | 2 +- pom.xml | 2 +- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-demos/semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-bom/pom.xml | 2 +- semantickernel-experimental/pom.xml | 2 +- 24 files changed, 25 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93186239..40503073 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ -# 1.3.1 +# 1.4.0 - Upgrade to azure-ai-openai 1.0.0-beta.12 +- Add vector stores with vector search support for Azure AI Search, Redis, JDBC with Postgres, MySQL, SQLite and HSQLDB. Moving these features out of the experimental stage. # 1.3.0 diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index e0c334d1..4cf6a36b 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index e8a3f6a9..a8867b44 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index 56f93045..3ffe0ebd 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 1f2a87f8..fc9af8d3 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index e6924679..54987e9c 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT ../pom.xml diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml index 7aeca60d..f2cc6df3 100644 --- a/data/semantickernel-data-azureaisearch/pom.xml +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index e2a7c699..1550f543 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-redis/pom.xml b/data/semantickernel-data-redis/pom.xml index 3666ca23..7522fcdd 100644 --- a/data/semantickernel-data-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT ../../pom.xml diff --git a/pom.xml b/pom.xml index f4119f47..833ae3a2 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT pom https://www.github.com/microsoft/semantic-kernel diff --git a/samples/pom.xml b/samples/pom.xml index e3772428..67c96fc5 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index eebdadd3..b0e4e1b5 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 29049770..cf291283 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index de435a33..831685d3 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index 2e40f729..a92216ce 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index f6b51a70..4d4f04cb 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index 1bee5197..ae8a4fe8 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index d4b8aec4..16b233c7 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index a7bd5f6e..288a83f7 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index 0c1d8dbf..a340900b 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index c9b95005..e274f993 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index 82c7d252..b152811f 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index c1aba267..14af71ac 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT pom Semantic Kernel Java BOM diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 4c93f575..6f2e651b 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.3.1-SNAPSHOT + 1.4.0-SNAPSHOT semantickernel-experimental From 92055a464a88739aafaf93c836531fb660bc6096 Mon Sep 17 00:00:00 2001 From: GitHub Date: Tue, 29 Oct 2024 17:30:03 +0000 Subject: [PATCH 090/277] [maven-release-plugin] prepare release java-1.4.0 --- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- data/semantickernel-data-azureaisearch/pom.xml | 6 ++---- data/semantickernel-data-jdbc/pom.xml | 6 ++---- data/semantickernel-data-redis/pom.xml | 6 ++---- pom.xml | 4 ++-- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-bom/pom.xml | 4 ++-- semantickernel-experimental/pom.xml | 2 +- 23 files changed, 28 insertions(+), 34 deletions(-) diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index 4cf6a36b..294031e0 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.0-SNAPSHOT + 1.4.0 ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index a8867b44..4e360aec 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.0-SNAPSHOT + 1.4.0 ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index 3ffe0ebd..30e93da7 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.0-SNAPSHOT + 1.4.0 ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index fc9af8d3..4388d808 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.4.0-SNAPSHOT + 1.4.0 ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index 54987e9c..4def0edc 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.0-SNAPSHOT + 1.4.0 ../pom.xml diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml index f2cc6df3..4b8a1a24 100644 --- a/data/semantickernel-data-azureaisearch/pom.xml +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -1,12 +1,10 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.4.0-SNAPSHOT + 1.4.0 ../../pom.xml diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 1550f543..56ac156a 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -1,12 +1,10 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.4.0-SNAPSHOT + 1.4.0 ../../pom.xml diff --git a/data/semantickernel-data-redis/pom.xml b/data/semantickernel-data-redis/pom.xml index 7522fcdd..e84993c0 100644 --- a/data/semantickernel-data-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -1,12 +1,10 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.4.0-SNAPSHOT + 1.4.0 ../../pom.xml diff --git a/pom.xml b/pom.xml index 833ae3a2..33bcd61f 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.0-SNAPSHOT + 1.4.0 pom https://www.github.com/microsoft/semantic-kernel @@ -837,6 +837,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - HEAD + java-1.4.0 diff --git a/samples/pom.xml b/samples/pom.xml index 67c96fc5..90b9b53a 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.0-SNAPSHOT + 1.4.0 ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index b0e4e1b5..9a4ec818 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.0-SNAPSHOT + 1.4.0 ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index cf291283..ec001ea3 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.4.0-SNAPSHOT + 1.4.0 ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index 831685d3..39b70f25 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.0-SNAPSHOT + 1.4.0 ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index a92216ce..dfb070a9 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.0-SNAPSHOT + 1.4.0 ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index 4d4f04cb..9878ef8e 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.0-SNAPSHOT + 1.4.0 ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index ae8a4fe8..d5e17385 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.0-SNAPSHOT + 1.4.0 ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 16b233c7..03cbdf41 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.0-SNAPSHOT + 1.4.0 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index 288a83f7..f7fc1bc1 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.0-SNAPSHOT + 1.4.0 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index a340900b..60824adf 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.0-SNAPSHOT + 1.4.0 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index e274f993..2999c15a 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.0-SNAPSHOT + 1.4.0 ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index b152811f..87c2dbe6 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.0-SNAPSHOT + 1.4.0 ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 14af71ac..d147e0de 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.4.0-SNAPSHOT + 1.4.0 pom Semantic Kernel Java BOM @@ -290,6 +290,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - HEAD + java-1.4.0 diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 6f2e651b..b352112e 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.0-SNAPSHOT + 1.4.0 semantickernel-experimental From 07188f947c4051d3869923557684391a0bb795b8 Mon Sep 17 00:00:00 2001 From: GitHub Date: Tue, 29 Oct 2024 17:30:03 +0000 Subject: [PATCH 091/277] [maven-release-plugin] prepare for next development iteration --- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- data/semantickernel-data-azureaisearch/pom.xml | 2 +- data/semantickernel-data-jdbc/pom.xml | 2 +- data/semantickernel-data-redis/pom.xml | 2 +- pom.xml | 4 ++-- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-bom/pom.xml | 4 ++-- semantickernel-experimental/pom.xml | 2 +- 23 files changed, 25 insertions(+), 25 deletions(-) diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index 294031e0..187961bc 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.0 + 1.4.1-SNAPSHOT ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index 4e360aec..950cfa62 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.0 + 1.4.1-SNAPSHOT ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index 30e93da7..ab756185 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.0 + 1.4.1-SNAPSHOT ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 4388d808..49c5f286 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.4.0 + 1.4.1-SNAPSHOT ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index 4def0edc..c88c19b3 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.0 + 1.4.1-SNAPSHOT ../pom.xml diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml index 4b8a1a24..83ef7964 100644 --- a/data/semantickernel-data-azureaisearch/pom.xml +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.0 + 1.4.1-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 56ac156a..76d57be9 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.0 + 1.4.1-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-redis/pom.xml b/data/semantickernel-data-redis/pom.xml index e84993c0..7d26a385 100644 --- a/data/semantickernel-data-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.0 + 1.4.1-SNAPSHOT ../../pom.xml diff --git a/pom.xml b/pom.xml index 33bcd61f..e4ca4797 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.0 + 1.4.1-SNAPSHOT pom https://www.github.com/microsoft/semantic-kernel @@ -837,6 +837,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.4.0 + HEAD diff --git a/samples/pom.xml b/samples/pom.xml index 90b9b53a..9b1f4ea1 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.0 + 1.4.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index 9a4ec818..08f5522f 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.0 + 1.4.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index ec001ea3..51be5816 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.4.0 + 1.4.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index 39b70f25..b3862adc 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.0 + 1.4.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index dfb070a9..662da823 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.0 + 1.4.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index 9878ef8e..346d6a83 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.0 + 1.4.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index d5e17385..c8b40a25 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.0 + 1.4.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 03cbdf41..6978bdee 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.0 + 1.4.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index f7fc1bc1..f8e3d010 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.0 + 1.4.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index 60824adf..6aab17b8 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.0 + 1.4.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index 2999c15a..197cb0c4 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.0 + 1.4.1-SNAPSHOT ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index 87c2dbe6..d67acb88 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.0 + 1.4.1-SNAPSHOT ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index d147e0de..c74f0c3e 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.4.0 + 1.4.1-SNAPSHOT pom Semantic Kernel Java BOM @@ -290,6 +290,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.4.0 + HEAD diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index b352112e..2ef8f7cd 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.0 + 1.4.1-SNAPSHOT semantickernel-experimental From 0c2ce39a04264379dc7748e787f6d38a257365bd Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 31 Oct 2024 01:36:11 -0700 Subject: [PATCH 092/277] Update redis tests --- .../semantickernel/tests/data/redis/RedisVectorStoreTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisVectorStoreTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisVectorStoreTest.java index cfff9ea0..93ee1c20 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisVectorStoreTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisVectorStoreTest.java @@ -6,7 +6,6 @@ import com.microsoft.semantickernel.data.redis.RedisVectorStore; import com.microsoft.semantickernel.data.redis.RedisVectorStoreOptions; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.tests.data.jdbc.Hotel; import com.redis.testcontainers.RedisContainer; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; From df3defa14ef584147e517efa4323f2f26e8a20b1 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Thu, 31 Oct 2024 21:40:32 +0000 Subject: [PATCH 093/277] Add function calling telemetry --- .../chatcompletion/OpenAIChatCompletion.java | 46 +-- .../OpenAITextGenerationService.java | 11 - .../aiservices/openai/OtelCaptureTest.java | 51 --- .../semantickernel-syntax-examples/pom.xml | 6 + .../java/FunctionTelemetry_Example.java | 337 ++++++++++++++++++ .../java/KernelFunctionYaml_Example.java | 21 +- .../java/applicationinsights.json | 16 + semantickernel-api/pom.xml | 8 +- .../telemetry/ChatCompletionSpan.java | 116 ++++++ .../telemetry/FunctionSpan.java | 66 ++++ .../telemetry/SemanticKernelTelemetry.java | 89 ++--- .../SemanticKernelTelemetrySpan.java | 88 +++++ .../orchestration/FunctionInvocation.java | 22 +- .../orchestration/InvocationContext.java | 34 +- .../KernelFunctionFromMethod.java | 29 +- .../KernelFunctionFromPrompt.java | 20 +- 16 files changed, 805 insertions(+), 155 deletions(-) create mode 100644 samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/FunctionTelemetry_Example.java create mode 100644 samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/resources/com/microsoft/semantickernel/samples/syntaxexamples/java/applicationinsights.json create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/ChatCompletionSpan.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/FunctionSpan.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/SemanticKernelTelemetrySpan.java diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java index 6d822687..b842e43d 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java @@ -50,6 +50,7 @@ import com.microsoft.semantickernel.hooks.PreChatCompletionEvent; import com.microsoft.semantickernel.hooks.PreToolCallEvent; import com.microsoft.semantickernel.implementation.CollectionUtil; +import com.microsoft.semantickernel.implementation.telemetry.ChatCompletionSpan; import com.microsoft.semantickernel.implementation.telemetry.SemanticKernelTelemetry; import com.microsoft.semantickernel.orchestration.FunctionResult; import com.microsoft.semantickernel.orchestration.FunctionResultMetadata; @@ -69,7 +70,6 @@ import com.microsoft.semantickernel.services.chatcompletion.message.ChatMessageContentType; import com.microsoft.semantickernel.services.chatcompletion.message.ChatMessageImageContent; import com.microsoft.semantickernel.services.openai.OpenAiServiceBuilder; -import io.opentelemetry.api.trace.Span; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -419,26 +419,32 @@ private Mono internalChatMessageContentsAsync( invocationContext))) .getOptions(); - Span span = SemanticKernelTelemetry.startChatCompletionSpan( - getModelId(), - SemanticKernelTelemetry.OPEN_AI_PROVIDER, - options.getMaxTokens(), - options.getTemperature(), - options.getTopP()); - return getClient() - .getChatCompletionsWithResponse(getDeploymentName(), options, - OpenAIRequestSettings.getRequestOptions()) - .flatMap(completionsResult -> { - if (completionsResult.getStatusCode() >= 400) { - SemanticKernelTelemetry.endSpanWithError(span); - return Mono.error(new AIException(ErrorCodes.SERVICE_ERROR, - "Request failed: " + completionsResult.getStatusCode())); - } - SemanticKernelTelemetry.endSpanWithUsage(span, - completionsResult.getValue().getUsage()); + return Mono.deferContextual(contextView -> { + ChatCompletionSpan span = ChatCompletionSpan.startChatCompletionSpan( + SemanticKernelTelemetry.getTelemetry(invocationContext), + contextView, + getModelId(), + SemanticKernelTelemetry.OPEN_AI_PROVIDER, + options.getMaxTokens(), + options.getTemperature(), + options.getTopP()); + + return getClient() + .getChatCompletionsWithResponse(getDeploymentName(), options, + OpenAIRequestSettings.getRequestOptions()) + .contextWrite(span.getReactorContextModifier()) + .flatMap(completionsResult -> { + if (completionsResult.getStatusCode() >= 400) { + return Mono.error(new AIException(ErrorCodes.SERVICE_ERROR, + "Request failed: " + completionsResult.getStatusCode())); + } - return Mono.just(completionsResult.getValue()); - }) + return Mono.just(completionsResult.getValue()); + }) + .doOnError(span::endSpanWithError) + .doOnSuccess(span::endSpanWithUsage) + .doOnTerminate(span::close); + }) .flatMap(completions -> { List responseMessages = completions diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAITextGenerationService.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAITextGenerationService.java index 864c5f71..ec04c568 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAITextGenerationService.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAITextGenerationService.java @@ -14,8 +14,6 @@ import com.microsoft.semantickernel.services.StreamingTextContent; import com.microsoft.semantickernel.services.textcompletion.TextContent; import com.microsoft.semantickernel.services.textcompletion.TextGenerationService; -import com.microsoft.semantickernel.implementation.telemetry.SemanticKernelTelemetry; -import io.opentelemetry.api.trace.Span; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -84,23 +82,14 @@ protected Mono> internalCompleteTextAsync( CompletionsOptions completionsOptions = getCompletionsOptions(text, requestSettings); - Span span = SemanticKernelTelemetry.startTextCompletionSpan( - getModelId(), - SemanticKernelTelemetry.OPEN_AI_PROVIDER, - completionsOptions.getMaxTokens(), - completionsOptions.getTemperature(), - completionsOptions.getTopP()); return getClient() .getCompletionsWithResponse(getDeploymentName(), completionsOptions, OpenAIRequestSettings.getRequestOptions()) .flatMap(completionsResult -> { if (completionsResult.getStatusCode() >= 400) { - SemanticKernelTelemetry.endSpanWithError(span); return Mono.error(new AIException(ErrorCodes.SERVICE_ERROR, "Request failed: " + completionsResult.getStatusCode())); } - SemanticKernelTelemetry.endSpanWithUsage(span, - completionsResult.getValue().getUsage()); return Mono.just(completionsResult.getValue()); }) .map(completions -> { diff --git a/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/OtelCaptureTest.java b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/OtelCaptureTest.java index c8bde155..c9136c55 100644 --- a/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/OtelCaptureTest.java +++ b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/OtelCaptureTest.java @@ -75,57 +75,6 @@ public static void shutdown() { otel.shutdown(); } - @Test - public void otelTextCaptureTest() { - - OpenAIAsyncClient openAIAsyncClient = Mockito.mock(OpenAIAsyncClient.class); - - CompletionsUsage completionsUsage = Mockito.mock(CompletionsUsage.class); - Mockito.when(completionsUsage.getCompletionTokens()).thenReturn(22); - Mockito.when(completionsUsage.getPromptTokens()).thenReturn(55); - - Completions completions = Mockito.mock(Completions.class); - Mockito.when(completions.getUsage()).thenReturn(completionsUsage); - - Response response = Mockito.mock(Response.class); - Mockito.when(response.getStatusCode()).thenReturn(200); - Mockito.when(response.getValue()).thenReturn(completions); - - Mockito.when(openAIAsyncClient.getCompletionsWithResponse( - Mockito.any(), - Mockito.any(), - Mockito.any())).thenAnswer(invocation -> Mono.just(response)); - - TextGenerationService client = OpenAITextGenerationService.builder() - .withOpenAIAsyncClient(openAIAsyncClient) - .withModelId("a-model") - .build(); - - try { - client.getTextContentsAsync( - "foo", - null, - null).block(); - } catch (Exception e) { - // Expect to fail - } - - Assertions.assertFalse(spans.isEmpty()); - Assertions.assertEquals("a-model", - spans.get(0).getAttributes().get(AttributeKey.stringKey("gen_ai.request.model"))); - Assertions.assertEquals("text.completions", - spans.get(0).getAttributes().get(AttributeKey.stringKey("gen_ai.operation.name"))); - Assertions.assertEquals("openai", - spans.get(0).getAttributes().get(AttributeKey.stringKey("gen_ai.system"))); - Assertions.assertEquals(22, - spans.get(0).getAttributes() - .get(AttributeKey.longKey("gen_ai.response.completion_tokens"))); - Assertions.assertEquals(55, - spans.get(0).getAttributes() - .get(AttributeKey.longKey("gen_ai.response.prompt_tokens"))); - - } - @Test public void otelChatCaptureTest() { OpenAIAsyncClient openAIAsyncClient = Mockito.mock(OpenAIAsyncClient.class); diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 51be5816..4411858d 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -26,6 +26,11 @@ + + io.opentelemetry.instrumentation + opentelemetry-reactor-3.1 + 2.9.0-alpha + com.microsoft.semantic-kernel semantickernel-api @@ -165,6 +170,7 @@ com.microsoft.semantickernel.samples.syntaxexamples.${sample} + false diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/FunctionTelemetry_Example.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/FunctionTelemetry_Example.java new file mode 100644 index 00000000..1d228250 --- /dev/null +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/FunctionTelemetry_Example.java @@ -0,0 +1,337 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.syntaxexamples.java; + +import com.azure.ai.openai.OpenAIAsyncClient; +import com.azure.ai.openai.OpenAIClientBuilder; +import com.azure.core.credential.AzureKeyCredential; +import com.azure.core.credential.KeyCredential; +import com.microsoft.semantickernel.Kernel; +import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIChatCompletion; +import com.microsoft.semantickernel.exceptions.ConfigurationException; +import com.microsoft.semantickernel.implementation.telemetry.SemanticKernelTelemetry; +import com.microsoft.semantickernel.orchestration.InvocationContext; +import com.microsoft.semantickernel.orchestration.InvocationReturnMode; +import com.microsoft.semantickernel.orchestration.ToolCallBehavior; +import com.microsoft.semantickernel.plugin.KernelPluginFactory; +import com.microsoft.semantickernel.samples.syntaxexamples.functions.Example59_OpenAIFunctionCalling.PetPlugin; +import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; +import com.microsoft.semantickernel.semanticfunctions.annotations.KernelFunctionParameter; +import com.microsoft.semantickernel.services.ServiceNotFoundException; +import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; +import com.microsoft.semantickernel.services.chatcompletion.ChatHistory; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.context.Scope; +import java.io.IOException; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Locale; +import reactor.core.publisher.Mono; + +public class FunctionTelemetry_Example { + /* + * // Get the Application Insights agent from + * https://github.com/microsoft/ApplicationInsights-Java, e.g: + * ``` + * wget -O "/tmp/applicationinsights-agent-3.6.1.jar" + * "https://github.com/microsoft/ApplicationInsights-Java/releases/download/3.6.1/applicationinsights-agent-3.6.1.jar" + * ``` + * + * // Get your application insights connection string from the Azure portal + * ``` + * CLIENT_ENDPOINT="" \ + * AZURE_CLIENT_KEY="" \ + * APPLICATIONINSIGHTS_CONNECTION_STRING="" \ + * MAVEN_OPTS="-javaagent:/tmp/applicationinsights-agent-3.6.1.jar" \ + * ../../../mvnw package exec:java -Dsample="java.FunctionTelemetry_Example" + * ``` + * + * If you open the Application Insights "Live metrics" view while running this example, you + * should see the telemetry in real-time. + * Otherwise within a few minutes, you should see the telemetry in the Application Insights -> + * Investigate -> Transaction search ui in the Azure portal. + */ + + private static final String CLIENT_KEY = System.getenv("CLIENT_KEY"); + private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY"); + + // Only required if AZURE_CLIENT_KEY is set + private static final String CLIENT_ENDPOINT = System.getenv("CLIENT_ENDPOINT"); + private static final String MODEL_ID = "gpt-4o"; + + public static void main(String[] args) + throws ConfigurationException, IOException, NoSuchMethodException, InterruptedException { + requestsWithSpanContext(); + testNestedCalls(); + requestsWithScope(); + + Thread.sleep(1000); + } + + private static void requestsWithSpanContext() throws IOException { + Span fakeRequest = GlobalOpenTelemetry.getTracer("Custom") + .spanBuilder("GET /requestsWithSpanContext") + .setSpanKind(SpanKind.SERVER) + .setAttribute("http.request.method", "GET") + .setAttribute("url.path", "/requestsWithSpanContext") + .setAttribute("url.scheme", "http") + .startSpan(); + + // Pass span context to the telemetry object to correlate telemetry with the request + SemanticKernelTelemetry telemetry = new SemanticKernelTelemetry( + GlobalOpenTelemetry.getTracer("Custom"), + fakeRequest.getSpanContext()); + + sequentialFunctionCalls(telemetry); + + fakeRequest.setStatus(StatusCode.OK); + fakeRequest.end(); + } + + private static void requestsWithScope() throws IOException { + Span fakeRequest = GlobalOpenTelemetry.getTracer("Custom") + .spanBuilder("GET /requestsWithScope") + .setSpanKind(SpanKind.SERVER) + .setAttribute("http.request.method", "GET") + .setAttribute("url.path", "/requestsWithScope") + .setAttribute("url.scheme", "http") + .startSpan(); + + // Pass span context to the telemetry object to correlate telemetry with the request + SemanticKernelTelemetry telemetry = new SemanticKernelTelemetry(); + + try (Scope scope = fakeRequest.makeCurrent()) { + sequentialFunctionCalls(telemetry); + } + + fakeRequest.setStatus(StatusCode.OK); + fakeRequest.end(); + } + + public static void sequentialFunctionCalls(SemanticKernelTelemetry telemetry) { + + OpenAIAsyncClient client; + + if (AZURE_CLIENT_KEY != null) { + client = new OpenAIClientBuilder() + .credential(new AzureKeyCredential(AZURE_CLIENT_KEY)) + .endpoint(CLIENT_ENDPOINT) + .buildAsyncClient(); + + } else { + client = new OpenAIClientBuilder() + .credential(new KeyCredential(CLIENT_KEY)) + .buildAsyncClient(); + } + + ChatCompletionService chat = OpenAIChatCompletion.builder() + .withModelId(MODEL_ID) + .withOpenAIAsyncClient(client) + .build(); + + var plugin = KernelPluginFactory.createFromObject(new PetPlugin(), "PetPlugin"); + + var kernel = Kernel.builder() + .withAIService(ChatCompletionService.class, chat) + .withPlugin(plugin) + .build(); + + var chatHistory = new ChatHistory(); + chatHistory.addUserMessage( + "What is the name and type of the pet with id ca2fc6bc-1307-4da6-a009-d7bf88dec37b?"); + + var messages = chat.getChatMessageContentsAsync( + chatHistory, + kernel, + InvocationContext.builder() + .withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(true)) + .withReturnMode(InvocationReturnMode.FULL_HISTORY) + .withTelemetry(telemetry) + .build()) + .block(); + + chatHistory = new ChatHistory(messages); + + System.out.println( + "THE NAME AND TYPE IS: " + chatHistory.getLastMessage().get().getContent()); + } + + public static void testNestedCalls() { + + OpenAIAsyncClient client; + + if (AZURE_CLIENT_KEY != null) { + client = new OpenAIClientBuilder() + .credential(new AzureKeyCredential(AZURE_CLIENT_KEY)) + .endpoint(CLIENT_ENDPOINT) + .buildAsyncClient(); + + } else { + client = new OpenAIClientBuilder() + .credential(new KeyCredential(CLIENT_KEY)) + .buildAsyncClient(); + } + + ChatCompletionService chat = OpenAIChatCompletion.builder() + .withModelId(MODEL_ID) + .withOpenAIAsyncClient(client) + .build(); + + var plugin = KernelPluginFactory.createFromObject(new TextAnalysisPlugin(), + "TextAnalysisPlugin"); + + var kernel = Kernel.builder() + .withAIService(ChatCompletionService.class, chat) + .withPlugin(plugin) + .build(); + + SemanticKernelTelemetry telemetry = new SemanticKernelTelemetry(); + + Span span = GlobalOpenTelemetry.getTracer("Test") + .spanBuilder("testNestedCalls span") + .setSpanKind(SpanKind.SERVER) + .startSpan(); + + try (Scope scope = span.makeCurrent()) { + String analysed = kernel + .invokePromptAsync( + """ + Analyse the following text: + Hello There + """, + KernelFunctionArguments.builder().build(), + InvocationContext.builder() + .withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(true)) + .withReturnMode(InvocationReturnMode.NEW_MESSAGES_ONLY) + .withTelemetry(telemetry) + .build()) + .withResultType(String.class) + .map(result -> { + return result.getResult(); + }) + .block(); + System.out.println(analysed); + } finally { + span.end(); + } + + } + + public static class TextAnalysisPlugin { + + @DefineKernelFunction(description = "Change all string chars to uppercase.", name = "Uppercase") + public String uppercase( + @KernelFunctionParameter(description = "Text to uppercase", name = "input") String text) { + return text.toUpperCase(Locale.ROOT); + } + + @DefineKernelFunction(name = "sha256sum", description = "Calculates a sha256 of the input", returnType = "string") + public Mono sha256sum( + @KernelFunctionParameter(name = "input", description = "The input to checksum", type = String.class) String input, + Kernel kernel, + SemanticKernelTelemetry telemetry) throws NoSuchAlgorithmException { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] hash = digest.digest(input.getBytes(StandardCharsets.UTF_8)); + String hashStr = new BigInteger(1, hash).toString(16); + + return kernel + .invokePromptAsync( + """ + Uppercase the following text: + === BEGIN TEXT === + %s + === END TEXT === + """.formatted(hashStr) + .stripIndent(), + null, + InvocationContext.builder() + .withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(true)) + .withReturnMode(InvocationReturnMode.NEW_MESSAGES_ONLY) + .withTelemetry(telemetry) + .build()) + .withResultType(String.class) + .map(result -> { + return result.getResult(); + }); + } + + @DefineKernelFunction(name = "formatAnswer", description = "Formats an answer", returnType = "string") + public Mono formatAnswer( + @KernelFunctionParameter(name = "input", description = "The input to format", type = String.class) String input, + Kernel kernel, + SemanticKernelTelemetry telemetry) throws ServiceNotFoundException { + + return kernel + .invokePromptAsync( + """ + Translate the following text into Italian: + === BEGIN TEXT === + %s + === END TEXT === + """.formatted(input) + .stripIndent()) + .withResultType(String.class) + .map(result -> { + return result.getResult(); + }); + } + + @DefineKernelFunction(name = "analyseInput", description = "Gives a text analysis of the input", returnType = "string") + public Mono analyseInput( + @KernelFunctionParameter(name = "input", description = "The input to analyse", type = String.class) String input, + Kernel kernel, + SemanticKernelTelemetry telemetry) throws ServiceNotFoundException { + + return kernel + .invokePromptAsync( + """ + Calculating sha256sum of the following text: + === BEGIN TEXT === + %s + === END TEXT === + """.formatted(input) + .stripIndent(), + null, + InvocationContext.builder() + .withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(true)) + .withReturnMode(InvocationReturnMode.NEW_MESSAGES_ONLY) + .withTelemetry(telemetry) + .build()) + .withResultType(String.class) + .map(result -> { + return result.getResult(); + }) + .flatMap(answer -> { + return kernel + .invokePromptAsync( + """ + Format the following text: + === BEGIN TEXT === + %s + === END TEXT === + """.formatted(answer) + .stripIndent()) + .withInvocationContext( + InvocationContext.builder() + .withToolCallBehavior( + ToolCallBehavior.allowAllKernelFunctions(true)) + .withReturnMode(InvocationReturnMode.NEW_MESSAGES_ONLY) + .withTelemetry(telemetry) + .build()) + .withArguments(null) + .withTelemetry(telemetry) + .withResultType(String.class); + }) + .map(it -> { + return it.getResult(); + }); + } + + } + +} diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/KernelFunctionYaml_Example.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/KernelFunctionYaml_Example.java index 900513c8..ec38aae0 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/KernelFunctionYaml_Example.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/KernelFunctionYaml_Example.java @@ -10,13 +10,14 @@ import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIChatCompletion; import com.microsoft.semantickernel.exceptions.ConfigurationException; import com.microsoft.semantickernel.implementation.EmbeddedResourceLoader; +import com.microsoft.semantickernel.implementation.telemetry.SemanticKernelTelemetry; import com.microsoft.semantickernel.orchestration.FunctionResult; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionYaml; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; -import com.microsoft.semantickernel.services.textcompletion.TextGenerationService; import java.io.IOException; +import javax.annotation.Nullable; public class KernelFunctionYaml_Example { @@ -29,7 +30,10 @@ public class KernelFunctionYaml_Example { .getOrDefault("MODEL_ID", "gpt-35-turbo"); public static void main(String[] args) throws ConfigurationException, IOException { + run(null); + } + public static void run(@Nullable SemanticKernelTelemetry telemetry) throws IOException { OpenAIAsyncClient client; if (AZURE_CLIENT_KEY != null) { @@ -51,12 +55,13 @@ public static void main(String[] args) throws ConfigurationException, IOExceptio Builder kernelBuilder = Kernel.builder() .withAIService(ChatCompletionService.class, openAIChatCompletion); - semanticKernelTemplate(kernelBuilder.build()); - handlebarsTemplate(kernelBuilder.build()); - + semanticKernelTemplate(kernelBuilder.build(), telemetry); + handlebarsTemplate(kernelBuilder.build(), telemetry); } - private static void handlebarsTemplate(Kernel kernel) throws IOException { + private static void handlebarsTemplate(Kernel kernel, + @Nullable SemanticKernelTelemetry telemetry) + throws IOException { String yaml = EmbeddedResourceLoader.readFile("GenerateStoryHandlebars.yaml", KernelFunctionYaml_Example.class); @@ -69,12 +74,15 @@ private static void handlebarsTemplate(Kernel kernel) throws IOException { .withVariable("length", 5) .withVariable("topic", "dogs") .build()) + .withTelemetry(telemetry) .block(); System.out.println(result.getResult()); } - private static void semanticKernelTemplate(Kernel kernel) throws IOException { + private static void semanticKernelTemplate(Kernel kernel, + @Nullable SemanticKernelTelemetry telemetry) + throws IOException { String yaml = EmbeddedResourceLoader.readFile("GenerateStory.yaml", KernelFunctionYaml_Example.class); @@ -87,6 +95,7 @@ private static void semanticKernelTemplate(Kernel kernel) throws IOException { .withVariable("length", 5) .withVariable("topic", "cats") .build()) + .withTelemetry(telemetry) .block(); System.out.println(result.getResult()); diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/resources/com/microsoft/semantickernel/samples/syntaxexamples/java/applicationinsights.json b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/resources/com/microsoft/semantickernel/samples/syntaxexamples/java/applicationinsights.json new file mode 100644 index 00000000..a5c0a227 --- /dev/null +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/resources/com/microsoft/semantickernel/samples/syntaxexamples/java/applicationinsights.json @@ -0,0 +1,16 @@ +{ + "connectionString": "InstrumentationKey=00000000-0000-0000-0000-0000-000000000000", + "preview": { + "processors": [ + { + "type": "span", + "include": { + "matchType": "regexp", + "spanNames": [ + ".*" + ] + } + } + ] + } +} \ No newline at end of file diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index d67acb88..2846ea9d 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -1,6 +1,7 @@ - + 4.0.0 @@ -15,6 +16,11 @@ Semantic Kernel API Defines the public interface for the Semantic Kernel + + io.opentelemetry.instrumentation + opentelemetry-reactor-3.1 + 2.9.0-alpha + com.azure azure-ai-openai diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/ChatCompletionSpan.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/ChatCompletionSpan.java new file mode 100644 index 00000000..87945860 --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/ChatCompletionSpan.java @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.implementation.telemetry; + +import com.azure.ai.openai.models.ChatCompletions; +import com.azure.ai.openai.models.CompletionsUsage; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanBuilder; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.context.Scope; +import java.util.function.Function; +import javax.annotation.Nullable; +import reactor.util.context.Context; +import reactor.util.context.ContextView; + +public class ChatCompletionSpan extends SemanticKernelTelemetrySpan { + + public ChatCompletionSpan( + Span span, + Function reactorContextModifier, + Scope spanScope, + Scope contextScope) { + super(span, reactorContextModifier, spanScope, contextScope); + } + + public static ChatCompletionSpan startChatCompletionSpan( + SemanticKernelTelemetry telemetry, + ContextView contextView, + @Nullable String modelName, + String modelProvider, + @Nullable Integer maxTokens, + @Nullable Double temperature, + @Nullable Double topP) { + return startCompletionSpan( + telemetry, + contextView, + "chat.completions", + modelName, + modelProvider, + maxTokens, + temperature, topP); + } + + public ChatCompletionSpan startTextCompletionSpan( + SemanticKernelTelemetry telemetry, + ContextView contextView, + @Nullable String modelName, + String modelProvider, + @Nullable Integer maxTokens, + @Nullable Double temperature, + @Nullable Double topP) { + return startCompletionSpan( + telemetry, + contextView, + "text.completions", + modelName, + modelProvider, + maxTokens, + temperature, topP); + } + + public static ChatCompletionSpan startCompletionSpan( + SemanticKernelTelemetry telemetry, + ContextView contextView, + String operationName, + @Nullable String modelName, + String modelProvider, + @Nullable Integer maxTokens, + @Nullable Double temperature, + @Nullable Double topP) { + if (modelName == null) { + modelName = "unknown"; + } + + SpanBuilder builder = telemetry.spanBuilder(operationName + " " + modelName) + .setSpanKind(SpanKind.CLIENT) + .setAttribute("gen_ai.request.model", modelName) + .setAttribute("gen_ai.operation.name", operationName) + .setAttribute("gen_ai.system", modelProvider); + + if (maxTokens != null) { + builder.setAttribute("gen_ai.request.max_tokens", maxTokens); + } + if (temperature != null) { + builder.setAttribute("gen_ai.request.temperature", temperature); + } + if (topP != null) { + builder.setAttribute("gen_ai.request.top_p", topP); + } + + Span span = builder.startSpan(); + + return build( + span, + contextView, + (contextModifier, spanScope, contextScope) -> new ChatCompletionSpan( + span, + contextModifier, + spanScope, + contextScope)); + } + + public void endSpanWithUsage(ChatCompletions chatCompletions) { + CompletionsUsage usage = chatCompletions.getUsage(); + getSpan().setStatus(StatusCode.OK); + getSpan() + .setAttribute("gen_ai.response.completion_tokens", usage.getCompletionTokens()); + getSpan().setAttribute("gen_ai.response.prompt_tokens", usage.getPromptTokens()); + close(); + } + + public void endSpanWithError(Throwable throwable) { + getSpan().setStatus(StatusCode.ERROR, throwable.getMessage()); + close(); + } +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/FunctionSpan.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/FunctionSpan.java new file mode 100644 index 00000000..601b4130 --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/FunctionSpan.java @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.implementation.telemetry; + +import com.microsoft.semantickernel.orchestration.FunctionResult; +import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanBuilder; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.context.Scope; +import java.util.function.Function; +import reactor.util.context.Context; +import reactor.util.context.ContextView; + +public class FunctionSpan extends SemanticKernelTelemetrySpan { + + public FunctionSpan( + Span span, + Function reactorContextModifier, + Scope spanScope, + Scope contextScope) { + super(span, reactorContextModifier, spanScope, contextScope); + } + + public static FunctionSpan build( + SemanticKernelTelemetry telemetry, + ContextView contextView, + String pluginName, + String name, + KernelFunctionArguments arguments) { + + SpanBuilder builder = telemetry.spanBuilder( + String.format("function_invocation %s-%s", pluginName, name)) + .setSpanKind(SpanKind.INTERNAL) + .setAttribute("semantic_kernel.function.invocation.name", name) + .setAttribute("semantic_kernel.function.invocation.plugin_name", pluginName); + + Span span = builder.startSpan(); + + return build( + span, + contextView, + (contextModifier, spanScope, contextScope) -> new FunctionSpan( + span, + contextModifier, + spanScope, + contextScope)); + } + + public void onFunctionSuccess(FunctionResult result) { + try { + getSpan().setStatus(StatusCode.OK); + } finally { + close(); + } + } + + public void onFunctionError(Throwable error) { + try { + getSpan().setStatus(StatusCode.ERROR, error.getMessage()); + getSpan().recordException(error); + } finally { + close(); + } + } +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/SemanticKernelTelemetry.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/SemanticKernelTelemetry.java index 2ba4b846..4121881d 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/SemanticKernelTelemetry.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/SemanticKernelTelemetry.java @@ -1,79 +1,54 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.implementation.telemetry; -import com.azure.ai.openai.models.CompletionsUsage; +import com.microsoft.semantickernel.orchestration.InvocationContext; import io.opentelemetry.api.GlobalOpenTelemetry; -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanBuilder; -import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.Tracer; import javax.annotation.Nullable; public class SemanticKernelTelemetry { public static final String OPEN_AI_PROVIDER = "openai"; - public static Span startChatCompletionSpan( - @Nullable String modelName, - String modelProvider, - @Nullable Integer maxTokens, - @Nullable Double temperature, - @Nullable Double topP) { - return startCompletionSpan("chat.completions", modelName, modelProvider, maxTokens, - temperature, topP); - } + private final Tracer tracer; - public static Span startTextCompletionSpan( - @Nullable String modelName, - String modelProvider, - @Nullable Integer maxTokens, - @Nullable Double temperature, - @Nullable Double topP) { - return startCompletionSpan("text.completions", modelName, modelProvider, maxTokens, - temperature, topP); - } + @Nullable + private final SpanContext spanContext; - private static Span startCompletionSpan( - String operationName, - @Nullable String modelName, - String modelProvider, - @Nullable Integer maxTokens, - @Nullable Double temperature, - @Nullable Double topP) { - OpenTelemetry otel = GlobalOpenTelemetry.get(); + public SemanticKernelTelemetry( + Tracer tracer, + @Nullable SpanContext spanContext) { - if (modelName == null) { - modelName = "unknown"; - } - SpanBuilder builder = otel - .getTracer("SemanticKernel") - .spanBuilder(operationName + " " + modelName) - .setAttribute("gen_ai.request.model", modelName) - .setAttribute("gen_ai.operation.name", operationName) - .setAttribute("gen_ai.system", modelProvider); + this.tracer = tracer; + this.spanContext = spanContext; + } - if (maxTokens != null) { - builder.setAttribute("gen_ai.request.max_tokens", maxTokens); - } - if (temperature != null) { - builder.setAttribute("gen_ai.request.temperature", temperature); - } - if (topP != null) { - builder.setAttribute("gen_ai.request.top_p", topP); - } + public SemanticKernelTelemetry() { + this( + GlobalOpenTelemetry.getTracer("SemanticKernel"), + null); + } - return builder.startSpan(); + public static SemanticKernelTelemetry getTelemetry( + @Nullable InvocationContext invocationContext) { + if (invocationContext != null) { + return invocationContext.getTelemetry(); + } + return new SemanticKernelTelemetry(); } - public static void endSpanWithUsage(Span span, CompletionsUsage usage) { - span.setStatus(StatusCode.OK); - span.setAttribute("gen_ai.response.completion_tokens", usage.getCompletionTokens()); - span.setAttribute("gen_ai.response.prompt_tokens", usage.getPromptTokens()); - span.end(); + private Tracer getTracer() { + return tracer; } - public static void endSpanWithError(Span span) { - span.setStatus(StatusCode.ERROR); - span.end(); + public SpanBuilder spanBuilder(String operationName) { + SpanBuilder sb = tracer.spanBuilder(operationName); + + if (spanContext != null) { + sb.addLink(spanContext); + } + return sb; } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/SemanticKernelTelemetrySpan.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/SemanticKernelTelemetrySpan.java new file mode 100644 index 00000000..95aacc03 --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/SemanticKernelTelemetrySpan.java @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.implementation.telemetry; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.opentelemetry.instrumentation.reactor.v3_1.ContextPropagationOperator; +import java.io.Closeable; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; +import org.slf4j.Logger; +import reactor.util.context.ContextView; + +public abstract class SemanticKernelTelemetrySpan implements Closeable { + + private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger( + SemanticKernelTelemetrySpan.class); + + private final Span span; + private final Function reactorContextModifier; + private final Scope spanScope; + private final Scope contextScope; + private final AtomicBoolean closed = new AtomicBoolean(false); + + public SemanticKernelTelemetrySpan(Span span, + Function reactorContextModifier, + Scope spanScope, Scope contextScope) { + this.span = span; + this.reactorContextModifier = reactorContextModifier; + this.spanScope = spanScope; + this.contextScope = contextScope; + } + + public interface SpanConstructor { + + public T build( + Function contextModifier, + Scope spanScope, + Scope contextScope); + } + + // Does need to be closed but as we are doing this in a reactive app, cant enforce the try with resources + @SuppressWarnings("MustBeClosedChecker") + public static T build( + Span span, + ContextView contextView, + SpanConstructor builder) { + LOGGER.trace("Starting Span: {}", span); + + Context currentOtelContext = ContextPropagationOperator + .getOpenTelemetryContextFromContextView( + contextView, + Context.current()); + + Context otelContext = span.storeInContext(currentOtelContext); + Scope contextScope = otelContext.makeCurrent(); + Scope spanScope = span.makeCurrent(); + + Function reactorContextModifier = ctx -> { + return ContextPropagationOperator.storeOpenTelemetryContext(ctx, otelContext); + }; + + return builder.build(reactorContextModifier, spanScope, contextScope); + } + + public Function getReactorContextModifier() { + return reactorContextModifier; + } + + public void close() { + if (closed.compareAndSet(false, true)) { + LOGGER.trace("Closing span: {}", span); + if (span.isRecording()) { + span.end(); + } + if (contextScope != null) { + contextScope.close(); + } + if (spanScope != null) { + spanScope.close(); + } + } + } + + public Span getSpan() { + return span; + } +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java index 630ce286..435d8538 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java @@ -11,6 +11,7 @@ import com.microsoft.semantickernel.hooks.KernelHook; import com.microsoft.semantickernel.hooks.KernelHooks; import com.microsoft.semantickernel.hooks.KernelHooks.UnmodifiableKernelHooks; +import com.microsoft.semantickernel.implementation.telemetry.SemanticKernelTelemetry; import com.microsoft.semantickernel.localization.SemanticKernelResources; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; @@ -47,6 +48,8 @@ public class FunctionInvocation extends Mono> { protected PromptExecutionSettings promptExecutionSettings; @Nullable protected ToolCallBehavior toolCallBehavior; + @Nullable + protected SemanticKernelTelemetry telemetry; private boolean isSubscribed = false; @@ -312,6 +315,17 @@ public FunctionInvocation withTypes(ContextVariableTypes contextVariableTypes return this; } + /** + * Supply a tracer to the function invocation. + * + * @param tracer The tracer to supply to the function invocation. + * @return this {@code FunctionInvocation} for fluent chaining. + */ + public FunctionInvocation withTelemetry(SemanticKernelTelemetry telemetry) { + this.telemetry = telemetry; + return this; + } + /** * Use an invocation context variable to supply the types, tool call behavior, prompt execution * settings, and kernel hooks to the function invocation. @@ -329,6 +343,7 @@ public FunctionInvocation withInvocationContext( withToolCallBehavior(invocationContext.getToolCallBehavior()); withPromptExecutionSettings(invocationContext.getPromptExecutionSettings()); addKernelHooks(invocationContext.getKernelHooks()); + withTelemetry(invocationContext.getTelemetry()); return this; } @@ -356,6 +371,10 @@ public void subscribe(CoreSubscriber> coreSubscriber) function.getPluginName(), function.getName()); } + if (telemetry == null) { + telemetry = new SemanticKernelTelemetry(); + } + isSubscribed = true; performSubscribe( @@ -369,7 +388,8 @@ public void subscribe(CoreSubscriber> coreSubscriber) promptExecutionSettings, toolCallBehavior, contextVariableTypes, - InvocationReturnMode.NEW_MESSAGES_ONLY)); + InvocationReturnMode.NEW_MESSAGES_ONLY, + telemetry)); } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/InvocationContext.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/InvocationContext.java index 3f1ba7e0..6fd3f0d2 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/InvocationContext.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/InvocationContext.java @@ -6,6 +6,7 @@ import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; import com.microsoft.semantickernel.hooks.KernelHooks; import com.microsoft.semantickernel.hooks.KernelHooks.UnmodifiableKernelHooks; +import com.microsoft.semantickernel.implementation.telemetry.SemanticKernelTelemetry; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.annotation.Nullable; @@ -24,6 +25,7 @@ public class InvocationContext { private final ToolCallBehavior toolCallBehavior; private final ContextVariableTypes contextVariableTypes; private final InvocationReturnMode invocationReturnMode; + private final SemanticKernelTelemetry telemetry; /** * Create a new instance of InvocationContext. @@ -38,7 +40,8 @@ protected InvocationContext( @Nullable PromptExecutionSettings promptExecutionSettings, @Nullable ToolCallBehavior toolCallBehavior, @Nullable ContextVariableTypes contextVariableTypes, - InvocationReturnMode invocationReturnMode) { + InvocationReturnMode invocationReturnMode, + SemanticKernelTelemetry telemetry) { this.hooks = unmodifiableClone(hooks); this.promptExecutionSettings = promptExecutionSettings; this.toolCallBehavior = toolCallBehavior; @@ -48,6 +51,7 @@ protected InvocationContext( } else { this.contextVariableTypes = new ContextVariableTypes(contextVariableTypes); } + this.telemetry = telemetry; } /** @@ -59,6 +63,7 @@ protected InvocationContext() { this.toolCallBehavior = null; this.contextVariableTypes = new ContextVariableTypes(); this.invocationReturnMode = InvocationReturnMode.NEW_MESSAGES_ONLY; + this.telemetry = null; } /** @@ -73,12 +78,14 @@ protected InvocationContext(@Nullable InvocationContext context) { this.toolCallBehavior = null; this.contextVariableTypes = new ContextVariableTypes(); this.invocationReturnMode = InvocationReturnMode.NEW_MESSAGES_ONLY; + this.telemetry = null; } else { this.hooks = context.hooks; this.promptExecutionSettings = context.promptExecutionSettings; this.toolCallBehavior = context.toolCallBehavior; this.contextVariableTypes = context.contextVariableTypes; this.invocationReturnMode = context.invocationReturnMode; + this.telemetry = context.telemetry; } } @@ -114,7 +121,8 @@ public static Builder copy(InvocationContext context) { .withKernelHooks(context.getKernelHooks()) .withContextVariableConverter(context.contextVariableTypes) .withPromptExecutionSettings(context.getPromptExecutionSettings()) - .withToolCallBehavior(context.getToolCallBehavior()); + .withToolCallBehavior(context.getToolCallBehavior()) + .withTelemetry(context.getTelemetry()); } /** @@ -166,6 +174,10 @@ public InvocationReturnMode returnMode() { return invocationReturnMode; } + public SemanticKernelTelemetry getTelemetry() { + return telemetry; + } + /** * Builder for {@link InvocationContext}. */ @@ -179,6 +191,8 @@ public static class Builder implements SemanticKernelBuilder @Nullable private ToolCallBehavior toolCallBehavior; private InvocationReturnMode invocationReturnMode = InvocationReturnMode.NEW_MESSAGES_ONLY; + @Nullable + private SemanticKernelTelemetry telemetry; /** * Add kernel hooks to the builder. @@ -252,10 +266,24 @@ public Builder withReturnMode(InvocationReturnMode invocationReturnMode) { return this; } + /** + * Add a tracer to the builder. + * + * @param tracer the tracer to add. + * @return this {@link Builder} + */ + public Builder withTelemetry(@Nullable SemanticKernelTelemetry telemetry) { + this.telemetry = telemetry; + return this; + } + @Override public InvocationContext build() { + if (telemetry == null) { + telemetry = new SemanticKernelTelemetry(); + } return new InvocationContext(hooks, promptExecutionSettings, toolCallBehavior, - contextVariableTypes, invocationReturnMode); + contextVariableTypes, invocationReturnMode, telemetry); } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java index 680d93c0..7173a7d4 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java @@ -14,6 +14,8 @@ import com.microsoft.semantickernel.hooks.FunctionInvokedEvent; import com.microsoft.semantickernel.hooks.FunctionInvokingEvent; import com.microsoft.semantickernel.hooks.KernelHooks; +import com.microsoft.semantickernel.implementation.telemetry.FunctionSpan; +import com.microsoft.semantickernel.implementation.telemetry.SemanticKernelTelemetry; import com.microsoft.semantickernel.localization.SemanticKernelResources; import com.microsoft.semantickernel.orchestration.FunctionResult; import com.microsoft.semantickernel.orchestration.InvocationContext; @@ -150,9 +152,10 @@ private static MethodDetails getMethodDetails( /** * Gets the function from the method. - * @param method the method to invoke + * + * @param method the method to invoke * @param instance the instance to invoke the method on - * @param the return type of the function + * @param the return type of the function * @return the function representing the method */ @SuppressWarnings("unchecked") @@ -367,6 +370,8 @@ private static Object getArgumentValue( if (Kernel.class.isAssignableFrom(targetArgType)) { return kernel; + } else if (SemanticKernelTelemetry.class.isAssignableFrom(targetArgType)) { + return invocationContext.getTelemetry(); } String variableName = getGetVariableName(parameter); @@ -692,6 +697,7 @@ private static InputVariable toKernelParameterMetadata(Parameter parameter) { /** * Gets the constants from an enum type. + * * @param type the type to get the enum constants from * @return a list of the enum constants or {@code null} if the type is not an enum */ @@ -726,11 +732,27 @@ public Mono> invokeAsync( @Nullable KernelFunctionArguments arguments, @Nullable ContextVariableType variableType, @Nullable InvocationContext invocationContext) { - return function.invokeAsync(kernel, this, arguments, variableType, invocationContext); + + return Mono.deferContextual(contextView -> { + FunctionSpan span = FunctionSpan.build( + SemanticKernelTelemetry.getTelemetry(invocationContext), + contextView, + this.getPluginName(), + this.getName(), + arguments); + + return function + .invokeAsync(kernel, this, arguments, variableType, invocationContext) + .contextWrite(span.getReactorContextModifier()) + .doOnSuccess(span::onFunctionSuccess) + .doOnError(span::onFunctionError) + .doOnTerminate(span::close); + }); } /** * Concrete implementation of the abstract method in KernelFunction. + * * @param the return type of the function */ public interface ImplementationFunc { @@ -775,6 +797,7 @@ default FunctionResult invoke( /** * A builder for {@link KernelFunction}. + * * @param the return type of the function */ public static class Builder { diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java index adcea185..5c2c5412 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java @@ -10,6 +10,8 @@ import com.microsoft.semantickernel.hooks.KernelHooks; import com.microsoft.semantickernel.hooks.PromptRenderedEvent; import com.microsoft.semantickernel.hooks.PromptRenderingEvent; +import com.microsoft.semantickernel.implementation.telemetry.FunctionSpan; +import com.microsoft.semantickernel.implementation.telemetry.SemanticKernelTelemetry; import com.microsoft.semantickernel.localization.SemanticKernelResources; import com.microsoft.semantickernel.orchestration.FunctionResult; import com.microsoft.semantickernel.orchestration.InvocationContext; @@ -274,8 +276,22 @@ public Mono> invokeAsync( @Nullable KernelFunctionArguments arguments, @Nullable ContextVariableType variableType, @Nullable InvocationContext invocationContext) { - return invokeInternalAsync(kernel, arguments, variableType, invocationContext) - .takeLast(1).single(); + return Mono.deferContextual(contextView -> { + FunctionSpan span = FunctionSpan.build( + SemanticKernelTelemetry.getTelemetry(invocationContext), + contextView, + this.getPluginName(), + this.getName(), + arguments); + + return invokeInternalAsync(kernel, arguments, variableType, invocationContext) + .contextWrite(span.getReactorContextModifier()) + .takeLast(1) + .single() + .doOnSuccess(span::onFunctionSuccess) + .doOnError(span::onFunctionError) + .doOnTerminate(span::close); + }); } /** From ac0f5692f8cd693591e4a6d801fa28395c2917b1 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Tue, 5 Nov 2024 18:37:54 +0000 Subject: [PATCH 094/277] add watchdog timer --- .../SemanticKernelTelemetrySpan.java | 49 +++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/SemanticKernelTelemetrySpan.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/SemanticKernelTelemetrySpan.java index 95aacc03..2ef6413c 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/SemanticKernelTelemetrySpan.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/SemanticKernelTelemetrySpan.java @@ -6,9 +6,12 @@ import io.opentelemetry.context.Scope; import io.opentelemetry.instrumentation.reactor.v3_1.ContextPropagationOperator; import java.io.Closeable; +import java.time.Duration; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; import org.slf4j.Logger; +import reactor.core.Disposable; +import reactor.core.publisher.Mono; import reactor.util.context.ContextView; public abstract class SemanticKernelTelemetrySpan implements Closeable { @@ -16,12 +19,30 @@ public abstract class SemanticKernelTelemetrySpan implements Closeable { private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger( SemanticKernelTelemetrySpan.class); + private static final long SPAN_TIMEOUT_MS = Long.parseLong((String) System.getProperties() + .getOrDefault("semantickernel.telemetry.span_timeout", "120000")); + private final Span span; private final Function reactorContextModifier; private final Scope spanScope; private final Scope contextScope; private final AtomicBoolean closed = new AtomicBoolean(false); + // Timeout to close the span if it was not closed within the specified time to avoid memory leaks + private final Disposable watchdog; + + // This is a finalizer guardian to ensure that the span is closed if it was not closed explicitly + @SuppressWarnings("unused") + private final Object finalizerGuardian = new Object() { + @Override + protected void finalize() { + if (closed.get() == false) { + LOGGER.warn("Span was not closed"); + close(); + } + } + }; + public SemanticKernelTelemetrySpan(Span span, Function reactorContextModifier, Scope spanScope, Scope contextScope) { @@ -29,6 +50,15 @@ public SemanticKernelTelemetrySpan(Span span, this.reactorContextModifier = reactorContextModifier; this.spanScope = spanScope; this.contextScope = contextScope; + + watchdog = Mono.just(1) + .delay(Duration.ofMillis(SPAN_TIMEOUT_MS)) + .subscribe(i -> { + if (closed.get() == false) { + LOGGER.warn("Span was not closed, timing out"); + close(); + } + }); } public interface SpanConstructor { @@ -71,14 +101,27 @@ public void close() { if (closed.compareAndSet(false, true)) { LOGGER.trace("Closing span: {}", span); if (span.isRecording()) { - span.end(); + try { + span.end(); + } catch (Exception e) { + LOGGER.error("Error closing span", e); + } } if (contextScope != null) { - contextScope.close(); + try { + contextScope.close(); + } catch (Exception e) { + LOGGER.error("Error closing context scope", e); + } } if (spanScope != null) { - spanScope.close(); + try { + spanScope.close(); + } catch (Exception e) { + LOGGER.error("Error closing span scope", e); + } } + watchdog.dispose(); } } From 3b40384374c533be24d1318135fbdc00fb96fe7d Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Mon, 11 Nov 2024 15:57:13 +0000 Subject: [PATCH 095/277] add parameter types to tool call arguments --- .../openai/chatcompletion/OpenAIFunction.java | 33 ++++++++++++++++++- .../samples/demos/lights/LightsPlugin.java | 4 +-- .../KernelFunctionFromMethod.java | 2 +- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java index 0ed048e8..79c7d617 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java @@ -13,6 +13,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -149,7 +150,13 @@ public Map getProperties() { private static String getSchemaForFunctionParameter(@Nullable InputVariable parameter) { List entries = new ArrayList<>(); - entries.add("\"type\":\"string\""); + String type = "string"; + + if (parameter != null && parameter.getType() != null) { + type = getJavaTypeToOpenAiFunctionType(parameter.getType()); + } + + entries.add("\"type\":\"" + type + "\""); // Add description if present if (parameter != null && parameter.getDescription() != null && !parameter.getDescription() @@ -179,4 +186,28 @@ private static String getSchemaForFunctionParameter(@Nullable InputVariable para return "{" + schema + "}"; } + + private static String getJavaTypeToOpenAiFunctionType(String javaType) { + switch (javaType.toLowerCase(Locale.ROOT)) { + case "boolean": + return "boolean"; + case "integer": + case "int": + case "long": + case "short": + case "byte": + return "integer"; + case "double": + case "float": + return "number"; + case "string": + return "string"; + case "array": + return "array"; + case "void": + return "null"; + default: + return "object"; + } + } } diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightsPlugin.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightsPlugin.java index 55d70b02..d2f4ed08 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightsPlugin.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightsPlugin.java @@ -26,8 +26,8 @@ public List getLights() { @DefineKernelFunction(name = "change_state", description = "Changes the state of the light") public LightModel changeState( - @KernelFunctionParameter(name = "id", description = "The ID of the light to change") int id, - @KernelFunctionParameter(name = "isOn", description = "The new state of the light") boolean isOn) { + @KernelFunctionParameter(name = "id", description = "The ID of the light to change", type = int.class) int id, + @KernelFunctionParameter(name = "isOn", description = "The new state of the light", type = boolean.class) boolean isOn) { System.out.println("Changing light " + id + " " + isOn); Optional light = lights.stream() .filter(l -> l.getId() == id) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java index 680d93c0..2765e958 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java @@ -476,7 +476,7 @@ private static Object toObjectType( ContextVariableTypeConverter c = sourceType.getConverter(); Object converted = c.toObject(invocationContext.getContextVariableTypes(), sourceValue, - targetArgType); + targetArgType, false); if (converted != null) { return converted; } From 564b27228c75224bfe7adc7ccfa7d2b70dbbac7c Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Fri, 15 Nov 2024 17:21:58 +0000 Subject: [PATCH 096/277] Add thread safety on database creation --- .../jdbc/JDBCVectorStoreQueryProvider.java | 140 ++++++++++-------- .../jdbc/JDBCVectorStoreRecordCollection.java | 10 +- .../SQLiteVectorStoreQueryProvider.java | 7 + 3 files changed, 91 insertions(+), 66 deletions(-) diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java index c1cc6273..ac21c61f 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -29,9 +29,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; +import javax.annotation.concurrent.GuardedBy; import javax.sql.DataSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,6 +57,8 @@ public class JDBCVectorStoreQueryProvider private final String collectionsTable; private final String prefixForCollectionTables; + private final Object dbCreationLock = new Object(); + @SuppressFBWarnings("EI_EXPOSE_REP2") // DataSource is not exposed protected JDBCVectorStoreQueryProvider( @Nonnull DataSource dataSource, @@ -89,12 +94,13 @@ protected JDBCVectorStoreQueryProvider( /** * Creates a new instance of the JDBCVectorStoreQueryProvider class. - * @param dataSource the data source - * @param collectionsTable the collections table + * + * @param dataSource the data source + * @param collectionsTable the collections table * @param prefixForCollectionTables the prefix for collection tables - * @param supportedKeyTypes the supported key types - * @param supportedDataTypes the supported data types - * @param supportedVectorTypes the supported vector types + * @param supportedKeyTypes the supported key types + * @param supportedDataTypes the supported data types + * @param supportedVectorTypes the supported vector types */ public JDBCVectorStoreQueryProvider( @SuppressFBWarnings("EI_EXPOSE_REP2") @Nonnull DataSource dataSource, @@ -276,48 +282,57 @@ public boolean collectionExists(String collectionName) { */ @Override @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") + @GuardedBy("dbCreationLock") // SQL query is generated dynamically with valid identifiers public void createCollection(String collectionName, VectorStoreRecordDefinition recordDefinition) { - // No approximate search is supported in JDBCVectorStoreQueryProvider - if (recordDefinition.getVectorFields().stream() - .anyMatch( - field -> field.getIndexKind() != null && field.getIndexKind() != IndexKind.FLAT - && field.getIndexKind() != IndexKind.UNDEFINED)) { - LOGGER - .warn(String.format("Indexes are not supported in %s. Ignoring indexKind property.", - this.getClass().getName())); - } - - String createStorageTable = formatQuery("CREATE TABLE IF NOT EXISTS %s (" - + "%s VARCHAR(255) PRIMARY KEY, " - + "%s, " - + "%s);", - getCollectionTableName(collectionName), - getKeyColumnName(recordDefinition.getKeyField()), - getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), - getSupportedDataTypes()), - getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getVectorFields()), - getSupportedVectorTypes())); + synchronized (dbCreationLock) { + // No approximate search is supported in JDBCVectorStoreQueryProvider + if (recordDefinition.getVectorFields().stream() + .anyMatch( + field -> field.getIndexKind() != null && field.getIndexKind() != IndexKind.FLAT + && field.getIndexKind() != IndexKind.UNDEFINED)) { + LOGGER + .warn(String.format( + "Indexes are not supported in %s. Ignoring indexKind property.", + this.getClass().getName())); + } - String insertCollectionQuery = formatQuery("INSERT INTO %s (collectionId) VALUES (?)", - validateSQLidentifier(collectionsTable)); + String createStorageTable = formatQuery("CREATE TABLE IF NOT EXISTS %s (" + + "%s VARCHAR(255) PRIMARY KEY, " + + "%s, " + + "%s);", + getCollectionTableName(collectionName), + getKeyColumnName(recordDefinition.getKeyField()), + getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), + getSupportedDataTypes()), + getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getVectorFields()), + getSupportedVectorTypes())); + + String insertCollectionQuery = this.getInsertCollectionQuery(collectionsTable); + + try (Connection connection = dataSource.getConnection(); + PreparedStatement createTable = connection.prepareStatement(createStorageTable)) { + createTable.execute(); + } catch (SQLException e) { + throw new SKException("Failed to create collection", e); + } - try (Connection connection = dataSource.getConnection(); - PreparedStatement createTable = connection.prepareStatement(createStorageTable)) { - createTable.execute(); - } catch (SQLException e) { - throw new SKException("Failed to create collection", e); + try (Connection connection = dataSource.getConnection(); + PreparedStatement insert = connection.prepareStatement(insertCollectionQuery)) { + insert.setObject(1, collectionName); + insert.execute(); + } catch (SQLException e) { + throw new SKException("Failed to insert collection", e); + } } + } - try (Connection connection = dataSource.getConnection(); - PreparedStatement insert = connection.prepareStatement(insertCollectionQuery)) { - insert.setObject(1, collectionName); - insert.execute(); - } catch (SQLException e) { - throw new SKException("Failed to insert collection", e); - } + protected String getInsertCollectionQuery(String collectionsTable) { + return formatQuery( + "INSERT IGNORE INTO %s (collectionId) VALUES (?)", + validateSQLidentifier(collectionsTable)); } /** @@ -327,26 +342,29 @@ public void createCollection(String collectionName, * @throws SKException if an error occurs while deleting the collection */ @Override + @GuardedBy("dbCreationLock") public void deleteCollection(String collectionName) { - String deleteCollectionOperation = formatQuery("DELETE FROM %s WHERE collectionId = ?", - validateSQLidentifier(collectionsTable)); - String dropTableOperation = formatQuery("DROP TABLE %s", - getCollectionTableName(collectionName)); - - try (Connection connection = dataSource.getConnection(); - PreparedStatement deleteCollection = connection - .prepareStatement(deleteCollectionOperation)) { - deleteCollection.setObject(1, collectionName); - deleteCollection.execute(); - } catch (SQLException e) { - throw new SKException("Failed to delete collection", e); - } + synchronized (dbCreationLock) { + String deleteCollectionOperation = formatQuery("DELETE FROM %s WHERE collectionId = ?", + validateSQLidentifier(collectionsTable)); + String dropTableOperation = formatQuery("DROP TABLE %s", + getCollectionTableName(collectionName)); + + try (Connection connection = dataSource.getConnection(); + PreparedStatement deleteCollection = connection + .prepareStatement(deleteCollectionOperation)) { + deleteCollection.setObject(1, collectionName); + deleteCollection.execute(); + } catch (SQLException e) { + throw new SKException("Failed to delete collection", e); + } - try (Connection connection = dataSource.getConnection(); - PreparedStatement dropTable = connection.prepareStatement(dropTableOperation)) { - dropTable.execute(); - } catch (SQLException e) { - throw new SKException("Failed to drop table", e); + try (Connection connection = dataSource.getConnection(); + PreparedStatement dropTable = connection.prepareStatement(dropTableOperation)) { + dropTable.execute(); + } catch (SQLException e) { + throw new SKException("Failed to drop table", e); + } } } @@ -518,8 +536,8 @@ protected List getRecordsWithFilter(String collectionName, * * @param the record type * @param collectionName the collection name - * @param vector the vector to search with - * @param options the search options + * @param vector the vector to search with + * @param options the search options * @param recordDefinition the record definition * @param mapper the mapper, responsible for mapping the result set to the record * type. @@ -622,8 +640,8 @@ public String getFilter(VectorSearchFilter filter, } /** - * Gets the filter parameters for the given vector search filter to associate with the filter string - * generated by the getFilter method. + * Gets the filter parameters for the given vector search filter to associate with the filter + * string generated by the getFilter method. * * @param filter The filter to get the filter parameters for. * @return The filter parameters. diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java index fdcdd6d3..1c1cea5d 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -6,8 +6,8 @@ import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; -import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.options.DeleteRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; @@ -27,10 +27,9 @@ import reactor.core.scheduler.Schedulers; /** - * The JDBCVectorStoreRecordCollection class represents a collection of records - * in a JDBC vector store. It implements the SQLVectorStoreRecordCollection - * interface and provides methods for managing the collection, such as creating, - * deleting, and upserting records. + * The JDBCVectorStoreRecordCollection class represents a collection of records in a JDBC vector + * store. It implements the SQLVectorStoreRecordCollection interface and provides methods for + * managing the collection, such as creating, deleting, and upserting records. * * @param the type of the records in the collection */ @@ -322,6 +321,7 @@ public Mono> searchAsync(List vector, /** * Builder for a JDBCVectorStoreRecordCollection. + * * @param the type of the records in the collection */ public static class Builder diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java index 3ef126e6..57de1225 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java @@ -112,6 +112,13 @@ public void upsertRecords(String collectionName, List records, } } + @Override + protected String getInsertCollectionQuery(String collectionsTable) { + return formatQuery( + "INSERT OR IGNORE INTO %s (collectionId) VALUES (?)", + validateSQLidentifier(collectionsTable)); + } + /** * A builder for {@code SQLiteVectorStoreQueryProvider}. */ From 29c59b5284d22772a4089e590fd3a546d56609b7 Mon Sep 17 00:00:00 2001 From: "omar.mahamid" Date: Sat, 16 Nov 2024 13:54:27 +0200 Subject: [PATCH 097/277] enhance cosine similarity --- .../data/vectorsearch/VectorOperations.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java index cbc1e314..cabf91ea 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java @@ -38,15 +38,17 @@ public static float cosineSimilarity(@Nonnull List x, @Nonnull List Date: Sat, 16 Nov 2024 13:58:19 +0200 Subject: [PATCH 098/277] float instead of double --- .../data/vectorsearch/VectorOperations.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java index cabf91ea..b43f990b 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java @@ -38,9 +38,9 @@ public static float cosineSimilarity(@Nonnull List x, @Nonnull List x, @Nonnull List Date: Sat, 16 Nov 2024 14:01:40 +0200 Subject: [PATCH 099/277] Update VectorOperations.java --- .../semantickernel/data/vectorsearch/VectorOperations.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java index b43f990b..ba86cb7b 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java @@ -48,6 +48,10 @@ public static float cosineSimilarity(@Nonnull List x, @Nonnull List Date: Tue, 19 Nov 2024 02:28:30 -0800 Subject: [PATCH 100/277] Add hybridSearchAsync to AzureAISearchVectorStoreRecordCollection --- ...reAISearchVectorStoreRecordCollection.java | 129 ++++++++++++++---- 1 file changed, 106 insertions(+), 23 deletions(-) diff --git a/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index c4f44c42..b1c15b82 100644 --- a/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -10,6 +10,7 @@ import com.azure.search.documents.indexes.models.VectorSearchProfile; import com.azure.search.documents.models.IndexDocumentsResult; import com.azure.search.documents.models.IndexingResult; +import com.azure.search.documents.models.ScoringParameter; import com.azure.search.documents.models.SearchOptions; import com.azure.search.documents.models.VectorQuery; import com.azure.search.documents.models.VectorizableTextQuery; @@ -39,6 +40,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Vector; import java.util.stream.Collectors; import javax.annotation.Nonnull; import reactor.core.publisher.Flux; @@ -288,10 +290,8 @@ public Mono deleteBatchAsync(List keys, DeleteRecordOptions option }).collect(Collectors.toList())).then(); } - private Mono> searchAndMapAsync( - List vectorQueries, VectorSearchOptions options, - GetRecordOptions getRecordOptions) { - + private SearchOptions configureVectorSearchOptions( + List vectorQueries, VectorSearchOptions options) { String filter = AzureAISearchVectorStoreCollectionSearchMapping.getInstance() .getFilter(options.getVectorSearchFilter(), recordDefinition); @@ -299,7 +299,6 @@ private Mono> searchAndMapAsync( .setFilter(filter) .setTop(options.getTop()) .setSkip(options.getSkip()) - .setScoringParameters() .setVectorSearchOptions(new com.azure.search.documents.models.VectorSearchOptions() .setQueries(vectorQueries)); @@ -307,10 +306,16 @@ private Mono> searchAndMapAsync( searchOptions.setSelect(nonVectorFields.toArray(new String[0])); } + return searchOptions; + } + + private Mono> searchAndMapAsync(String query, + SearchOptions searchOptions, + boolean includeVectors) { VectorStoreRecordMapper mapper = this.options .getVectorStoreRecordMapper(); - return this.searchAsyncClient.search(null, searchOptions) + return this.searchAsyncClient.search(query, searchOptions) .flatMap(response -> { Record record; @@ -318,7 +323,7 @@ private Mono> searchAndMapAsync( if (mapper != null && mapper.getStorageModelToRecordMapper() != null) { record = mapper .mapStorageModelToRecord(response.getDocument(SearchDocument.class), - getRecordOptions); + new GetRecordOptions(includeVectors)); } else { record = response.getDocument(this.options.getRecordClass()); } @@ -329,7 +334,9 @@ record = response.getDocument(this.options.getRecordClass()); } /** - * Vectorizable text search. This method searches for records that are similar to the given text. + * Vectorizable text search. This method searches for records that are similar to the given text after vectorization. + *

+ * Vectorizer configuration must be set up in the Azure AI Search index. * * @param searchText The text to search with. * @param options The options to use for the search. @@ -353,8 +360,9 @@ public Mono> searchAsync(String searchText, : firstVectorFieldName).getEffectiveStorageName()) .setKNearestNeighborsCount(options.getTop())); - return searchAndMapAsync(vectorQueries, options, - new GetRecordOptions(options.isIncludeVectors())); + return searchAndMapAsync(null, + configureVectorSearchOptions(vectorQueries, options), + options.isIncludeVectors()); } /** @@ -367,22 +375,97 @@ public Mono> searchAsync(String searchText, @Override public Mono> searchAsync(List vector, VectorSearchOptions options) { - if (firstVectorFieldName == null) { - throw new SKException("No vector fields defined. Cannot perform vector search"); - } + return hybridSearchAsync(null, vector, options, null); + } - if (options == null) { - options = VectorSearchOptions.createDefault(firstVectorFieldName); + /** + * Hybrid search. This method searches for records that are similar to the given text and vector. + * + * @param searchText The text to search with. + * If null, only vector search is performed. + * @param vector The vector to search with. + * If null, only full text search is performed. + * @param options The vector search options used for the search. + * @param additionalSearchOptions AzureAI search additional options. + * If Filter, Top, Skip, Select or VectorSearchOptions are not null, they will be used instead of the default options. + *

+ * If null, default search options are used. + */ + public Mono> hybridSearchAsync(String searchText, + List vector, VectorSearchOptions options, SearchOptions additionalSearchOptions) { + SearchOptions searchOptions = new SearchOptions(); + + if (vector != null) { + if (firstVectorFieldName == null) { + throw new SKException("No vector fields defined. Cannot perform vector search"); + } + + if (options == null) { + options = VectorSearchOptions.createDefault(firstVectorFieldName); + } + + List vectorQueries = new ArrayList<>(); + vectorQueries.add(new VectorizedQuery(vector) + .setFields(recordDefinition.getField(options.getVectorFieldName() != null + ? options.getVectorFieldName() + : firstVectorFieldName).getEffectiveStorageName()) + .setKNearestNeighborsCount(options.getTop())); + + // Configure default vector search options + searchOptions = configureVectorSearchOptions(vectorQueries, options); } - List vectorQueries = new ArrayList<>(); - vectorQueries.add(new VectorizedQuery(vector) - .setFields(recordDefinition.getField(options.getVectorFieldName() != null - ? options.getVectorFieldName() - : firstVectorFieldName).getEffectiveStorageName()) - .setKNearestNeighborsCount(options.getTop())); + // Configure additional search options + if (additionalSearchOptions != null) { + searchOptions + .setQueryType(additionalSearchOptions.getQueryType()) + .setSemanticSearchOptions(additionalSearchOptions.getSemanticSearchOptions()) + .setFacets(additionalSearchOptions.getFacets() != null + ? additionalSearchOptions.getFacets().toArray(new String[0]) + : null) + .setHighlightFields(additionalSearchOptions.getHighlightFields() != null + ? additionalSearchOptions.getHighlightFields().toArray(new String[0]) + : null) + .setHighlightPreTag(additionalSearchOptions.getHighlightPreTag()) + .setHighlightPostTag(additionalSearchOptions.getHighlightPostTag()) + .setMinimumCoverage(additionalSearchOptions.getMinimumCoverage()) + .setOrderBy(additionalSearchOptions.getOrderBy() != null + ? additionalSearchOptions.getOrderBy().toArray(new String[0]) + : null) + .setScoringParameters(additionalSearchOptions.getScoringParameters() != null + ? additionalSearchOptions.getScoringParameters().stream() + .map(s -> new ScoringParameter(s.getName(), s.getValues())) + .toArray(ScoringParameter[]::new) + : null) + .setScoringProfile(additionalSearchOptions.getScoringProfile()) + .setSearchFields(additionalSearchOptions.getSearchFields() != null + ? additionalSearchOptions.getSearchFields().toArray(new String[0]) + : null) + .setIncludeTotalCount(additionalSearchOptions.isTotalCountIncluded()) + .setSearchMode(additionalSearchOptions.getSearchMode()) + .setScoringStatistics(additionalSearchOptions.getScoringStatistics()) + .setSessionId(additionalSearchOptions.getSessionId()); + + // Override default vector options if provided + if (additionalSearchOptions.getFilter() != null) { + searchOptions.setFilter(additionalSearchOptions.getFilter()); + } + if (additionalSearchOptions.getTop() != null) { + searchOptions.setTop(additionalSearchOptions.getTop()); + } + if (additionalSearchOptions.getSkip() != null) { + searchOptions.setSkip(additionalSearchOptions.getSkip()); + } + if (additionalSearchOptions.getVectorSearchOptions() != null) { + searchOptions + .setVectorSearchOptions(additionalSearchOptions.getVectorSearchOptions()); + } + if (additionalSearchOptions.getSelect() != null) { + searchOptions.setSelect(additionalSearchOptions.getSelect().toArray(new String[0])); + } + } - return searchAndMapAsync(vectorQueries, options, - new GetRecordOptions(options.isIncludeVectors())); + return searchAndMapAsync(searchText, searchOptions, + options != null && options.isIncludeVectors()); } } From 8192efc4a2050c3a2a17aafdf2ac28509b81abbe Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Tue, 3 Dec 2024 16:21:08 +0000 Subject: [PATCH 101/277] Update changelog --- CHANGELOG.md | 8 ++++++++ .../semantickernel-text-splitter-plugin/pom.xml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40503073..34d9960e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +# 1.4.1 + +- Add Otel Telemetry on function invocations +- Fix bug to add type information to OpenAI function parameters +- Improve efficiency of cosine similarity calculation +- Fix concurrency bugs on database creation +- Add sample demonstrating a text splitter for chunking text for embedding + # 1.4.0 - Upgrade to azure-ai-openai 1.0.0-beta.12 diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml index e0157c82..9e26aaf4 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.3.1-SNAPSHOT + 1.4.1-SNAPSHOT ../pom.xml From 0acfd53fdcfa0f79a6688d110d09ce71309a7fb5 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 3 Dec 2024 09:23:55 -0800 Subject: [PATCH 102/277] Update --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34d9960e..aff20f82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - Improve efficiency of cosine similarity calculation - Fix concurrency bugs on database creation - Add sample demonstrating a text splitter for chunking text for embedding +- Add hybridSearchAsync support to Azure AI Search # 1.4.0 From 38ed8d0fc697fbb1ce15a4b4eb4454de4fdae38a Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 3 Dec 2024 10:29:45 -0800 Subject: [PATCH 103/277] Rename syntaxexamples 'plugins' to 'plugin' --- .../microsoft/semantickernel/tests/WiremockExamplesIT.java | 4 ++-- .../semantickernel/samples/syntaxexamples/RunAll.java | 4 ++-- .../{plugins => plugin}/Example07_BingAndGooglePlugins.java | 2 +- .../Example10_DescribeAllPluginsAndFunctions.java | 4 +--- .../Example13_ConversationSummaryPlugin.java | 4 +--- 5 files changed, 7 insertions(+), 11 deletions(-) rename samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/{plugins => plugin}/Example07_BingAndGooglePlugins.java (99%) rename samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/{plugins => plugin}/Example10_DescribeAllPluginsAndFunctions.java (97%) rename samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/{plugins => plugin}/Example13_ConversationSummaryPlugin.java (98%) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/WiremockExamplesIT.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/WiremockExamplesIT.java index 0eeabc13..f01b40d1 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/WiremockExamplesIT.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/WiremockExamplesIT.java @@ -32,8 +32,8 @@ import java.util.List; import java.util.Queue; import java.util.stream.Stream; -import com.microsoft.semantickernel.samples.syntaxexamples.plugins.Example10_DescribeAllPluginsAndFunctions; -import com.microsoft.semantickernel.samples.syntaxexamples.plugins.Example13_ConversationSummaryPlugin; +import com.microsoft.semantickernel.samples.syntaxexamples.plugin.Example10_DescribeAllPluginsAndFunctions; +import com.microsoft.semantickernel.samples.syntaxexamples.plugin.Example13_ConversationSummaryPlugin; import com.microsoft.semantickernel.samples.syntaxexamples.template.Example06_TemplateLanguage; import com.microsoft.semantickernel.samples.syntaxexamples.template.Example56_TemplateMethodFunctionsWithMultipleArguments; import com.microsoft.semantickernel.samples.syntaxexamples.template.Example64_MultiplePromptTemplates; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/RunAll.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/RunAll.java index f634a331..44a97979 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/RunAll.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/RunAll.java @@ -20,8 +20,8 @@ import com.microsoft.semantickernel.samples.syntaxexamples.functions.Example60_AdvancedMethodFunctions; import com.microsoft.semantickernel.samples.syntaxexamples.java.KernelFunctionYaml_Example; import com.microsoft.semantickernel.samples.syntaxexamples.memory.VectorStoreWithAzureAISearch; -import com.microsoft.semantickernel.samples.syntaxexamples.plugins.Example10_DescribeAllPluginsAndFunctions; -import com.microsoft.semantickernel.samples.syntaxexamples.plugins.Example13_ConversationSummaryPlugin; +import com.microsoft.semantickernel.samples.syntaxexamples.plugin.Example10_DescribeAllPluginsAndFunctions; +import com.microsoft.semantickernel.samples.syntaxexamples.plugin.Example13_ConversationSummaryPlugin; import com.microsoft.semantickernel.samples.syntaxexamples.template.Example06_TemplateLanguage; import com.microsoft.semantickernel.samples.syntaxexamples.template.Example56_TemplateMethodFunctionsWithMultipleArguments; import com.microsoft.semantickernel.samples.syntaxexamples.template.Example64_MultiplePromptTemplates; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugins/Example07_BingAndGooglePlugins.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugin/Example07_BingAndGooglePlugins.java similarity index 99% rename from samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugins/Example07_BingAndGooglePlugins.java rename to samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugin/Example07_BingAndGooglePlugins.java index 70a8ef28..6ca71745 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugins/Example07_BingAndGooglePlugins.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugin/Example07_BingAndGooglePlugins.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.samples.syntaxexamples.plugins; +package com.microsoft.semantickernel.samples.syntaxexamples.plugin; import com.azure.ai.openai.OpenAIAsyncClient; import com.azure.ai.openai.OpenAIClientBuilder; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugins/Example10_DescribeAllPluginsAndFunctions.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugin/Example10_DescribeAllPluginsAndFunctions.java similarity index 97% rename from samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugins/Example10_DescribeAllPluginsAndFunctions.java rename to samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugin/Example10_DescribeAllPluginsAndFunctions.java index 1205b335..7e4969d4 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugins/Example10_DescribeAllPluginsAndFunctions.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugin/Example10_DescribeAllPluginsAndFunctions.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.samples.syntaxexamples.plugins; +package com.microsoft.semantickernel.samples.syntaxexamples.plugin; import java.nio.file.Path; import java.util.Locale; @@ -11,7 +11,6 @@ import com.microsoft.semantickernel.Kernel; import com.microsoft.semantickernel.Kernel.Builder; import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIChatCompletion; -import com.microsoft.semantickernel.aiservices.openai.textcompletion.OpenAITextGenerationService; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; import com.microsoft.semantickernel.plugin.KernelPluginFactory; import com.microsoft.semantickernel.samples.plugins.text.TextPlugin; @@ -21,7 +20,6 @@ import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; import com.microsoft.semantickernel.semanticfunctions.annotations.KernelFunctionParameter; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; -import com.microsoft.semantickernel.services.textcompletion.TextGenerationService; public class Example10_DescribeAllPluginsAndFunctions { diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugins/Example13_ConversationSummaryPlugin.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugin/Example13_ConversationSummaryPlugin.java similarity index 98% rename from samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugins/Example13_ConversationSummaryPlugin.java rename to samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugin/Example13_ConversationSummaryPlugin.java index 9cdafbb7..b185c894 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugins/Example13_ConversationSummaryPlugin.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugin/Example13_ConversationSummaryPlugin.java @@ -1,5 +1,5 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.samples.syntaxexamples.plugins; +package com.microsoft.semantickernel.samples.syntaxexamples.plugin; import com.azure.ai.openai.OpenAIAsyncClient; import com.azure.ai.openai.OpenAIClientBuilder; @@ -7,7 +7,6 @@ import com.azure.core.credential.KeyCredential; import com.microsoft.semantickernel.Kernel; import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIChatCompletion; -import com.microsoft.semantickernel.aiservices.openai.textcompletion.OpenAITextGenerationService; import com.microsoft.semantickernel.exceptions.ConfigurationException; import com.microsoft.semantickernel.orchestration.FunctionResult; import com.microsoft.semantickernel.plugin.KernelPlugin; @@ -15,7 +14,6 @@ import com.microsoft.semantickernel.samples.plugins.ConversationSummaryPlugin; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments.Builder; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; -import com.microsoft.semantickernel.services.textcompletion.TextGenerationService; import reactor.core.publisher.Mono; /** From 0d16edbf36d77ff5f9bb0fbd306971acf45770e1 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 3 Dec 2024 11:42:26 -0800 Subject: [PATCH 104/277] Add compile-17 to text-splitter --- .../tests/ImportingMultiplePluginsTest.java | 4 +++- .../semantickernel/tests/ResponseSchemaTest.java | 2 ++ .../semantickernel-text-splitter-plugin/pom.xml | 13 +++++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ImportingMultiplePluginsTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ImportingMultiplePluginsTest.java index dc7abe14..a555937e 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ImportingMultiplePluginsTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ImportingMultiplePluginsTest.java @@ -4,11 +4,13 @@ import com.microsoft.semantickernel.Kernel; import com.microsoft.semantickernel.plugin.KernelPlugin; import com.microsoft.semantickernel.plugin.KernelPluginFactory; +import org.junit.Ignore; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class ImportingMultiplePluginsTest { - + @Disabled @Test public void canImportMultiplePlugins() { KernelPlugin summarize = KernelPluginFactory.importPluginFromResourcesDirectory( diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ResponseSchemaTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ResponseSchemaTest.java index 099aaa36..77d4b4a3 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ResponseSchemaTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ResponseSchemaTest.java @@ -28,6 +28,7 @@ import java.io.StringWriter; import org.apache.commons.text.StringEscapeUtils; import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.mockito.Mockito; import reactor.core.publisher.Mono; @@ -95,6 +96,7 @@ public void sendsResponseSchemaFromTemplate() throws IOException { ); } + @Disabled @Test public void sendsResponseSchema() { OpenAIAsyncClient client = getOpenAIAsyncClient( diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml index 9e26aaf4..5c7145ad 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml @@ -62,4 +62,17 @@ test + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 17 + 17 + + + + From 85b7014d898ca75f320b78effc4020677999eac4 Mon Sep 17 00:00:00 2001 From: GitHub Date: Tue, 3 Dec 2024 12:25:27 -0800 Subject: [PATCH 105/277] [maven-release-plugin] prepare release java-1.4.1 --- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- data/semantickernel-data-azureaisearch/pom.xml | 2 +- data/semantickernel-data-jdbc/pom.xml | 2 +- data/semantickernel-data-redis/pom.xml | 2 +- pom.xml | 4 ++-- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 5 ++--- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 5 ++--- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- .../semantickernel-text-splitter-plugin/pom.xml | 5 ++--- semantickernel-api/pom.xml | 5 ++--- semantickernel-bom/pom.xml | 4 ++-- semantickernel-experimental/pom.xml | 2 +- 24 files changed, 30 insertions(+), 34 deletions(-) diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index 187961bc..c1304ad9 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.1-SNAPSHOT + 1.4.1 ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index 950cfa62..80997d8b 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.1-SNAPSHOT + 1.4.1 ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index ab756185..7c546e22 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.1-SNAPSHOT + 1.4.1 ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 49c5f286..02bcfee6 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.4.1-SNAPSHOT + 1.4.1 ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index c88c19b3..3690972f 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.1-SNAPSHOT + 1.4.1 ../pom.xml diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml index 83ef7964..2340de1d 100644 --- a/data/semantickernel-data-azureaisearch/pom.xml +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.1-SNAPSHOT + 1.4.1 ../../pom.xml diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 76d57be9..22aa98da 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.1-SNAPSHOT + 1.4.1 ../../pom.xml diff --git a/data/semantickernel-data-redis/pom.xml b/data/semantickernel-data-redis/pom.xml index 7d26a385..bb47b58d 100644 --- a/data/semantickernel-data-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.1-SNAPSHOT + 1.4.1 ../../pom.xml diff --git a/pom.xml b/pom.xml index e4ca4797..19cebbee 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.1-SNAPSHOT + 1.4.1 pom https://www.github.com/microsoft/semantic-kernel @@ -837,6 +837,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - HEAD + java-1.4.1 diff --git a/samples/pom.xml b/samples/pom.xml index 9b1f4ea1..2a21e5ef 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.1-SNAPSHOT + 1.4.1 ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index 08f5522f..bb5764e7 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.1-SNAPSHOT + 1.4.1 ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 58c036ba..04a8833b 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -1,11 +1,10 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-concepts - 1.4.1-SNAPSHOT + 1.4.1 ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index b3862adc..47be7f7e 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.1-SNAPSHOT + 1.4.1 ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index 662da823..62caea3a 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.1-SNAPSHOT + 1.4.1 ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index 346d6a83..f6a080f2 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.1-SNAPSHOT + 1.4.1 ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index c8b40a25..90052da5 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.1-SNAPSHOT + 1.4.1 ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index d788980e..f2587332 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.1-SNAPSHOT + 1.4.1 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index 9241c19e..e1f36672 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -1,11 +1,10 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.1-SNAPSHOT + 1.4.1 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index 6aab17b8..58759e93 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.1-SNAPSHOT + 1.4.1 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index 197cb0c4..3716f194 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.1-SNAPSHOT + 1.4.1 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml index 5c7145ad..5c4da03c 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml @@ -1,11 +1,10 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.1-SNAPSHOT + 1.4.1 ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index 2846ea9d..714d46da 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -1,13 +1,12 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.4.1-SNAPSHOT + 1.4.1 ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index c74f0c3e..baf880ed 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.4.1-SNAPSHOT + 1.4.1 pom Semantic Kernel Java BOM @@ -290,6 +290,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - HEAD + java-1.4.1 diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 2ef8f7cd..cd0496bc 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.1-SNAPSHOT + 1.4.1 semantickernel-experimental From ef4b9257a16a454a7baf70aa1b204f2f5971ab1d Mon Sep 17 00:00:00 2001 From: GitHub Date: Tue, 3 Dec 2024 12:25:30 -0800 Subject: [PATCH 106/277] [maven-release-plugin] prepare for next development iteration --- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- data/semantickernel-data-azureaisearch/pom.xml | 2 +- data/semantickernel-data-jdbc/pom.xml | 2 +- data/semantickernel-data-redis/pom.xml | 2 +- pom.xml | 4 ++-- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- .../semantickernel-text-splitter-plugin/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-bom/pom.xml | 4 ++-- semantickernel-experimental/pom.xml | 2 +- 24 files changed, 26 insertions(+), 26 deletions(-) diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index c1304ad9..68db44dd 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.1 + 1.4.2-SNAPSHOT ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index 80997d8b..cd94e059 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.1 + 1.4.2-SNAPSHOT ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index 7c546e22..16fa0b84 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.1 + 1.4.2-SNAPSHOT ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 02bcfee6..cdb223b6 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.4.1 + 1.4.2-SNAPSHOT ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index 3690972f..523fd465 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.1 + 1.4.2-SNAPSHOT ../pom.xml diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml index 2340de1d..d420d030 100644 --- a/data/semantickernel-data-azureaisearch/pom.xml +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.1 + 1.4.2-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 22aa98da..107a9a40 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.1 + 1.4.2-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-redis/pom.xml b/data/semantickernel-data-redis/pom.xml index bb47b58d..e61af886 100644 --- a/data/semantickernel-data-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.1 + 1.4.2-SNAPSHOT ../../pom.xml diff --git a/pom.xml b/pom.xml index 19cebbee..e008ebd6 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.1 + 1.4.2-SNAPSHOT pom https://www.github.com/microsoft/semantic-kernel @@ -837,6 +837,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.4.1 + HEAD diff --git a/samples/pom.xml b/samples/pom.xml index 2a21e5ef..3acfc5fb 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.1 + 1.4.2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index bb5764e7..58125312 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.1 + 1.4.2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 04a8833b..e047daf7 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.4.1 + 1.4.2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index 47be7f7e..3a36a00f 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.1 + 1.4.2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index 62caea3a..e9b8feb0 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.1 + 1.4.2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index f6a080f2..3ea69052 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.1 + 1.4.2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index 90052da5..734961a0 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.1 + 1.4.2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index f2587332..80348724 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.1 + 1.4.2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index e1f36672..a5abdff8 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.1 + 1.4.2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index 58759e93..c90cb79c 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.1 + 1.4.2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index 3716f194..ab2bb5df 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.1 + 1.4.2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml index 5c4da03c..ef206c25 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.1 + 1.4.2-SNAPSHOT ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index 714d46da..261cf2e2 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.1 + 1.4.2-SNAPSHOT ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index baf880ed..49ca1566 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.4.1 + 1.4.2-SNAPSHOT pom Semantic Kernel Java BOM @@ -290,6 +290,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.4.1 + HEAD diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index cd0496bc..15b3a1cc 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.1 + 1.4.2-SNAPSHOT semantickernel-experimental From f59e19ce1bd66f3585b0b162da3da8663a93c72f Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Thu, 5 Dec 2024 18:54:07 +0000 Subject: [PATCH 107/277] Fix bad type matching on java types Fix npe --- CHANGELOG.md | 4 ++++ .../aiservices/openai/chatcompletion/OpenAIFunction.java | 9 +++++++++ .../com/microsoft/semantickernel/hooks/KernelHook.java | 7 +++++-- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aff20f82..592ca820 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 1.4.2 + +- Fix bug effecting using native Java methods with OpenAI tool calling + # 1.4.1 - Add Otel Telemetry on function invocations diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java index 79c7d617..ad6dbadb 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java @@ -189,21 +189,30 @@ private static String getSchemaForFunctionParameter(@Nullable InputVariable para private static String getJavaTypeToOpenAiFunctionType(String javaType) { switch (javaType.toLowerCase(Locale.ROOT)) { + case "java.lang.boolean": case "boolean": return "boolean"; + case "java.lang.integer": case "integer": case "int": + case "java.lang.long": case "long": + case "java.lang.short": case "short": + case "java.lang.byte": case "byte": return "integer"; + case "java.lang.double": case "double": + case "java.lang.float": case "float": return "number"; + case "java.lang.string": case "string": return "string"; case "array": return "array"; + case "java.lang.void": case "void": return "null"; default: diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/KernelHook.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/KernelHook.java index f8f4b939..93842049 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/KernelHook.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/KernelHook.java @@ -115,8 +115,11 @@ static ChatCompletionsOptions cloneOptionsWithMessages( .setN(options.getN()) .setResponseFormat(options.getResponseFormat()) .setSeed(options.getSeed()) - .setStream(options.isStream()) - .setToolChoice(options.getToolChoice()); + .setStream(options.isStream()); + + if (options.getToolChoice() != null) { + newOptions.setToolChoice(options.getToolChoice()); + } if (options.getFunctionCall() != null) { newOptions = newOptions.setFunctionCall(options.getFunctionCall()); From 2846dde7239a65b48c0fac6641dd5e6c4bd516dc Mon Sep 17 00:00:00 2001 From: GitHub Date: Thu, 5 Dec 2024 19:45:59 +0000 Subject: [PATCH 108/277] [maven-release-plugin] prepare release java-1.4.2 --- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- data/semantickernel-data-azureaisearch/pom.xml | 2 +- data/semantickernel-data-jdbc/pom.xml | 2 +- data/semantickernel-data-redis/pom.xml | 2 +- pom.xml | 4 ++-- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- .../semantickernel-text-splitter-plugin/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-bom/pom.xml | 4 ++-- semantickernel-experimental/pom.xml | 2 +- 24 files changed, 26 insertions(+), 26 deletions(-) diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index 68db44dd..9fa20e54 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2-SNAPSHOT + 1.4.2 ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index cd94e059..c0ebdd8e 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2-SNAPSHOT + 1.4.2 ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index 16fa0b84..fcb6ac56 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2-SNAPSHOT + 1.4.2 ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index cdb223b6..1f2b52ab 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.4.2-SNAPSHOT + 1.4.2 ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index 523fd465..adc12a93 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2-SNAPSHOT + 1.4.2 ../pom.xml diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml index d420d030..c0a70ef0 100644 --- a/data/semantickernel-data-azureaisearch/pom.xml +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2-SNAPSHOT + 1.4.2 ../../pom.xml diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 107a9a40..f229617f 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2-SNAPSHOT + 1.4.2 ../../pom.xml diff --git a/data/semantickernel-data-redis/pom.xml b/data/semantickernel-data-redis/pom.xml index e61af886..47122787 100644 --- a/data/semantickernel-data-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2-SNAPSHOT + 1.4.2 ../../pom.xml diff --git a/pom.xml b/pom.xml index e008ebd6..0d9fd57d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2-SNAPSHOT + 1.4.2 pom https://www.github.com/microsoft/semantic-kernel @@ -837,6 +837,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - HEAD + java-1.4.2 diff --git a/samples/pom.xml b/samples/pom.xml index 3acfc5fb..48ec422d 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2-SNAPSHOT + 1.4.2 ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index 58125312..21c1bbc4 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.2-SNAPSHOT + 1.4.2 ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index e047daf7..b3150ab6 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.4.2-SNAPSHOT + 1.4.2 ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index 3a36a00f..a6e5a01b 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.2-SNAPSHOT + 1.4.2 ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index e9b8feb0..77fb1f85 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.2-SNAPSHOT + 1.4.2 ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index 3ea69052..fc127340 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.2-SNAPSHOT + 1.4.2 ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index 734961a0..1c3d7454 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.2-SNAPSHOT + 1.4.2 ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 80348724..984fe126 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.2-SNAPSHOT + 1.4.2 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index a5abdff8..aa5d1437 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.2-SNAPSHOT + 1.4.2 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index c90cb79c..99b29b1e 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.2-SNAPSHOT + 1.4.2 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index ab2bb5df..62b4e281 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.2-SNAPSHOT + 1.4.2 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml index ef206c25..028532e2 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.2-SNAPSHOT + 1.4.2 ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index 261cf2e2..4a5c8de6 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2-SNAPSHOT + 1.4.2 ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 49ca1566..a2e796d6 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.4.2-SNAPSHOT + 1.4.2 pom Semantic Kernel Java BOM @@ -290,6 +290,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - HEAD + java-1.4.2 diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 15b3a1cc..20fd1243 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2-SNAPSHOT + 1.4.2 semantickernel-experimental From f822232478b010195b2541d81008b147a8f60312 Mon Sep 17 00:00:00 2001 From: GitHub Date: Thu, 5 Dec 2024 19:45:59 +0000 Subject: [PATCH 109/277] [maven-release-plugin] prepare for next development iteration --- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- data/semantickernel-data-azureaisearch/pom.xml | 2 +- data/semantickernel-data-jdbc/pom.xml | 2 +- data/semantickernel-data-redis/pom.xml | 2 +- pom.xml | 4 ++-- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- .../semantickernel-text-splitter-plugin/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-bom/pom.xml | 4 ++-- semantickernel-experimental/pom.xml | 2 +- 24 files changed, 26 insertions(+), 26 deletions(-) diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index 9fa20e54..88676005 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2 + 1.4.3-SNAPSHOT ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index c0ebdd8e..63fac182 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2 + 1.4.3-SNAPSHOT ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index fcb6ac56..76c2d9bb 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2 + 1.4.3-SNAPSHOT ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 1f2b52ab..7cb17ce2 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.4.2 + 1.4.3-SNAPSHOT ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index adc12a93..437dff8f 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2 + 1.4.3-SNAPSHOT ../pom.xml diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml index c0a70ef0..74d9fe21 100644 --- a/data/semantickernel-data-azureaisearch/pom.xml +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2 + 1.4.3-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index f229617f..39c0fbd2 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2 + 1.4.3-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-redis/pom.xml b/data/semantickernel-data-redis/pom.xml index 47122787..2ae74fbe 100644 --- a/data/semantickernel-data-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2 + 1.4.3-SNAPSHOT ../../pom.xml diff --git a/pom.xml b/pom.xml index 0d9fd57d..c9e649ab 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2 + 1.4.3-SNAPSHOT pom https://www.github.com/microsoft/semantic-kernel @@ -837,6 +837,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.4.2 + HEAD diff --git a/samples/pom.xml b/samples/pom.xml index 48ec422d..d1069761 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2 + 1.4.3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index 21c1bbc4..f2b8646e 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.2 + 1.4.3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index b3150ab6..14e51a97 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.4.2 + 1.4.3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index a6e5a01b..2ddd75cc 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.2 + 1.4.3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index 77fb1f85..1ecbea82 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.2 + 1.4.3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index fc127340..c012d147 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.2 + 1.4.3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index 1c3d7454..f3fc3090 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.2 + 1.4.3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 984fe126..5a8b5886 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.2 + 1.4.3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index aa5d1437..b91ab27c 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.2 + 1.4.3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index 99b29b1e..ff2bb46f 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.2 + 1.4.3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index 62b4e281..f8ad6720 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.2 + 1.4.3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml index 028532e2..32995921 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.2 + 1.4.3-SNAPSHOT ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index 4a5c8de6..9fb640ae 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2 + 1.4.3-SNAPSHOT ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index a2e796d6..30265d9f 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.4.2 + 1.4.3-SNAPSHOT pom Semantic Kernel Java BOM @@ -290,6 +290,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.4.2 + HEAD diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 20fd1243..5ee83333 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.2 + 1.4.3-SNAPSHOT semantickernel-experimental From 5d6d8d0a37cf58c575083df6a322b1d83254a288 Mon Sep 17 00:00:00 2001 From: Sami MEJRI Date: Sat, 15 Feb 2025 20:17:04 +0100 Subject: [PATCH 110/277] Use stax-api newInstance() instead of newFactory() for Android compatibility. --- .../implementation/chatcompletion/ChatXMLPromptParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/chatcompletion/ChatXMLPromptParser.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/chatcompletion/ChatXMLPromptParser.java index c1c9592a..8cc1cc97 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/chatcompletion/ChatXMLPromptParser.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/chatcompletion/ChatXMLPromptParser.java @@ -109,7 +109,7 @@ private static ChatPromptParseVisitor getFunctionDefinitions(String promp // try (InputStream is = new ByteArrayInputStream(prompt.getBytes(StandardCharsets.UTF_8))) { - XMLInputFactory factory = XMLInputFactory.newFactory(); + XMLInputFactory factory = XMLInputFactory.newInstance(); XMLEventReader reader = factory.createXMLEventReader(is); FunctionDefinition functionDefinition = null; Map parameters = new HashMap<>(); From 87dec3b3d3f01b3e71c4ba5b92de4d60997ee2e3 Mon Sep 17 00:00:00 2001 From: SridharNagaraj Date: Thu, 20 Feb 2025 12:26:55 +0530 Subject: [PATCH 111/277] Support OpenAIFunction Custom object Schema --- .../openai/chatcompletion/OpenAIFunction.java | 37 +++++++- .../openai/chatcompletion/JsonSchemaTest.java | 87 +++++++++++++++++++ .../samples/demos/lights/App.java | 8 +- .../samples/demos/lights/LightModel.java | 7 ++ .../demos/lights/LightModelTypeConverter.java | 23 +++++ .../samples/demos/lights/LightsPlugin.java | 11 +++ 6 files changed, 163 insertions(+), 10 deletions(-) create mode 100644 samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightModelTypeConverter.java diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java index ad6dbadb..dbf6ff95 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java @@ -7,19 +7,25 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.microsoft.semantickernel.exceptions.SKException; +import com.microsoft.semantickernel.orchestration.responseformat.ResponseSchemaGenerator; import com.microsoft.semantickernel.semanticfunctions.InputVariable; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionMetadata; +import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; class OpenAIFunction { + private static final ConcurrentHashMap SCHEMA_CACHE = new ConcurrentHashMap<>(); private final String pluginName; private final String name; @@ -159,14 +165,17 @@ private static String getSchemaForFunctionParameter(@Nullable InputVariable para entries.add("\"type\":\"" + type + "\""); // Add description if present + String description =null; if (parameter != null && parameter.getDescription() != null && !parameter.getDescription() .isEmpty()) { - String description = parameter.getDescription(); + description = parameter.getDescription(); description = description.replaceAll("\\r?\\n|\\r", ""); description = description.replace("\"", "\\\""); - - description = String.format("\"description\":\"%s\"", description); - entries.add(description); + entries.add(String.format("\"description\":\"%s\"", description)); + } + // If custom type, generate schema + if("object".equalsIgnoreCase(type)) { + return getObjectSchema(parameter.getType(), description); } // Add enum options if parameter is an enum @@ -219,4 +228,24 @@ private static String getJavaTypeToOpenAiFunctionType(String javaType) { return "object"; } } + + private static String getObjectSchema(String type, String description){ + String schema= ""; + try { + if(SCHEMA_CACHE.containsKey(type)) { + schema= SCHEMA_CACHE.get(type); + } else { + Class clazz = Class.forName(type); + schema = ResponseSchemaGenerator.jacksonGenerator().generateSchema(clazz); + SCHEMA_CACHE.put(type, schema); + } + } catch (ClassNotFoundException | SKException ignored) { + + } + Map properties = BinaryData.fromString(schema).toObject(Map.class); + if(StringUtils.isNotBlank(description)) { + properties.put("description", description); + } + return BinaryData.fromObject(properties).toString(); + } } diff --git a/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/JsonSchemaTest.java b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/JsonSchemaTest.java index d0cdc0d9..3b42981a 100644 --- a/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/JsonSchemaTest.java +++ b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/JsonSchemaTest.java @@ -1,10 +1,17 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.aiservices.openai.chatcompletion; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.fasterxml.jackson.core.JsonProcessingException; import com.microsoft.semantickernel.orchestration.responseformat.JsonSchemaResponseFormat; +import com.microsoft.semantickernel.plugin.KernelPlugin; +import com.microsoft.semantickernel.plugin.KernelPluginFactory; +import com.microsoft.semantickernel.semanticfunctions.KernelFunction; +import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; +import com.microsoft.semantickernel.semanticfunctions.annotations.KernelFunctionParameter; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import reactor.core.publisher.Mono; public class JsonSchemaTest { @@ -24,4 +31,84 @@ public void jacksonGenerationTest() throws JsonProcessingException { "\"type\":\"object\",\"properties\":{\"bar\":{}}")); } + @Test + public void openAIFunctionTest() { + KernelPlugin plugin = KernelPluginFactory.createFromObject( + new TestPlugin(), + "test"); + + Assertions.assertNotNull(plugin); + Assertions.assertEquals(plugin.getName(), "test"); + Assertions.assertEquals(plugin.getFunctions().size(), 3); + + KernelFunction testFunction = plugin.getFunctions() + .get("asyncPersonFunction"); + OpenAIFunction openAIFunction = OpenAIFunction.build( + testFunction.getMetadata(), + plugin.getName()); + System.out.println(openAIFunction.getFunctionDefinition()); + + } + + + public static class TestPlugin { + + @DefineKernelFunction + public String testFunction( + @KernelFunctionParameter(name = "input", description = "input string") String input) { + return "test" + input; + } + + @DefineKernelFunction(returnType = "int") + public Mono asyncTestFunction( + @KernelFunctionParameter(name = "input") String input) { + return Mono.just(1); + } + + @DefineKernelFunction(returnType = "int", description = "test function description", + name = "asyncPersonFunction", returnDescription = "test return description") + public Mono asyncPersonFunction( + @KernelFunctionParameter(name = "person",description = "input person", type = Person.class) Person person, + @KernelFunctionParameter(name = "input", description = "input string") String input) { + return Mono.just(1); + } + } + + private static enum Title { + MS, + MRS, + MR + } + + public static class Person { + @JsonPropertyDescription("The name of the person.") + private String name; + @JsonPropertyDescription("The age of the person.") + private int age; + @JsonPropertyDescription("The title of the person.") + private Title title; + + + public Person(String name, int age) { + this.name = name; + this.age = age; + } + + public String getName() { + return name; + } + + public int getAge() { + return age; + } + + public Title getTitle() { + return title; + } + + public void setTitle(Title title) { + this.title = title; + } + } + } diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/App.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/App.java index 08a8f3b5..21a4b1c8 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/App.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/App.java @@ -70,9 +70,7 @@ public static void main(String[] args) throws Exception { ChatCompletionService.class); ContextVariableTypes - .addGlobalConverter(ContextVariableTypeConverter.builder(LightModel.class) - .toPromptString(new Gson()::toJson) - .build()); + .addGlobalConverter(new LightModelTypeConverter()); KernelHooks hook = new KernelHooks(); @@ -99,9 +97,7 @@ public static void main(String[] args) throws Exception { InvocationContext invocationContext = new Builder() .withReturnMode(InvocationReturnMode.LAST_MESSAGE_ONLY) .withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(true)) - .withContextVariableConverter(ContextVariableTypeConverter.builder(LightModel.class) - .toPromptString(new Gson()::toJson) - .build()) + .withContextVariableConverter(new LightModelTypeConverter()) .build(); // Create a history to store the conversation diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightModel.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightModel.java index 072763a9..e7958507 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightModel.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightModel.java @@ -1,10 +1,17 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.samples.demos.lights; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; + public class LightModel { + @JsonPropertyDescription("The unique identifier of the light") private int id; + + @JsonPropertyDescription("The name of the light") private String name; + + @JsonPropertyDescription("The state of the light") private Boolean isOn; public LightModel(int id, String name, Boolean isOn) { diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightModelTypeConverter.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightModelTypeConverter.java new file mode 100644 index 00000000..250de12e --- /dev/null +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightModelTypeConverter.java @@ -0,0 +1,23 @@ +package com.microsoft.semantickernel.samples.demos.lights; + +import com.google.gson.Gson; +import com.microsoft.semantickernel.contextvariables.ContextVariableTypeConverter; + +public class LightModelTypeConverter extends ContextVariableTypeConverter { + private static final Gson gson = new Gson(); + + public LightModelTypeConverter() { + super( + LightModel.class, + obj -> { + if(obj instanceof String) { + return gson.fromJson((String)obj, LightModel.class); + } else { + return gson.fromJson(gson.toJson(obj), LightModel.class); + } + }, + (types, lightModel) -> gson.toJson(lightModel), + json -> gson.fromJson(json, LightModel.class) + ); + } +} diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightsPlugin.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightsPlugin.java index d2f4ed08..fa11addb 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightsPlugin.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightsPlugin.java @@ -24,6 +24,17 @@ public List getLights() { return lights; } + @DefineKernelFunction(name = "add_light", description = "Adds a new light") + public String addLight( + @KernelFunctionParameter(name = "newLight", description = "new Light Details", type = LightModel.class) LightModel light) { + if( light != null) { + System.out.println("Adding light " + light.getName()); + lights.add(light); + return "Light added"; + } + return "Light failed to added"; + } + @DefineKernelFunction(name = "change_state", description = "Changes the state of the light") public LightModel changeState( @KernelFunctionParameter(name = "id", description = "The ID of the light to change", type = int.class) int id, From b71458aa8c500778da5c945460a0a552f770c6d1 Mon Sep 17 00:00:00 2001 From: SridharNagaraj Date: Sun, 23 Feb 2025 13:52:51 +0530 Subject: [PATCH 112/277] Remove cache and updated testcase --- .../aiservices/openai/chatcompletion/OpenAIFunction.java | 9 ++------- .../aiservices/openai/chatcompletion/JsonSchemaTest.java | 4 +++- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java index dbf6ff95..e1f2f249 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java @@ -25,7 +25,6 @@ import javax.annotation.Nullable; class OpenAIFunction { - private static final ConcurrentHashMap SCHEMA_CACHE = new ConcurrentHashMap<>(); private final String pluginName; private final String name; @@ -230,15 +229,11 @@ private static String getJavaTypeToOpenAiFunctionType(String javaType) { } private static String getObjectSchema(String type, String description){ - String schema= ""; + String schema= "{ \"type\" : \"object\" }"; try { - if(SCHEMA_CACHE.containsKey(type)) { - schema= SCHEMA_CACHE.get(type); - } else { Class clazz = Class.forName(type); schema = ResponseSchemaGenerator.jacksonGenerator().generateSchema(clazz); - SCHEMA_CACHE.put(type, schema); - } + } catch (ClassNotFoundException | SKException ignored) { } diff --git a/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/JsonSchemaTest.java b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/JsonSchemaTest.java index 3b42981a..68c8dae8 100644 --- a/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/JsonSchemaTest.java +++ b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/JsonSchemaTest.java @@ -46,7 +46,9 @@ public void openAIFunctionTest() { OpenAIFunction openAIFunction = OpenAIFunction.build( testFunction.getMetadata(), plugin.getName()); - System.out.println(openAIFunction.getFunctionDefinition()); + + String parameters = "{\"type\":\"object\",\"required\":[\"person\",\"input\"],\"properties\":{\"input\":{\"type\":\"string\",\"description\":\"input string\"},\"person\":{\"type\":\"object\",\"properties\":{\"age\":{\"type\":\"integer\",\"description\":\"The age of the person.\"},\"name\":{\"type\":\"string\",\"description\":\"The name of the person.\"},\"title\":{\"type\":\"string\",\"enum\":[\"MS\",\"MRS\",\"MR\"],\"description\":\"The title of the person.\"}},\"required\":[\"age\",\"name\",\"title\"],\"additionalProperties\":false,\"description\":\"input person\"}}}"; + Assertions.assertEquals(parameters, openAIFunction.getFunctionDefinition().getParameters().toString()); } From 0a8681b540546e29ffa1b9e76affe9a0052b044f Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Mon, 24 Feb 2025 10:24:30 -0800 Subject: [PATCH 113/277] Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 592ca820..79c1c9ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 1.4.3 + +- Bug fix for execution on Android (https://github.com/microsoft/semantic-kernel-java/pull/284) + # 1.4.2 - Fix bug effecting using native Java methods with OpenAI tool calling From 1f49fd62440290b019509d7c9bba2e0a12e4d17c Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Mon, 24 Feb 2025 18:28:28 -0800 Subject: [PATCH 114/277] Upgrade azure-ai-openai to 1.0.0-beta.14 --- CHANGELOG.md | 1 + pom.xml | 2 +- .../java/com/microsoft/semantickernel/hooks/KernelHook.java | 4 ++-- semantickernel-bom/pom.xml | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79c1c9ca..0630f207 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # 1.4.3 - Bug fix for execution on Android (https://github.com/microsoft/semantic-kernel-java/pull/284) +- Upgrade to azure-ai-openai 1.0.0-beta.14 # 1.4.2 diff --git a/pom.xml b/pom.xml index c9e649ab..82e0acbe 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ Parent pom for the Semantic Kernel Project - 1.0.0-beta.12 + 1.0.0-beta.14 10.18.2 0.10.21 false diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/KernelHook.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/KernelHook.java index 93842049..e4d3f530 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/KernelHook.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/KernelHook.java @@ -98,6 +98,7 @@ interface PreChatCompletionHook extends KernelHook { static ChatCompletionsOptions cloneOptionsWithMessages( ChatCompletionsOptions options, List messages) { + ChatCompletionsOptions newOptions = new ChatCompletionsOptions(messages) .setPresencePenalty(options.getPresencePenalty()) .setFrequencyPenalty(options.getFrequencyPenalty()) @@ -114,8 +115,7 @@ static ChatCompletionsOptions cloneOptionsWithMessages( .setFunctions(options.getFunctions()) .setN(options.getN()) .setResponseFormat(options.getResponseFormat()) - .setSeed(options.getSeed()) - .setStream(options.isStream()); + .setSeed(options.getSeed()); if (options.getToolChoice() != null) { newOptions.setToolChoice(options.getToolChoice()); diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 30265d9f..70d14cd5 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -115,7 +115,7 @@ com.azure azure-ai-openai - 1.0.0-beta.12 + 1.0.0-beta.14 com.azure From 7bb6883561a07e8bdd87ca9a1acce94a28d42053 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Mon, 24 Feb 2025 18:50:06 -0800 Subject: [PATCH 115/277] Fix tests --- .../microsoft/semantickernel/tests/ToolCallBehaviourTest.java | 4 ++-- ...chat_completions-12b6744e-443f-4fe7-82e2-55cc41195ff1.json | 2 +- ...chat_completions-71b07779-49a8-44e5-a60b-ee5b0a3ad697.json | 2 +- ...chat_completions-0c8046c5-74ad-4836-8aa9-09da60f367a2.json | 2 +- ...-003_completions-0215b128-4822-4368-ac3d-2f580a221f00.json | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ToolCallBehaviourTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ToolCallBehaviourTest.java index 09f88df0..5f80a6e0 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ToolCallBehaviourTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ToolCallBehaviourTest.java @@ -282,7 +282,7 @@ private ChatCompletionService getChatCompletionService() { public static MappingBuilder buildTextResponse(String bodyMatcher, String responseBody) { return post(urlEqualTo( - "//openai/deployments/gpt-35-turbo-2/completions?api-version=2024-08-01-preview")) + "//openai/deployments/gpt-35-turbo-2/completions?api-version=2025-01-01-preview")) .withRequestBody(new ContainsPattern(bodyMatcher)) .willReturn( aResponse() @@ -293,7 +293,7 @@ public static MappingBuilder buildTextResponse(String bodyMatcher, String respon public static MappingBuilder buildResponse(String bodyMatcher, String responseBody) { return post(urlEqualTo( - "//openai/deployments/gpt-35-turbo-2/chat/completions?api-version=2024-08-01-preview")) + "//openai/deployments/gpt-35-turbo-2/chat/completions?api-version=2025-01-01-preview")) .withRequestBody(new ContainsPattern(bodyMatcher)) .willReturn( aResponse() diff --git a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-12b6744e-443f-4fe7-82e2-55cc41195ff1.json b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-12b6744e-443f-4fe7-82e2-55cc41195ff1.json index cdc9cbd0..4b27e3ba 100644 --- a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-12b6744e-443f-4fe7-82e2-55cc41195ff1.json +++ b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-12b6744e-443f-4fe7-82e2-55cc41195ff1.json @@ -2,7 +2,7 @@ "priority": 1, "request": { "method": "POST", - "url": "//openai/deployments/gpt-35-turbo-2/chat/completions?api-version=2024-08-01-preview", + "url": "//openai/deployments/gpt-35-turbo-2/chat/completions?api-version=2025-01-01-preview", "bodyPatterns": [ { "contains": "That is all" diff --git a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-71b07779-49a8-44e5-a60b-ee5b0a3ad697.json b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-71b07779-49a8-44e5-a60b-ee5b0a3ad697.json index 0d71a457..208f334c 100644 --- a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-71b07779-49a8-44e5-a60b-ee5b0a3ad697.json +++ b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo-2_chat_completions-71b07779-49a8-44e5-a60b-ee5b0a3ad697.json @@ -1,7 +1,7 @@ { "request": { "method": "POST", - "url": "//openai/deployments/gpt-35-turbo-2/chat/completions?api-version=2024-08-01-preview" + "url": "//openai/deployments/gpt-35-turbo-2/chat/completions?api-version=2025-01-01-preview" }, "response": { "body": "{\"id\":\"chatcmpl-xxx\",\"object\":\"chat.completion\",\"created\":1707253061,\"model\":\"gpt-35-turbo\",\"prompt_filter_results\":[{\"prompt_index\":0,\"content_filter_results\":{\"hate\":{\"filtered\":false,\"severity\":\"safe\"},\"self_harm\":{\"filtered\":false,\"severity\":\"safe\"},\"sexual\":{\"filtered\":false,\"severity\":\"safe\"},\"violence\":{\"filtered\":false,\"severity\":\"safe\"}}}],\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"1979b4ce-5463-4cfb-8ec8-1d05c4b44ccf\"},\"content_filter_results\":{\"hate\":{\"filtered\":false,\"severity\":\"safe\"},\"self_harm\":{\"filtered\":false,\"severity\":\"safe\"},\"sexual\":{\"filtered\":false,\"severity\":\"safe\"},\"violence\":{\"filtered\":false,\"severity\":\"safe\"}}}],\"usage\":{\"prompt_tokens\":17,\"completion_tokens\":67,\"total_tokens\":84}}", diff --git a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo_chat_completions-0c8046c5-74ad-4836-8aa9-09da60f367a2.json b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo_chat_completions-0c8046c5-74ad-4836-8aa9-09da60f367a2.json index 53becbdf..766450c6 100644 --- a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo_chat_completions-0c8046c5-74ad-4836-8aa9-09da60f367a2.json +++ b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_gpt-35-turbo_chat_completions-0c8046c5-74ad-4836-8aa9-09da60f367a2.json @@ -1,7 +1,7 @@ { "request": { "method": "POST", - "url": "//openai/deployments/gpt-35-turbo/chat/completions?api-version=2024-08-01-preview" + "url": "//openai/deployments/gpt-35-turbo/chat/completions?api-version=2025-01-01-preview" }, "response": { "body": "{\"id\":\"chatcmpl-xxx\",\"object\":\"chat.completion\",\"created\":1707253039,\"model\":\"gpt-35-turbo\",\"prompt_filter_results\":[{\"prompt_index\":0,\"content_filter_results\":{\"hate\":{\"filtered\":false,\"severity\":\"safe\"},\"self_harm\":{\"filtered\":false,\"severity\":\"safe\"},\"sexual\":{\"filtered\":false,\"severity\":\"safe\"},\"violence\":{\"filtered\":false,\"severity\":\"safe\"}}}],\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"message\":{\"role\":\"assistant\",\"content\":\"ac9817bc-7e1a-48e4-b06c-0ff7618b88c6\"},\"content_filter_results\":{\"hate\":{\"filtered\":false,\"severity\":\"safe\"},\"self_harm\":{\"filtered\":false,\"severity\":\"safe\"},\"sexual\":{\"filtered\":false,\"severity\":\"safe\"},\"violence\":{\"filtered\":false,\"severity\":\"safe\"}}}],\"usage\":{\"prompt_tokens\":26,\"completion_tokens\":131,\"total_tokens\":157}}", diff --git a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_text-davinci-003_completions-0215b128-4822-4368-ac3d-2f580a221f00.json b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_text-davinci-003_completions-0215b128-4822-4368-ac3d-2f580a221f00.json index 7ac0518d..56bccb38 100644 --- a/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_text-davinci-003_completions-0215b128-4822-4368-ac3d-2f580a221f00.json +++ b/api-test/integration-tests/src/test/resources/wiremock/mappings/deployments_text-davinci-003_completions-0215b128-4822-4368-ac3d-2f580a221f00.json @@ -1,7 +1,7 @@ { "request": { "method": "POST", - "url": "//openai/deployments/text-davinci-003/completions?api-version=2024-08-01-preview" + "url": "//openai/deployments/text-davinci-003/completions?api-version=2025-01-01-preview" }, "response": { "body": "{\"id\":\"cmpl-xxx\",\"object\":\"text_completion\",\"created\":1707253062,\"model\":\"text-davinci-003\",\"prompt_filter_results\":[{\"prompt_index\":0,\"content_filter_results\":{\"hate\":{\"filtered\":false,\"severity\":\"safe\"},\"self_harm\":{\"filtered\":false,\"severity\":\"safe\"},\"sexual\":{\"filtered\":false,\"severity\":\"safe\"},\"violence\":{\"filtered\":false,\"severity\":\"safe\"}}}],\"choices\":[{\"text\":\"7949e981-c632-422f-9b76-335a2379cd83\",\"index\":0,\"finish_reason\":\"stop\",\"logprobs\":null,\"content_filter_results\":{\"hate\":{\"filtered\":false,\"severity\":\"safe\"},\"self_harm\":{\"filtered\":false,\"severity\":\"safe\"},\"sexual\":{\"filtered\":false,\"severity\":\"safe\"},\"violence\":{\"filtered\":false,\"severity\":\"safe\"}}}],\"usage\":{\"prompt_tokens\":10,\"completion_tokens\":26,\"total_tokens\":36}}", From afdfd96061f99bb0743710bfebe1db1777aa50bd Mon Sep 17 00:00:00 2001 From: GitHub Date: Tue, 25 Feb 2025 17:19:19 -0800 Subject: [PATCH 116/277] [maven-release-plugin] prepare release java-1.4.3 --- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- data/semantickernel-data-azureaisearch/pom.xml | 2 +- data/semantickernel-data-jdbc/pom.xml | 2 +- data/semantickernel-data-redis/pom.xml | 2 +- pom.xml | 4 ++-- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- .../semantickernel-text-splitter-plugin/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-bom/pom.xml | 4 ++-- semantickernel-experimental/pom.xml | 2 +- 24 files changed, 26 insertions(+), 26 deletions(-) diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index 88676005..8c6ac73d 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3-SNAPSHOT + 1.4.3 ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index 63fac182..8ca1d183 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3-SNAPSHOT + 1.4.3 ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index 76c2d9bb..34c1477d 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3-SNAPSHOT + 1.4.3 ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 7cb17ce2..64c122e9 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.4.3-SNAPSHOT + 1.4.3 ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index 437dff8f..674f733b 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3-SNAPSHOT + 1.4.3 ../pom.xml diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml index 74d9fe21..c53436bb 100644 --- a/data/semantickernel-data-azureaisearch/pom.xml +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3-SNAPSHOT + 1.4.3 ../../pom.xml diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 39c0fbd2..0379f981 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3-SNAPSHOT + 1.4.3 ../../pom.xml diff --git a/data/semantickernel-data-redis/pom.xml b/data/semantickernel-data-redis/pom.xml index 2ae74fbe..e2c18c2d 100644 --- a/data/semantickernel-data-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3-SNAPSHOT + 1.4.3 ../../pom.xml diff --git a/pom.xml b/pom.xml index 82e0acbe..ab526a96 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3-SNAPSHOT + 1.4.3 pom https://www.github.com/microsoft/semantic-kernel @@ -837,6 +837,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - HEAD + java-1.4.3 diff --git a/samples/pom.xml b/samples/pom.xml index d1069761..d78bc0dd 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3-SNAPSHOT + 1.4.3 ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index f2b8646e..70f831df 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.3-SNAPSHOT + 1.4.3 ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 14e51a97..a3775208 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.4.3-SNAPSHOT + 1.4.3 ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index 2ddd75cc..69c9d527 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.3-SNAPSHOT + 1.4.3 ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index 1ecbea82..7a6e76f4 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.3-SNAPSHOT + 1.4.3 ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index c012d147..ad5b020b 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.3-SNAPSHOT + 1.4.3 ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index f3fc3090..9277e1f8 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.3-SNAPSHOT + 1.4.3 ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 5a8b5886..904178d8 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.3-SNAPSHOT + 1.4.3 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index b91ab27c..d82def68 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.3-SNAPSHOT + 1.4.3 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index ff2bb46f..3bb0353c 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.3-SNAPSHOT + 1.4.3 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index f8ad6720..e06845d5 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.3-SNAPSHOT + 1.4.3 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml index 32995921..9f9a298a 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.3-SNAPSHOT + 1.4.3 ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index 9fb640ae..efe7d226 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3-SNAPSHOT + 1.4.3 ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 70d14cd5..e9f33862 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.4.3-SNAPSHOT + 1.4.3 pom Semantic Kernel Java BOM @@ -290,6 +290,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - HEAD + java-1.4.3 diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 5ee83333..ce5f8055 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3-SNAPSHOT + 1.4.3 semantickernel-experimental From f5986bdd5595bbb1de298081ad4cd9f4281803a2 Mon Sep 17 00:00:00 2001 From: GitHub Date: Tue, 25 Feb 2025 17:19:21 -0800 Subject: [PATCH 117/277] [maven-release-plugin] prepare for next development iteration --- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- data/semantickernel-data-azureaisearch/pom.xml | 2 +- data/semantickernel-data-jdbc/pom.xml | 2 +- data/semantickernel-data-redis/pom.xml | 2 +- pom.xml | 4 ++-- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- .../semantickernel-text-splitter-plugin/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-bom/pom.xml | 4 ++-- semantickernel-experimental/pom.xml | 2 +- 24 files changed, 26 insertions(+), 26 deletions(-) diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index 8c6ac73d..97c6957c 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3 + 1.4.4-SNAPSHOT ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index 8ca1d183..68803583 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3 + 1.4.4-SNAPSHOT ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index 34c1477d..56ad5183 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3 + 1.4.4-SNAPSHOT ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 64c122e9..49624531 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.4.3 + 1.4.4-SNAPSHOT ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index 674f733b..e5dde576 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3 + 1.4.4-SNAPSHOT ../pom.xml diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml index c53436bb..24cf4174 100644 --- a/data/semantickernel-data-azureaisearch/pom.xml +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3 + 1.4.4-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 0379f981..624779e9 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3 + 1.4.4-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-redis/pom.xml b/data/semantickernel-data-redis/pom.xml index e2c18c2d..69e7c293 100644 --- a/data/semantickernel-data-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3 + 1.4.4-SNAPSHOT ../../pom.xml diff --git a/pom.xml b/pom.xml index ab526a96..30963f54 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3 + 1.4.4-SNAPSHOT pom https://www.github.com/microsoft/semantic-kernel @@ -837,6 +837,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.4.3 + HEAD diff --git a/samples/pom.xml b/samples/pom.xml index d78bc0dd..d42af3c9 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3 + 1.4.4-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index 70f831df..d94740e8 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.3 + 1.4.4-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index a3775208..1f6bc181 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.4.3 + 1.4.4-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index 69c9d527..d7a6ec3b 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.3 + 1.4.4-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index 7a6e76f4..eb3c0afe 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.3 + 1.4.4-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index ad5b020b..fd9822eb 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.3 + 1.4.4-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index 9277e1f8..799d563b 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.3 + 1.4.4-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 904178d8..293da2d3 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.3 + 1.4.4-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index d82def68..bb8db139 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.3 + 1.4.4-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index 3bb0353c..fa9f378d 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.3 + 1.4.4-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index e06845d5..8c012a6e 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.3 + 1.4.4-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml index 9f9a298a..a10f1cfc 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.3 + 1.4.4-SNAPSHOT ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index efe7d226..3176c82e 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3 + 1.4.4-SNAPSHOT ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index e9f33862..bb4d766f 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.4.3 + 1.4.4-SNAPSHOT pom Semantic Kernel Java BOM @@ -290,6 +290,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.4.3 + HEAD diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index ce5f8055..1b3aa784 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.3 + 1.4.4-SNAPSHOT semantickernel-experimental From 7501d23e1bd2de141d68887385310c03541504ba Mon Sep 17 00:00:00 2001 From: Phil Jirsa Date: Thu, 27 Feb 2025 08:43:00 -0600 Subject: [PATCH 118/277] update opentelemetry span attributes --- .../implementation/telemetry/ChatCompletionSpan.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/ChatCompletionSpan.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/ChatCompletionSpan.java index 87945860..9fa465ed 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/ChatCompletionSpan.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/ChatCompletionSpan.java @@ -104,8 +104,8 @@ public void endSpanWithUsage(ChatCompletions chatCompletions) { CompletionsUsage usage = chatCompletions.getUsage(); getSpan().setStatus(StatusCode.OK); getSpan() - .setAttribute("gen_ai.response.completion_tokens", usage.getCompletionTokens()); - getSpan().setAttribute("gen_ai.response.prompt_tokens", usage.getPromptTokens()); + .setAttribute("gen_ai.usage.output_tokens", usage.getCompletionTokens()); + getSpan().setAttribute("gen_ai.usage.input_tokens", usage.getPromptTokens()); close(); } From 016aa2220f61610a713c98fe88dd4f0c24132273 Mon Sep 17 00:00:00 2001 From: Phil Jirsa Date: Sun, 2 Mar 2025 07:48:33 -0600 Subject: [PATCH 119/277] update otel test --- .../semantickernel/aiservices/openai/OtelCaptureTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/OtelCaptureTest.java b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/OtelCaptureTest.java index c9136c55..5c3444b3 100644 --- a/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/OtelCaptureTest.java +++ b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/OtelCaptureTest.java @@ -118,9 +118,9 @@ public void otelChatCaptureTest() { spans.get(0).getAttributes().get(AttributeKey.stringKey("gen_ai.system"))); Assertions.assertEquals(21, spans.get(0).getAttributes() - .get(AttributeKey.longKey("gen_ai.response.completion_tokens"))); + .get(AttributeKey.longKey("gen_ai.usage.output_tokens"))); Assertions.assertEquals(42, spans.get(0).getAttributes() - .get(AttributeKey.longKey("gen_ai.response.prompt_tokens"))); + .get(AttributeKey.longKey("gen_ai.usage.input_tokens"))); } } From f01c624981f060ea8238183101415777a4c4cb94 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Mon, 14 Apr 2025 22:19:12 -0700 Subject: [PATCH 120/277] Add KernelArguments and deprecated KernelFunctionArguments --- .../chatcompletion/GeminiChatCompletion.java | 4 +- .../chatcompletion/OpenAIChatCompletion.java | 8 +- .../OpenAIFunctionToolCall.java | 8 +- .../OpenAiChatCompletionTest.java | 4 +- .../tests/Example03_ArgumentsTest.java | 4 +- ...xample05_InlineFunctionDefinitionTest.java | 8 +- .../semantickernel/tests/KernelHooksTest.java | 4 +- .../semantickernel/tests/RenderingTest.java | 16 +- .../plugins/ConversationSummaryPlugin.java | 4 +- .../Example11_WebSearchQueries.java | 4 +- .../syntaxexamples/Example20_HuggingFace.java | 4 +- .../Example43_GetModelResult.java | 4 +- .../syntaxexamples/Example57_KernelHooks.java | 16 +- .../Example62_CustomAIServiceSelector.java | 6 +- .../Example30_ChatWithPrompts.java | 4 +- .../functions/Example03_Arguments.java | 4 +- .../Example05_InlineFunctionDefinition.java | 8 +- .../functions/Example09_FunctionTypes.java | 17 +- ...Example27_PromptFunctionsUsingChatGPT.java | 4 +- .../Example60_AdvancedMethodFunctions.java | 6 +- .../Example98_GeminiFunctionCalling.java | 4 +- .../java/CustomTypes_Example.java | 8 +- .../java/FunctionTelemetry_Example.java | 4 +- .../java/FunctionsHandlebars_Example.java | 4 +- .../java/FunctionsWithinPrompts_Example.java | 6 +- .../java/KernelFunctionYaml_Example.java | 6 +- .../Example07_BingAndGooglePlugins.java | 8 +- .../Example13_ConversationSummaryPlugin.java | 10 +- ...eMethodFunctionsWithMultipleArguments.java | 4 +- .../Example64_MultiplePromptTemplates.java | 4 +- .../com/microsoft/semantickernel/Main.java | 4 +- .../CreatingFunctions.java | 4 +- .../FunctionsWithinPrompts.java | 6 +- .../SerializingPrompts.java | 6 +- .../documentationexamples/Templates.java | 6 +- .../documentationexamples/UsingTheKernel.java | 6 +- .../plugins/ConversationSummaryPlugin.java | 4 +- .../openapi/OpenAPIHttpRequestPlugin.java | 14 +- .../SemanticKernelOpenAPIImporter.java | 4 +- .../com/microsoft/semantickernel/Kernel.java | 10 +- .../hooks/FunctionInvokedEvent.java | 10 +- .../hooks/FunctionInvokingEvent.java | 10 +- .../hooks/PreToolCallEvent.java | 8 +- .../hooks/PromptRenderedEvent.java | 10 +- .../hooks/PromptRenderingEvent.java | 10 +- .../telemetry/FunctionSpan.java | 4 +- .../tokenizer/DefaultPromptTemplate.java | 4 +- .../tokenizer/blocks/CodeBlock.java | 12 +- .../tokenizer/blocks/CodeRendering.java | 4 +- .../tokenizer/blocks/FunctionIdBlock.java | 4 +- .../tokenizer/blocks/NamedArgBlock.java | 6 +- .../tokenizer/blocks/TextBlock.java | 4 +- .../tokenizer/blocks/TextRendering.java | 4 +- .../tokenizer/blocks/ValBlock.java | 4 +- .../tokenizer/blocks/VarBlock.java | 4 +- .../orchestration/FunctionInvocation.java | 12 +- .../semanticfunctions/KernelArguments.java | 410 ++++++++++++++++++ .../semanticfunctions/KernelFunction.java | 4 +- .../KernelFunctionArguments.java | 142 +----- .../KernelFunctionFromMethod.java | 22 +- .../KernelFunctionFromPrompt.java | 8 +- .../semanticfunctions/PromptTemplate.java | 2 +- .../services/AIServiceSelector.java | 7 +- .../services/BaseAIServiceSelector.java | 10 +- .../services/OrderedAIServiceSelector.java | 88 ++-- .../handlebars/HandlebarsPromptTemplate.java | 24 +- .../KernelFunctionFromMethodTest.java | 32 +- .../PromptTemplateFactoryTest.java | 2 +- .../services/AIServiceSelectorTest.java | 4 +- .../HandlebarsPromptTemplateTest.java | 14 +- .../semantickernel/CodeTokenizerTest.java | 11 +- 71 files changed, 739 insertions(+), 410 deletions(-) create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java diff --git a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java index e226b4c6..2bd45c02 100644 --- a/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java +++ b/aiservices/google/src/main/java/com/microsoft/semantickernel/aiservices/google/chatcompletion/GeminiChatCompletion.java @@ -32,7 +32,7 @@ import com.microsoft.semantickernel.plugin.KernelPlugin; import com.microsoft.semantickernel.semanticfunctions.InputVariable; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.services.chatcompletion.AuthorRole; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; import com.microsoft.semantickernel.services.chatcompletion.ChatHistory; @@ -436,7 +436,7 @@ public Mono performFunctionCall(@Nullable Kernel kernel, ? new ContextVariableTypes() : invocationContext.getContextVariableTypes(); - KernelFunctionArguments.Builder arguments = KernelFunctionArguments.builder(); + KernelArguments.Builder arguments = KernelArguments.builder(); geminiFunction.getFunctionCall().getArgs().getFieldsMap().forEach((key, value) -> { arguments.withVariable(key, value.getStringValue()); }); diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java index b842e43d..33f46fc2 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java @@ -61,7 +61,7 @@ import com.microsoft.semantickernel.orchestration.responseformat.JsonResponseSchema; import com.microsoft.semantickernel.orchestration.responseformat.JsonSchemaResponseFormat; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.services.chatcompletion.AuthorRole; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; import com.microsoft.semantickernel.services.chatcompletion.ChatHistory; @@ -629,7 +629,7 @@ private Mono> invokeFunctionTool( contextVariableTypes)); function = hookResult.getFunction(); - KernelFunctionArguments arguments = hookResult.getArguments(); + KernelArguments arguments = hookResult.getArguments(); return function .invokeAsync(kernel) @@ -673,7 +673,7 @@ private OpenAIFunctionToolCall extractOpenAIFunctionToolCall( String pluginName = parts.length > 1 ? parts[0] : ""; String fnName = parts.length > 1 ? parts[1] : parts[0]; - KernelFunctionArguments arguments = KernelFunctionArguments.builder().build(); + KernelArguments arguments = KernelArguments.builder().build(); ObjectMapper mapper = new ObjectMapper(); JsonNode jsonToolCallArguments = mapper.readTree(toolCall.getFunction().getArguments()); @@ -1144,7 +1144,7 @@ private static ChatRequestAssistantMessage formAssistantMessage( asstMessage.setToolCalls( toolCalls.stream() .map(toolCall -> { - KernelFunctionArguments arguments = toolCall.getArguments(); + KernelArguments arguments = toolCall.getArguments(); String args = arguments != null && !arguments.isEmpty() ? arguments.entrySet().stream() diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunctionToolCall.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunctionToolCall.java index 18111330..b7999cf0 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunctionToolCall.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunctionToolCall.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.aiservices.openai.chatcompletion; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import javax.annotation.Nullable; /** @@ -23,7 +23,7 @@ public class OpenAIFunctionToolCall { ///

Gets a name/value collection of the arguments to the function, if any. @Nullable - private final KernelFunctionArguments arguments; + private final KernelArguments arguments; /** * Creates a new instance of the {@link OpenAIFunctionToolCall} class. @@ -37,7 +37,7 @@ public OpenAIFunctionToolCall( @Nullable String id, @Nullable String pluginName, String functionName, - @Nullable KernelFunctionArguments arguments) { + @Nullable KernelArguments arguments) { this.id = id; this.pluginName = pluginName; this.functionName = functionName; @@ -83,7 +83,7 @@ public String getFunctionName() { * @return A name/value collection of the arguments to the function, if any. */ @Nullable - public KernelFunctionArguments getArguments() { + public KernelArguments getArguments() { if (arguments == null) { return null; } diff --git a/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAiChatCompletionTest.java b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAiChatCompletionTest.java index 05aebc3d..5a1be82a 100644 --- a/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAiChatCompletionTest.java +++ b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAiChatCompletionTest.java @@ -13,7 +13,7 @@ import com.azure.json.implementation.DefaultJsonReader; import com.microsoft.semantickernel.implementation.EmbeddedResourceLoader; import com.microsoft.semantickernel.orchestration.FunctionResultMetadata; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.services.chatcompletion.AuthorRole; import com.microsoft.semantickernel.services.chatcompletion.ChatHistory; import java.nio.charset.Charset; @@ -46,7 +46,7 @@ public void serializesToolCallsCorrectly() { "a-tool-id", "pluginName", "funcName", - KernelFunctionArguments.builder() + KernelArguments.builder() .withVariable("id", "ca2fc6bc-1307-4da6-a009-d7bf88dec37b") .build())))); chatHistory.addMessage(new OpenAIChatMessageContent( diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/Example03_ArgumentsTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/Example03_ArgumentsTest.java index ac182124..dd283952 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/Example03_ArgumentsTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/Example03_ArgumentsTest.java @@ -6,7 +6,7 @@ import com.microsoft.semantickernel.plugin.KernelPlugin; import com.microsoft.semantickernel.plugin.KernelPluginFactory; import com.microsoft.semantickernel.samples.syntaxexamples.functions.Example03_Arguments.StaticTextPlugin; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -24,7 +24,7 @@ public void main() { KernelPlugin functionCollection = KernelPluginFactory .createFromObject(new StaticTextPlugin(), "text"); - KernelFunctionArguments arguments = KernelFunctionArguments.builder() + KernelArguments arguments = KernelArguments.builder() .withInput("Today is: ") .withVariable("day", "Monday") .build(); diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/Example05_InlineFunctionDefinitionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/Example05_InlineFunctionDefinitionTest.java index f74268b2..d791ec9c 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/Example05_InlineFunctionDefinitionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/Example05_InlineFunctionDefinitionTest.java @@ -10,9 +10,9 @@ import com.microsoft.semantickernel.orchestration.FunctionResult; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionFromPrompt; -import com.microsoft.semantickernel.services.textcompletion.TextGenerationService; + import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; @@ -68,7 +68,7 @@ public void main(WireMockRuntimeInfo wmRuntimeInfo) { var result = kernel.invokeAsync(excuseFunction) .withArguments( - KernelFunctionArguments.builder() + KernelArguments.builder() .withInput("I missed the F1 final race") .build()) .block(); @@ -79,7 +79,7 @@ public void main(WireMockRuntimeInfo wmRuntimeInfo) { result = kernel.invokeAsync(excuseFunction) .withArguments( - KernelFunctionArguments.builder() + KernelArguments.builder() .withInput("sorry I forgot your birthday") .build()) .block(); diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/KernelHooksTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/KernelHooksTest.java index 16b9cada..4a128fb1 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/KernelHooksTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/KernelHooksTest.java @@ -12,7 +12,7 @@ import com.microsoft.semantickernel.hooks.KernelHook.FunctionInvokingHook; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionFromPrompt; import com.microsoft.semantickernel.semanticfunctions.OutputVariable; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; @@ -90,7 +90,7 @@ public void getUsageAsync(WireMockRuntimeInfo wmRuntimeInfo) { kernel.invokeAsync( excuseFunction) .withArguments( - KernelFunctionArguments + KernelArguments .builder() .withVariable("input", "I missed the F1 final race") .build()) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/RenderingTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/RenderingTest.java index 3f09b50b..ff4655d2 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/RenderingTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/RenderingTest.java @@ -12,7 +12,7 @@ import com.microsoft.semantickernel.aiservices.openai.textcompletion.OpenAITextGenerationService; import com.microsoft.semantickernel.plugin.KernelPluginFactory; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.PromptTemplateConfig; import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; @@ -49,7 +49,7 @@ public void textSemanticKernelTemplateXml() { """) .withTemplateFormat(PromptTemplateConfig.SEMANTIC_KERNEL_TEMPLATE_FORMAT) .build()) - .withArguments(KernelFunctionArguments + .withArguments(KernelArguments .builder() .withVariable("value", "\"hello world\"") .build()) @@ -71,7 +71,7 @@ public void textSemanticKernelTemplate() { """) .withTemplateFormat(PromptTemplateConfig.SEMANTIC_KERNEL_TEMPLATE_FORMAT) .build()) - .withArguments(KernelFunctionArguments + .withArguments(KernelArguments .builder() .withVariable("value", "{{$ignore}}") .withVariable("ignore", "dont show") @@ -94,7 +94,7 @@ public void textHandleBarsTemplate() { """) .withTemplateFormat("handlebars") .build()) - .withArguments(KernelFunctionArguments + .withArguments(KernelArguments .builder() .withVariable("value", "{{ignore}}") .withVariable("ignore", "dont show") @@ -117,7 +117,7 @@ public void chatSemanticKernelTemplateXml() { """) .withTemplateFormat(PromptTemplateConfig.SEMANTIC_KERNEL_TEMPLATE_FORMAT) .build()) - .withArguments(KernelFunctionArguments + .withArguments(KernelArguments .builder() .withVariable("value", "\"hello world\"") .build()) @@ -139,7 +139,7 @@ public void chatSemanticKernelTemplate() { """) .withTemplateFormat(PromptTemplateConfig.SEMANTIC_KERNEL_TEMPLATE_FORMAT) .build()) - .withArguments(KernelFunctionArguments + .withArguments(KernelArguments .builder() .withVariable("value", "{{$ignore}}") .withVariable("ignore", "dont show") @@ -162,7 +162,7 @@ public void chatHandleBarsTemplate() { """) .withTemplateFormat("handlebars") .build()) - .withArguments(KernelFunctionArguments + .withArguments(KernelArguments .builder() .withVariable("value", "{{ignore}}") .withVariable("ignore", "dont show") @@ -185,7 +185,7 @@ public void chatSemanticKernelTemplate2() { """) .withTemplateFormat(PromptTemplateConfig.SEMANTIC_KERNEL_TEMPLATE_FORMAT) .build()) - .withArguments(KernelFunctionArguments + .withArguments(KernelArguments .builder() .withVariable("value", "{{$ignore}}") .withVariable("ignore", "dont show") diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/ConversationSummaryPlugin.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/ConversationSummaryPlugin.java index 089a4941..f61d1d87 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/ConversationSummaryPlugin.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/ConversationSummaryPlugin.java @@ -5,7 +5,7 @@ import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; import com.microsoft.semantickernel.semanticfunctions.annotations.KernelFunctionParameter; import com.microsoft.semantickernel.text.TextChunker; @@ -71,7 +71,7 @@ private static Mono processAsync(KernelFunction func, String inp // The first parameter is the input text. return func.invokeAsync(kernel) .withArguments( - new KernelFunctionArguments.Builder() + KernelArguments.builder() .withInput(paragraph) .build()) .withResultType( diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example11_WebSearchQueries.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example11_WebSearchQueries.java index 63abdbcc..1386cc61 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example11_WebSearchQueries.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example11_WebSearchQueries.java @@ -4,7 +4,7 @@ import com.microsoft.semantickernel.Kernel; import com.microsoft.semantickernel.plugin.KernelPluginFactory; import com.microsoft.semantickernel.samples.plugins.web.SearchUrlPlugin; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; public class Example11_WebSearchQueries { @@ -19,7 +19,7 @@ public static void main(String[] args) { // Run var ask = "What's the largest building in Europe?"; - var kernelArguments = KernelFunctionArguments.builder() + var kernelArguments = KernelArguments.builder() .withVariable("query", ask) .build(); diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example20_HuggingFace.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example20_HuggingFace.java index a388c130..14fccf0a 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example20_HuggingFace.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example20_HuggingFace.java @@ -5,7 +5,7 @@ import com.microsoft.semantickernel.Kernel; import com.microsoft.semantickernel.aiservices.huggingface.HuggingFaceClient; import com.microsoft.semantickernel.aiservices.huggingface.services.HuggingFaceTextGenerationService; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionFromPrompt; import com.microsoft.semantickernel.services.textcompletion.TextGenerationService; @@ -43,7 +43,7 @@ public static void runInferenceApiExampleAsync() { var result = kernel.invokeAsync(questionAnswerFunction) .withArguments( - KernelFunctionArguments.builder() + KernelArguments.builder() .withVariable("input", "What is New York?") .build()) .withResultType(String.class) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example43_GetModelResult.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example43_GetModelResult.java index 9dc86a68..7b1e5752 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example43_GetModelResult.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example43_GetModelResult.java @@ -10,7 +10,7 @@ import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIChatCompletion; import com.microsoft.semantickernel.orchestration.FunctionResult; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionFromPrompt; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; @@ -61,7 +61,7 @@ public static void main(String[] args) { FunctionResult result = kernel.invokeAsync( myFunction) .withArguments( - KernelFunctionArguments.builder() + KernelArguments.builder() .withVariable("input", "travel") .build()) .block(); diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example57_KernelHooks.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example57_KernelHooks.java index 0203f66c..ffa09a67 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example57_KernelHooks.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example57_KernelHooks.java @@ -24,7 +24,7 @@ import com.microsoft.semantickernel.hooks.PromptRenderedEvent; import com.microsoft.semantickernel.orchestration.FunctionResult; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionFromPrompt; import com.microsoft.semantickernel.semanticfunctions.OutputVariable; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; @@ -133,7 +133,7 @@ private static void getUsageAsync(Kernel kernel) { String input = "I missed the F1 final race"; var result = kernel.invokeAsync(excuseFunction) .withArguments( - KernelFunctionArguments + KernelArguments .builder() .withVariable("input", input) .build()) @@ -189,7 +189,7 @@ private static void getRenderedPromptAsync(Kernel kernel) { String input = "I missed the F1 final race"; var result = kernel.invokeAsync(excuseFunction) .withArguments( - KernelFunctionArguments + KernelArguments .builder() .withVariable("input", input) .build()) @@ -235,7 +235,7 @@ private static void changingResultAsync(Kernel kernel) { // Invoke prompt to trigger execution hooks. var result = kernel.invokeAsync(writerFunction) .withArguments( - KernelFunctionArguments.builder().build()) + KernelArguments.builder().build()) .block(); System.out.println("Function Result: " + result.getResult()); } @@ -275,7 +275,7 @@ private static void beforeInvokeCancellationAsync(Kernel kernel) { // Invoke prompt to trigger execution hooks. var result = kernel.invokeAsync(writerFunction) .withArguments( - KernelFunctionArguments.builder().build()) + KernelArguments.builder().build()) .block(); System.out.println("Function Result: " + result.getResult()); } catch (Exception e) { @@ -312,7 +312,7 @@ private static void afterInvokeCancellationAsync(Kernel kernel) { // Invoke prompt to trigger execution hooks. try { var result = kernel.invokeAsync(secondFunction) - .withArguments(KernelFunctionArguments.builder().build()) + .withArguments(KernelArguments.builder().build()) .block(); System.out.println("Function Result: " + result.getResult()); } catch (Exception e) { @@ -359,7 +359,7 @@ private static void chatCompletionHook(Kernel kernel) { // Invoke prompt to trigger execution hooks. var result = kernel.invokeAsync(writerFunction) .withArguments( - KernelFunctionArguments.builder().build()) + KernelArguments.builder().build()) .block(); System.out.println("Function Result: " + result.getResult()); } catch (Exception e) { @@ -403,7 +403,7 @@ private static void invocationHook(Kernel kernel) { try { // Invoke prompt to trigger execution hooks. var result = kernel.invokeAsync(writerFunction) - .withArguments(KernelFunctionArguments.builder().build()) + .withArguments(KernelArguments.builder().build()) .addKernelHooks(kernelHooks) .block(); System.out.println("Function Result: " + result.getResult()); diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example62_CustomAIServiceSelector.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example62_CustomAIServiceSelector.java index afa5b2e3..5e01d13a 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example62_CustomAIServiceSelector.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example62_CustomAIServiceSelector.java @@ -9,7 +9,7 @@ import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIChatCompletion; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionFromPrompt; import com.microsoft.semantickernel.services.AIService; import com.microsoft.semantickernel.services.AIServiceCollection; @@ -67,7 +67,7 @@ public static void main(String[] args) { var prompt = "Hello AI, what can you do for me?"; - KernelFunctionArguments arguments = KernelFunctionArguments.builder().build(); + KernelArguments arguments = KernelArguments.builder().build(); KernelFunction func = KernelFunctionFromPrompt .builder() @@ -98,7 +98,7 @@ public AIServiceSelection trySelectAIService( @Nullable KernelFunction function, - @Nullable KernelFunctionArguments arguments, + @Nullable KernelArguments arguments, Map, AIService> services) { // Just get the first one diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/Example30_ChatWithPrompts.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/Example30_ChatWithPrompts.java index 1357cc47..04a7a34f 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/Example30_ChatWithPrompts.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/Example30_ChatWithPrompts.java @@ -11,7 +11,7 @@ import com.microsoft.semantickernel.plugin.KernelPlugin; import com.microsoft.semantickernel.plugin.KernelPluginFactory; import com.microsoft.semantickernel.samples.plugins.TimePlugin; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.PromptTemplateConfig; import com.microsoft.semantickernel.semanticfunctions.PromptTemplateFactory; import com.microsoft.semantickernel.services.ServiceNotFoundException; @@ -82,7 +82,7 @@ public static void main(String[] args) throws FileNotFoundException, ServiceNotF // Adding required arguments referenced by the prompt templates. - var arguments = KernelFunctionArguments + var arguments = KernelArguments .builder() .withVariable("selectedText", selectedText) .withVariable("startTime", DateTimeFormatter.ofPattern("hh:mm:ss a zz").format( diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example03_Arguments.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example03_Arguments.java index 2096386d..7db37899 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example03_Arguments.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example03_Arguments.java @@ -5,7 +5,7 @@ import com.microsoft.semantickernel.orchestration.FunctionResult; import com.microsoft.semantickernel.plugin.KernelPlugin; import com.microsoft.semantickernel.plugin.KernelPluginFactory; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; import com.microsoft.semantickernel.semanticfunctions.annotations.KernelFunctionParameter; import java.util.Locale; @@ -24,7 +24,7 @@ public static void main(String[] args) { KernelPlugin functionCollection = KernelPluginFactory .createFromObject(new StaticTextPlugin(), "text"); - KernelFunctionArguments arguments = KernelFunctionArguments.builder() + KernelArguments arguments = KernelArguments.builder() .withInput("Today is: ") .withVariable("day", "Monday") .build(); diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example05_InlineFunctionDefinition.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example05_InlineFunctionDefinition.java index 7ecf9a0d..52edb0db 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example05_InlineFunctionDefinition.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example05_InlineFunctionDefinition.java @@ -11,10 +11,10 @@ import com.microsoft.semantickernel.orchestration.FunctionResult; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionFromPrompt; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; -import com.microsoft.semantickernel.services.textcompletion.TextGenerationService; + import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; @@ -82,7 +82,7 @@ public static void main(String[] args) throws ConfigurationException { var result = kernel .invokeAsync(excuseFunction) .withArguments( - KernelFunctionArguments.builder() + KernelArguments.builder() .withInput("I missed the F1 final race") .build()) .block(); @@ -90,7 +90,7 @@ public static void main(String[] args) throws ConfigurationException { result = kernel.invokeAsync(excuseFunction) .withArguments( - KernelFunctionArguments.builder() + KernelArguments.builder() .withInput("sorry I forgot your birthday") .build()) .block(); diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example09_FunctionTypes.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example09_FunctionTypes.java index c6c1fdff..4c14ef54 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example09_FunctionTypes.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example09_FunctionTypes.java @@ -9,7 +9,6 @@ import com.azure.core.credential.KeyCredential; import com.microsoft.semantickernel.Kernel; import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIChatCompletion; -import com.microsoft.semantickernel.aiservices.openai.textcompletion.OpenAITextGenerationService; import com.microsoft.semantickernel.contextvariables.ContextVariable; import com.microsoft.semantickernel.contextvariables.ContextVariableType; import com.microsoft.semantickernel.contextvariables.ContextVariableTypeConverter; @@ -17,11 +16,11 @@ import com.microsoft.semantickernel.orchestration.FunctionResult; import com.microsoft.semantickernel.plugin.KernelPlugin; import com.microsoft.semantickernel.plugin.KernelPluginFactory; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; import com.microsoft.semantickernel.semanticfunctions.annotations.KernelFunctionParameter; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; -import com.microsoft.semantickernel.services.textcompletion.TextGenerationService; + import java.nio.file.Path; import java.time.Instant; import java.time.OffsetDateTime; @@ -119,7 +118,7 @@ public static void main(String[] args) throws InterruptedException { result = kernel .invokeAsync(plugin.get("InputDateTimeWithStringResult")) .withArguments( - KernelFunctionArguments + KernelArguments .builder() .withVariable("currentDate", ContextVariable.of( @@ -135,7 +134,7 @@ public static void main(String[] args) throws InterruptedException { result = kernel.invokeAsync(plugin.get("MultipleInputsWithVoidResult")) .withArguments( - KernelFunctionArguments + KernelArguments .builder() .withVariable("x", "x string") .withVariable("y", 100) @@ -146,7 +145,7 @@ public static void main(String[] args) throws InterruptedException { result = kernel .invokeAsync(plugin.get("ComplexInputWithStringResult")) .withArguments( - KernelFunctionArguments + KernelArguments .builder() .withVariable( "complexObject", @@ -165,7 +164,7 @@ public String toString() { result = kernel .invokeAsync(plugin.get("InputStringTaskWithStringResult")) .withArguments( - KernelFunctionArguments + KernelArguments .builder() .withVariable("echoInput", "return this") .build()) @@ -175,7 +174,7 @@ public String toString() { result = kernel .invokeAsync(plugin.get("InputStringTaskWithVoidResult")) .withArguments( - KernelFunctionArguments + KernelArguments .builder() .withVariable("x", "x input") .build()) @@ -258,7 +257,7 @@ public String toString() { result = kernel.invokeAsync(plugin.get("MultipleComplexInputsWithVoidResult")) .withArguments( - KernelFunctionArguments + KernelArguments .builder() .withVariable("x", OffsetDateTime.of(1, 1, 1, 1, 1, 1, 1, ZoneOffset.UTC)) .withVariable("y", OffsetDateTime.of(1, 1, 1, 1, 1, 1, 1, ZoneOffset.UTC)) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example27_PromptFunctionsUsingChatGPT.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example27_PromptFunctionsUsingChatGPT.java index f72ec237..1f139e2f 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example27_PromptFunctionsUsingChatGPT.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example27_PromptFunctionsUsingChatGPT.java @@ -9,7 +9,7 @@ import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIChatCompletion; import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; public class Example27_PromptFunctionsUsingChatGPT { @@ -54,7 +54,7 @@ public static void main(String[] args) { var result = func.invokeAsync(kernel) .withArguments( - KernelFunctionArguments.builder() + KernelArguments.builder() .withVariable("input", "Jupiter") .build()) .withResultType(ContextVariableTypes.getGlobalVariableTypeForClass(String.class)) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example60_AdvancedMethodFunctions.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example60_AdvancedMethodFunctions.java index 87669a40..dfbaeb72 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example60_AdvancedMethodFunctions.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example60_AdvancedMethodFunctions.java @@ -5,7 +5,7 @@ import com.microsoft.semantickernel.contextvariables.ContextVariableTypeConverter; import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; import com.microsoft.semantickernel.plugin.KernelPluginFactory; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; import reactor.core.publisher.Mono; @@ -37,7 +37,7 @@ public static void main(String[] args) { var result = kernel .invokeAsync(FunctionsChainingPlugin.PluginName, "Function1") .withArguments( - KernelFunctionArguments + KernelArguments .builder() .build()) .withResultType(ContextVariableTypes.getGlobalVariableTypeForClass(MyCustomType.class)) @@ -82,7 +82,7 @@ public Mono function1Async(Kernel kernel) { // Execute another function return kernel .invokeAsync(PluginName, "Function2") - .withArguments(KernelFunctionArguments.builder().build()) + .withArguments(KernelArguments.builder().build()) .withResultType(ContextVariableTypes.getGlobalVariableTypeForClass( Example60_AdvancedMethodFunctions.MyCustomType.class)) .flatMap(value -> { diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example98_GeminiFunctionCalling.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example98_GeminiFunctionCalling.java index 0dd425f6..b585775d 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example98_GeminiFunctionCalling.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example98_GeminiFunctionCalling.java @@ -13,7 +13,7 @@ import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; import com.microsoft.semantickernel.orchestration.ToolCallBehavior; import com.microsoft.semantickernel.plugin.KernelPluginFactory; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionFromPrompt; import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; import com.microsoft.semantickernel.semanticfunctions.annotations.KernelFunctionParameter; @@ -140,7 +140,7 @@ public static void main(String[] args) throws NoSuchMethodException { var fn = kernel.getFunction(geminiFunction.getPluginName(), geminiFunction.getFunctionName()); - var arguments = KernelFunctionArguments.builder(); + var arguments = KernelArguments.builder(); geminiFunction.getFunctionCall().getArgs().getFieldsMap() .forEach((key, value) -> { arguments.withVariable(key, value.getStringValue()); diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/CustomTypes_Example.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/CustomTypes_Example.java index e809474d..fcf2cc81 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/CustomTypes_Example.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/CustomTypes_Example.java @@ -13,7 +13,7 @@ import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; import com.microsoft.semantickernel.contextvariables.converters.ContextVariableJacksonConverter; import com.microsoft.semantickernel.exceptions.ConfigurationException; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; import java.io.IOException; import java.util.Arrays; @@ -114,7 +114,7 @@ private static void exampleBuildingCustomConverter( Pet updated = kernel.invokePromptAsync( "Change Sandy's name to Daisy:\n{{$Sandy}}", - KernelFunctionArguments.builder() + KernelArguments.builder() .withVariable("Sandy", sandy, typeConverter) .build()) .withTypeConverter(typeConverter) @@ -139,7 +139,7 @@ public static void exampleUsingJackson(ChatCompletionService chatCompletionServi // Invoke the prompt with the custom converter Pet updated = kernel.invokePromptAsync( "Increase Sandy's age by a year:\n{{$Sandy}}", - KernelFunctionArguments.builder() + KernelArguments.builder() .withVariable("Sandy", sandy, typeConverter) .build()) .withTypeConverter(typeConverter) @@ -167,7 +167,7 @@ public static void exampleUsingGlobalTypes(ChatCompletionService chatCompletionS // No need to explicitly tell the invocation how to convert the type Pet updated = kernel.invokePromptAsync( "Sandy's is actually a cat correct this:\n{{$Sandy}}", - KernelFunctionArguments.builder() + KernelArguments.builder() .withVariable("Sandy", sandy) .build()) .withResultType(Pet.class) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/FunctionTelemetry_Example.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/FunctionTelemetry_Example.java index 1d228250..4ca35a80 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/FunctionTelemetry_Example.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/FunctionTelemetry_Example.java @@ -14,7 +14,7 @@ import com.microsoft.semantickernel.orchestration.ToolCallBehavior; import com.microsoft.semantickernel.plugin.KernelPluginFactory; import com.microsoft.semantickernel.samples.syntaxexamples.functions.Example59_OpenAIFunctionCalling.PetPlugin; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; import com.microsoft.semantickernel.semanticfunctions.annotations.KernelFunctionParameter; import com.microsoft.semantickernel.services.ServiceNotFoundException; @@ -204,7 +204,7 @@ public static void testNestedCalls() { Analyse the following text: Hello There """, - KernelFunctionArguments.builder().build(), + KernelArguments.builder().build(), InvocationContext.builder() .withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(true)) .withReturnMode(InvocationReturnMode.NEW_MESSAGES_ONLY) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/FunctionsHandlebars_Example.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/FunctionsHandlebars_Example.java index b13f0699..29ad3769 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/FunctionsHandlebars_Example.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/FunctionsHandlebars_Example.java @@ -4,7 +4,7 @@ import com.microsoft.semantickernel.Kernel; import com.microsoft.semantickernel.exceptions.ConfigurationException; import com.microsoft.semantickernel.plugin.KernelPluginFactory; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.KernelPromptTemplateFactory; import com.microsoft.semantickernel.semanticfunctions.PromptTemplateConfig; import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; @@ -41,7 +41,7 @@ public static void main(String[] args) throws ConfigurationException, IOExceptio var renderedPrompt = promptTemplate.renderAsync( kernel, - KernelFunctionArguments.builder() + KernelArguments.builder() .withVariable("choices", choices) .build(), null) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/FunctionsWithinPrompts_Example.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/FunctionsWithinPrompts_Example.java index ceb9983e..7e92e3e6 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/FunctionsWithinPrompts_Example.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/FunctionsWithinPrompts_Example.java @@ -12,7 +12,7 @@ import com.microsoft.semantickernel.plugin.KernelPlugin; import com.microsoft.semantickernel.plugin.KernelPluginFactory; import com.microsoft.semantickernel.samples.plugins.ConversationSummaryPlugin; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionFromPrompt; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; import com.microsoft.semantickernel.services.chatcompletion.ChatHistory; @@ -154,7 +154,7 @@ public static void main(String[] args) throws ConfigurationException, IOExceptio // Invoke handlebars prompt var intent = kernel.invokeAsync(getIntent) .withArguments( - KernelFunctionArguments.builder() + KernelArguments.builder() .withVariable("request", request) .withVariable("choices", choices) .withVariable("history", historyString) @@ -173,7 +173,7 @@ public static void main(String[] args) throws ConfigurationException, IOExceptio // Get chat response var chatResult = kernel.invokeAsync(chat) .withArguments( - KernelFunctionArguments.builder() + KernelArguments.builder() .withVariable("request", request) .withVariable("history", historyString) .build()) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/KernelFunctionYaml_Example.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/KernelFunctionYaml_Example.java index ec38aae0..10e9a014 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/KernelFunctionYaml_Example.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/KernelFunctionYaml_Example.java @@ -13,7 +13,7 @@ import com.microsoft.semantickernel.implementation.telemetry.SemanticKernelTelemetry; import com.microsoft.semantickernel.orchestration.FunctionResult; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionYaml; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; import java.io.IOException; @@ -70,7 +70,7 @@ private static void handlebarsTemplate(Kernel kernel, FunctionResult result = function .invokeAsync(kernel) .withArguments( - KernelFunctionArguments.builder() + KernelArguments.builder() .withVariable("length", 5) .withVariable("topic", "dogs") .build()) @@ -91,7 +91,7 @@ private static void semanticKernelTemplate(Kernel kernel, FunctionResult result = function .invokeAsync(kernel) .withArguments( - KernelFunctionArguments.builder() + KernelArguments.builder() .withVariable("length", 5) .withVariable("topic", "cats") .build()) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugin/Example07_BingAndGooglePlugins.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugin/Example07_BingAndGooglePlugins.java index 6ca71745..2e4462fd 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugin/Example07_BingAndGooglePlugins.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugin/Example07_BingAndGooglePlugins.java @@ -13,7 +13,7 @@ import com.microsoft.semantickernel.plugin.KernelPluginFactory; import com.microsoft.semantickernel.samples.plugins.web.WebSearchEnginePlugin; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionFromPrompt; import com.microsoft.semantickernel.semanticfunctions.KernelPromptTemplateFactory; import com.microsoft.semantickernel.semanticfunctions.PromptTemplate; @@ -84,7 +84,7 @@ private static void example1Async(Kernel kernel, String searchPluginName) { // Run var question = "What's the largest building in the world?"; - var kernelArguments = KernelFunctionArguments.builder() + var kernelArguments = KernelArguments.builder() .withVariable("query", question) .build(); @@ -166,7 +166,7 @@ private static void example2Async(Kernel kernel) { .withDefaultExecutionSettings(promptExecutionSettings) .build(); - var kernelArguments = KernelFunctionArguments.builder() + var kernelArguments = KernelArguments.builder() .withVariable("question", question) .withVariable("externalInformation", "") .build(); @@ -187,7 +187,7 @@ private static void example2Async(Kernel kernel) { System.out.println("Information found:"); System.out.println(information); - kernelArguments = KernelFunctionArguments.builder() + kernelArguments = KernelArguments.builder() .withVariable("question", question) .withVariable("externalInformation", information) .build(); diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugin/Example13_ConversationSummaryPlugin.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugin/Example13_ConversationSummaryPlugin.java index b185c894..63412b60 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugin/Example13_ConversationSummaryPlugin.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugin/Example13_ConversationSummaryPlugin.java @@ -12,7 +12,9 @@ import com.microsoft.semantickernel.plugin.KernelPlugin; import com.microsoft.semantickernel.plugin.KernelPluginFactory; import com.microsoft.semantickernel.samples.plugins.ConversationSummaryPlugin; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments.Builder; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments.Builder; +import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; import reactor.core.publisher.Mono; @@ -163,7 +165,7 @@ private static void getConversationActionItemsAsync() { conversationSummaryPlugin .get("GetConversationActionItems")) .withArguments( - new Builder() + KernelArguments.builder() .withInput(chatTranscript) .build()); System.out.println("Generated Action Items:"); @@ -179,7 +181,7 @@ private static void getConversationTopicsAsync() { Mono> summary = kernel .invokeAsync(conversationSummaryPlugin.get("GetConversationTopics")) .withArguments( - new Builder() + KernelArguments.builder() .withInput(chatTranscript) .build()); @@ -201,7 +203,7 @@ private static void conversationSummaryPluginAsync() { .getFunctions() .get("SummarizeConversation")) .withArguments( - new Builder() + KernelArguments.builder() .withInput(chatTranscript) .build()) .block(); diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/template/Example56_TemplateMethodFunctionsWithMultipleArguments.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/template/Example56_TemplateMethodFunctionsWithMultipleArguments.java index dc663d2d..e26babe6 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/template/Example56_TemplateMethodFunctionsWithMultipleArguments.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/template/Example56_TemplateMethodFunctionsWithMultipleArguments.java @@ -10,7 +10,7 @@ import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; import com.microsoft.semantickernel.plugin.KernelPluginFactory; import com.microsoft.semantickernel.samples.plugins.text.TextPlugin; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionFromPrompt; import com.microsoft.semantickernel.semanticfunctions.KernelPromptTemplateFactory; import com.microsoft.semantickernel.semanticfunctions.PromptTemplateConfig; @@ -53,7 +53,7 @@ public static void main(String[] args) { System.out.println("======== TemplateMethodFunctionsWithMultipleArguments ========"); - var arguments = KernelFunctionArguments.builder() + var arguments = KernelArguments.builder() .withVariable("word2", " Potter") .build(); diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/template/Example64_MultiplePromptTemplates.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/template/Example64_MultiplePromptTemplates.java index ae534e70..75568a6e 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/template/Example64_MultiplePromptTemplates.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/template/Example64_MultiplePromptTemplates.java @@ -9,7 +9,7 @@ import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIChatCompletion; import com.microsoft.semantickernel.semanticfunctions.AggregatorPromptTemplateFactory; import com.microsoft.semantickernel.semanticfunctions.HandlebarsPromptTemplateFactory; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionFromPrompt; import com.microsoft.semantickernel.semanticfunctions.KernelPromptTemplateFactory; import com.microsoft.semantickernel.semanticfunctions.PromptTemplateFactory; @@ -75,7 +75,7 @@ public static void runPrompt(Kernel kernel, String templateFormat, String prompt .withPromptTemplateFactory(templateFactory) .build(); - var arguments = KernelFunctionArguments.builder() + var arguments = KernelArguments.builder() .withVariable("name", "Bob") .build(); diff --git a/samples/semantickernel-demos/sk-presidio-sample/src/main/java/com/microsoft/semantickernel/Main.java b/samples/semantickernel-demos/sk-presidio-sample/src/main/java/com/microsoft/semantickernel/Main.java index 366d5014..ad14c3ca 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/src/main/java/com/microsoft/semantickernel/Main.java +++ b/samples/semantickernel-demos/sk-presidio-sample/src/main/java/com/microsoft/semantickernel/Main.java @@ -13,7 +13,7 @@ import com.microsoft.semantickernel.presidio.AnonymizedText; import com.microsoft.semantickernel.presidio.AnonymizedTextConverter; import com.microsoft.semantickernel.presidio.RedactorPlugin; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.services.ServiceNotFoundException; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; import com.microsoft.semantickernel.services.chatcompletion.ChatHistory; @@ -50,7 +50,7 @@ public static void main(String[] args) throws InterruptedException { .invokeAsync("redactor", "redact") .withResultType(AnonymizedText.class) .withArguments( - KernelFunctionArguments.builder() + KernelArguments.builder() .withVariable("input", text) .build()) .block() diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/CreatingFunctions.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/CreatingFunctions.java index 2cbb5baa..ec3b9b95 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/CreatingFunctions.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/CreatingFunctions.java @@ -12,7 +12,7 @@ import com.microsoft.semantickernel.orchestration.ToolCallBehavior; import com.microsoft.semantickernel.plugin.KernelPluginFactory; import com.microsoft.semantickernel.samples.plugins.MathPlugin; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; import com.microsoft.semantickernel.services.chatcompletion.ChatHistory; import java.util.Scanner; @@ -54,7 +54,7 @@ public static void main(String[] args) { // Test the math plugin var answer = kernel .invokeAsync(kernel.getFunction("MathPlugin", "sqrt")) - .withArguments(KernelFunctionArguments + .withArguments(KernelArguments .builder() .withVariable("number1", 12.0) .build()) diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/FunctionsWithinPrompts.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/FunctionsWithinPrompts.java index f271b757..de394835 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/FunctionsWithinPrompts.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/FunctionsWithinPrompts.java @@ -16,7 +16,7 @@ import com.microsoft.semantickernel.plugin.KernelPluginFactory; import com.microsoft.semantickernel.samples.plugins.ConversationSummaryPlugin; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.services.chatcompletion.AuthorRole; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; import com.microsoft.semantickernel.services.chatcompletion.ChatHistory; @@ -139,7 +139,7 @@ public static void main(String[] args) { System.console().printf("User > "); String request = System.console().readLine(); - KernelFunctionArguments arguments = KernelFunctionArguments.builder() + KernelArguments arguments = KernelArguments.builder() .withVariable("request", request) .withVariable("choices", choices) .withVariable("history", history) @@ -162,7 +162,7 @@ public static void main(String[] args) { // Get chat response FunctionResult chatResult = chat.invokeAsync(kernel) .withArguments( - KernelFunctionArguments.builder() + KernelArguments.builder() .withVariable("request", request) .withVariable("history", history) .build()) diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/SerializingPrompts.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/SerializingPrompts.java index 3b5ba591..233177e7 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/SerializingPrompts.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/SerializingPrompts.java @@ -11,7 +11,7 @@ import com.microsoft.semantickernel.plugin.KernelPluginFactory; import com.microsoft.semantickernel.samples.plugins.ConversationSummaryPlugin; import com.microsoft.semantickernel.semanticfunctions.HandlebarsPromptTemplateFactory; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionYaml; import com.microsoft.semantickernel.services.chatcompletion.AuthorRole; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; @@ -119,7 +119,7 @@ public static void main(String[] args) throws IOException { // var intent = kernel.invokeAsync(getIntent) - .withArguments(KernelFunctionArguments.builder() + .withArguments(KernelArguments.builder() .withVariable("request", userInput) .withVariable("choices", choices) .withVariable("history", historyString) @@ -134,7 +134,7 @@ public static void main(String[] args) throws IOException { } var reply = kernel.invokeAsync(prompts.get("Chat")) - .withArguments(KernelFunctionArguments.builder() + .withArguments(KernelArguments.builder() .withVariable("request", userInput) .withVariable("history", String.join("\n", diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/Templates.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/Templates.java index 41b09651..b830cd29 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/Templates.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/Templates.java @@ -14,7 +14,7 @@ import com.microsoft.semantickernel.plugin.KernelPluginFactory; import com.microsoft.semantickernel.samples.plugins.ConversationSummaryPlugin; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionFromPrompt; import com.microsoft.semantickernel.services.chatcompletion.AuthorRole; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; @@ -154,7 +154,7 @@ public static void main(String[] args) { System.out.print("User > "); String request = scanner.nextLine(); - KernelFunctionArguments arguments = KernelFunctionArguments.builder() + KernelArguments arguments = KernelArguments.builder() .withVariable("request", request) .withVariable("choices", choices) .withVariable("chatHistory", history) @@ -180,7 +180,7 @@ public static void main(String[] args) { // Get chat response FunctionResult chatResult = chat.invokeAsync(kernel) .withArguments( - KernelFunctionArguments.builder() + KernelArguments.builder() .withVariable("request", request) .withVariable("history", history, chatHistoryType) .build()) diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/UsingTheKernel.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/UsingTheKernel.java index 73f137dc..ebb0e497 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/UsingTheKernel.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/UsingTheKernel.java @@ -10,7 +10,7 @@ import com.microsoft.semantickernel.plugin.KernelPlugin; import com.microsoft.semantickernel.plugin.KernelPluginFactory; import com.microsoft.semantickernel.samples.plugins.MathPlugin; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; public class UsingTheKernel { @@ -76,7 +76,7 @@ public static void main(String[] args) { var result = poemPlugin.get("ShortPoem") .invokeAsync(kernel) .withArguments( - KernelFunctionArguments.builder() + KernelArguments.builder() .withInput("The cat sat on a mat") .build()) .withResultType(String.class) @@ -88,7 +88,7 @@ public static void main(String[] args) { var root = mathPlugin.get("sqrt") .invokeAsync(kernel) .withArguments( - KernelFunctionArguments.builder() + KernelArguments.builder() .withInput(12) .build()) .withResultType(Double.class) diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/plugins/ConversationSummaryPlugin.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/plugins/ConversationSummaryPlugin.java index 247fc48c..1bd16371 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/plugins/ConversationSummaryPlugin.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/plugins/ConversationSummaryPlugin.java @@ -5,7 +5,7 @@ import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; import com.microsoft.semantickernel.semanticfunctions.annotations.KernelFunctionParameter; import com.microsoft.semantickernel.text.TextChunker; @@ -71,7 +71,7 @@ private static Mono processAsync(KernelFunction func, String inp // The first parameter is the input text. return func.invokeAsync(kernel) .withArguments( - new KernelFunctionArguments.Builder() + new KernelArguments.Builder() .withInput(paragraph) .build()) .withResultType( diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/src/main/java/com/microsoft/semantickernel/samples/openapi/OpenAPIHttpRequestPlugin.java b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/src/main/java/com/microsoft/semantickernel/samples/openapi/OpenAPIHttpRequestPlugin.java index e37b0891..58b3e525 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/src/main/java/com/microsoft/semantickernel/samples/openapi/OpenAPIHttpRequestPlugin.java +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/src/main/java/com/microsoft/semantickernel/samples/openapi/OpenAPIHttpRequestPlugin.java @@ -12,7 +12,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.contextvariables.ContextVariable; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import io.swagger.v3.oas.models.Operation; import io.swagger.v3.oas.models.PathItem; import io.swagger.v3.oas.models.parameters.Parameter; @@ -66,7 +66,7 @@ public OpenAPIHttpRequestPlugin( * @param arguments The arguments to the http request. * @return The body of the response. */ - public Mono execute(KernelFunctionArguments arguments) { + public Mono execute(KernelArguments arguments) { String body = getBody(arguments); String query = buildQueryString(arguments); String path = buildQueryPath(arguments); @@ -113,7 +113,7 @@ public Mono execute(KernelFunctionArguments arguments) { .doOnNext(response -> LOGGER.debug("Request response: {}", response)); } - private static @Nullable String getBody(KernelFunctionArguments arguments) { + private static @Nullable String getBody(KernelArguments arguments) { String body = null; if (arguments.containsKey("requestbody")) { ContextVariable requestBody = arguments.get("requestbody"); @@ -130,7 +130,7 @@ public Mono execute(KernelFunctionArguments arguments) { return body; } - private String buildQueryPath(KernelFunctionArguments arguments) { + private String buildQueryPath(KernelArguments arguments) { return getParameterStreamOfArguments(arguments) .filter(p -> p instanceof PathParameter) .reduce(path, (path, parameter) -> { @@ -142,7 +142,7 @@ private String buildQueryPath(KernelFunctionArguments arguments) { } private static String getRenderedParameter( - KernelFunctionArguments arguments, String name) { + KernelArguments arguments, String name) { ContextVariable value = arguments.get(name); if (value == null) { @@ -156,7 +156,7 @@ private static String getRenderedParameter( return URLEncoder.encode(rendered, StandardCharsets.US_ASCII); } - private String buildQueryString(KernelFunctionArguments arguments) { + private String buildQueryString(KernelArguments arguments) { return getParameterStreamOfArguments(arguments) .filter(p -> p instanceof QueryParameter) .map(parameter -> { @@ -168,7 +168,7 @@ private String buildQueryString(KernelFunctionArguments arguments) { } private Stream getParameterStreamOfArguments( - KernelFunctionArguments arguments) { + KernelArguments arguments) { if (operation.getParameters() == null) { return Stream.empty(); } diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/src/main/java/com/microsoft/semantickernel/samples/openapi/SemanticKernelOpenAPIImporter.java b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/src/main/java/com/microsoft/semantickernel/samples/openapi/SemanticKernelOpenAPIImporter.java index c1be538d..2c01f076 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/src/main/java/com/microsoft/semantickernel/samples/openapi/SemanticKernelOpenAPIImporter.java +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/src/main/java/com/microsoft/semantickernel/samples/openapi/SemanticKernelOpenAPIImporter.java @@ -15,7 +15,7 @@ import com.microsoft.semantickernel.plugin.KernelPlugin; import com.microsoft.semantickernel.semanticfunctions.InputVariable; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.OutputVariable; import io.swagger.v3.oas.models.OpenAPI; import io.swagger.v3.oas.models.Operation; @@ -518,7 +518,7 @@ private static KernelFunction buildKernelFunction( try { Method method = OpenAPIHttpRequestPlugin.class.getMethod("execute", - KernelFunctionArguments.class); + KernelArguments.class); return KernelFunction .createFromMethod(method, plugin) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/Kernel.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/Kernel.java index 57e46165..cf38760f 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/Kernel.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/Kernel.java @@ -9,7 +9,7 @@ import com.microsoft.semantickernel.orchestration.InvocationContext; import com.microsoft.semantickernel.plugin.KernelPlugin; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.services.AIService; import com.microsoft.semantickernel.services.AIServiceCollection; import com.microsoft.semantickernel.services.AIServiceSelection; @@ -173,7 +173,7 @@ public FunctionInvocation invokePromptAsync(@Nonnull String prompt) { * @see KernelFunction#invokeAsync(Kernel) */ public FunctionInvocation invokePromptAsync(@Nonnull String prompt, - @Nonnull KernelFunctionArguments arguments) { + @Nonnull KernelArguments arguments) { KernelFunction function = KernelFunction.createFromPrompt(prompt).build(); return function.invokeAsync(this) @@ -192,7 +192,7 @@ public FunctionInvocation invokePromptAsync(@Nonnull String prompt, */ public FunctionInvocation invokePromptAsync(@Nonnull String prompt, - @Nonnull KernelFunctionArguments arguments, @Nonnull InvocationContext invocationContext) { + @Nonnull KernelArguments arguments, @Nonnull InvocationContext invocationContext) { KernelFunction function = KernelFunction.createFromPrompt(prompt).build(); @@ -278,7 +278,7 @@ public List> getFunctions() { * addition to any hooks provided to a function. * * @return The {@code KernelHooks} used throughout the kernel. - * @see KernelFunction#invokeAsync(Kernel, KernelFunctionArguments, ContextVariableType, + * @see KernelFunction#invokeAsync(Kernel, KernelArguments, ContextVariableType, * InvocationContext) */ @SuppressFBWarnings("EI_EXPOSE_REP") @@ -303,7 +303,7 @@ public AIServiceSelector getServiceSelector() { * @return The service of the specified type from the kernel. * @throws ServiceNotFoundException if the service is not found. * @see com.microsoft.semantickernel.services.AIServiceSelector#trySelectAIService(Class, - * KernelFunction, com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments) + * KernelFunction, KernelArguments) */ public T getService(Class clazz) throws ServiceNotFoundException { AIServiceSelection selector = serviceSelector diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/FunctionInvokedEvent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/FunctionInvokedEvent.java index 8f2d7eed..7a39cdfe 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/FunctionInvokedEvent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/FunctionInvokedEvent.java @@ -3,7 +3,7 @@ import com.microsoft.semantickernel.orchestration.FunctionResult; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.annotation.Nullable; @@ -16,7 +16,7 @@ public class FunctionInvokedEvent implements KernelHookEvent { private final KernelFunction function; @Nullable - private final KernelFunctionArguments arguments; + private final KernelArguments arguments; private final FunctionResult result; /** @@ -28,10 +28,10 @@ public class FunctionInvokedEvent implements KernelHookEvent { */ public FunctionInvokedEvent( KernelFunction function, - @Nullable KernelFunctionArguments arguments, + @Nullable KernelArguments arguments, FunctionResult result) { this.function = function; - this.arguments = KernelFunctionArguments.builder().withVariables(arguments).build(); + this.arguments = KernelArguments.builder().withVariables(arguments).build(); this.result = result; } @@ -51,7 +51,7 @@ public KernelFunction getFunction() { */ @SuppressFBWarnings("EI_EXPOSE_REP") @Nullable - public KernelFunctionArguments getArguments() { + public KernelArguments getArguments() { return arguments; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/FunctionInvokingEvent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/FunctionInvokingEvent.java index ab765cab..ca360a76 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/FunctionInvokingEvent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/FunctionInvokingEvent.java @@ -2,7 +2,7 @@ package com.microsoft.semantickernel.hooks; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.annotation.Nullable; @@ -16,7 +16,7 @@ public class FunctionInvokingEvent implements KernelHookEvent { private final KernelFunction function; - private final KernelFunctionArguments arguments; + private final KernelArguments arguments; /** * Creates a new instance of the FunctionInvokingEvent class. @@ -25,9 +25,9 @@ public class FunctionInvokingEvent implements KernelHookEvent { * @param arguments The arguments that are being passed to the function */ public FunctionInvokingEvent(KernelFunction function, - @Nullable KernelFunctionArguments arguments) { + @Nullable KernelArguments arguments) { this.function = function; - this.arguments = KernelFunctionArguments.builder().withVariables(arguments).build(); + this.arguments = KernelArguments.builder().withVariables(arguments).build(); } /** @@ -45,7 +45,7 @@ public KernelFunction getFunction() { * @return the arguments */ @SuppressFBWarnings("EI_EXPOSE_REP") - public KernelFunctionArguments getArguments() { + public KernelArguments getArguments() { return arguments; } } \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PreToolCallEvent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PreToolCallEvent.java index c991efca..42430f08 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PreToolCallEvent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PreToolCallEvent.java @@ -3,7 +3,7 @@ import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.annotation.Nullable; @@ -15,7 +15,7 @@ public class PreToolCallEvent implements KernelHookEvent { private final ContextVariableTypes contextVariableTypes; private final String functionName; @Nullable - private final KernelFunctionArguments arguments; + private final KernelArguments arguments; private final KernelFunction function; /** @@ -29,7 +29,7 @@ public class PreToolCallEvent implements KernelHookEvent { @SuppressFBWarnings("EI_EXPOSE_REP2") public PreToolCallEvent( String functionName, - @Nullable KernelFunctionArguments arguments, + @Nullable KernelArguments arguments, KernelFunction function, ContextVariableTypes contextVariableTypes) { this.functionName = functionName; @@ -44,7 +44,7 @@ public PreToolCallEvent( */ @SuppressFBWarnings("EI_EXPOSE_REP") @Nullable - public KernelFunctionArguments getArguments() { + public KernelArguments getArguments() { return arguments; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PromptRenderedEvent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PromptRenderedEvent.java index f3f34972..348d3bf1 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PromptRenderedEvent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PromptRenderedEvent.java @@ -2,7 +2,7 @@ package com.microsoft.semantickernel.hooks; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.annotation.Nullable; @@ -12,7 +12,7 @@ public class PromptRenderedEvent implements KernelHookEvent { private final KernelFunction function; - private final KernelFunctionArguments arguments; + private final KernelArguments arguments; private final String prompt; /** @@ -24,10 +24,10 @@ public class PromptRenderedEvent implements KernelHookEvent { */ public PromptRenderedEvent( KernelFunction function, - @Nullable KernelFunctionArguments arguments, + @Nullable KernelArguments arguments, String prompt) { this.function = function; - this.arguments = KernelFunctionArguments.builder().withVariables(arguments).build(); + this.arguments = KernelArguments.builder().withVariables(arguments).build(); this.prompt = prompt; } @@ -46,7 +46,7 @@ public KernelFunction getFunction() { * @return the arguments */ @SuppressFBWarnings("EI_EXPOSE_REP") - public KernelFunctionArguments getArguments() { + public KernelArguments getArguments() { return arguments; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PromptRenderingEvent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PromptRenderingEvent.java index 7bba17a3..fe9c5445 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PromptRenderingEvent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PromptRenderingEvent.java @@ -2,7 +2,7 @@ package com.microsoft.semantickernel.hooks; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.annotation.Nullable; @@ -12,7 +12,7 @@ public class PromptRenderingEvent implements KernelHookEvent { private final KernelFunction function; - private final KernelFunctionArguments arguments; + private final KernelArguments arguments; /** * Creates a new instance of the {@link PromptRenderingEvent} class. @@ -21,9 +21,9 @@ public class PromptRenderingEvent implements KernelHookEvent { * @param arguments the arguments */ public PromptRenderingEvent(KernelFunction function, - @Nullable KernelFunctionArguments arguments) { + @Nullable KernelArguments arguments) { this.function = function; - this.arguments = KernelFunctionArguments.builder().withVariables(arguments).build(); + this.arguments = KernelArguments.builder().withVariables(arguments).build(); } /** @@ -41,7 +41,7 @@ public KernelFunction getFunction() { * @return the arguments */ @SuppressFBWarnings("EI_EXPOSE_REP") - public KernelFunctionArguments getArguments() { + public KernelArguments getArguments() { return arguments; } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/FunctionSpan.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/FunctionSpan.java index 601b4130..72cc0260 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/FunctionSpan.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/telemetry/FunctionSpan.java @@ -2,7 +2,7 @@ package com.microsoft.semantickernel.implementation.telemetry; import com.microsoft.semantickernel.orchestration.FunctionResult; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanBuilder; import io.opentelemetry.api.trace.SpanKind; @@ -27,7 +27,7 @@ public static FunctionSpan build( ContextView contextView, String pluginName, String name, - KernelFunctionArguments arguments) { + KernelArguments arguments) { SpanBuilder builder = telemetry.spanBuilder( String.format("function_invocation %s-%s", pluginName, name)) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/DefaultPromptTemplate.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/DefaultPromptTemplate.java index d0b70083..15b80a66 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/DefaultPromptTemplate.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/DefaultPromptTemplate.java @@ -13,7 +13,7 @@ import com.microsoft.semantickernel.implementation.templateengine.tokenizer.blocks.VarBlock; import com.microsoft.semantickernel.orchestration.InvocationContext; import com.microsoft.semantickernel.semanticfunctions.InputVariable; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.PromptTemplate; import com.microsoft.semantickernel.semanticfunctions.PromptTemplateConfig; import com.microsoft.semantickernel.templateengine.semantickernel.TemplateException; @@ -137,7 +137,7 @@ private static PromptTemplateConfig addMissingInputVariables( @Override public Mono renderAsync( Kernel kernel, - @Nullable KernelFunctionArguments arguments, + @Nullable KernelArguments arguments, @Nullable InvocationContext context) { ContextVariableTypes types; diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/CodeBlock.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/CodeBlock.java index e294c925..cd6fffaa 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/CodeBlock.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/CodeBlock.java @@ -9,7 +9,7 @@ import com.microsoft.semantickernel.localization.SemanticKernelResources; import com.microsoft.semantickernel.orchestration.FunctionResult; import com.microsoft.semantickernel.orchestration.InvocationContext; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionMetadata; import com.microsoft.semantickernel.templateengine.semantickernel.TemplateException; import com.microsoft.semantickernel.templateengine.semantickernel.TemplateException.ErrorCodes; @@ -97,7 +97,7 @@ private boolean isValidFunctionCall() { @Override public Mono renderCodeAsync( Kernel kernel, - @Nullable KernelFunctionArguments arguments, + @Nullable KernelArguments arguments, @Nullable InvocationContext context) { if (!this.isValid()) { throw new TemplateException(ErrorCodes.SYNTAX_ERROR); @@ -136,7 +136,7 @@ public Mono renderCodeAsync( private Mono> renderFunctionCallAsync( FunctionIdBlock fBlock, Kernel kernel, - @Nullable KernelFunctionArguments arguments, + @Nullable KernelArguments arguments, InvocationContext context, ContextVariableType resultType) { @@ -145,7 +145,7 @@ private Mono> renderFunctionCallAsync( if (this.tokens.size() > 1) { //Cloning the original arguments to avoid side effects - arguments added to the original arguments collection as a result of rendering template variables. arguments = this.enrichFunctionArguments(kernel, fBlock, - KernelFunctionArguments.builder().withVariables(arguments).build(), + KernelArguments.builder().withVariables(arguments).build(), context); } @@ -168,10 +168,10 @@ private Mono> renderFunctionCallAsync( /// The prompt rendering arguments. /// The function arguments. /// Occurs when any argument other than the first is not a named argument. - private KernelFunctionArguments enrichFunctionArguments( + private KernelArguments enrichFunctionArguments( Kernel kernel, FunctionIdBlock fBlock, - KernelFunctionArguments arguments, + KernelArguments arguments, @Nullable InvocationContext context) { Block firstArg = this.tokens.get(1); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/CodeRendering.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/CodeRendering.java index 7a411a09..673bcf68 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/CodeRendering.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/CodeRendering.java @@ -3,7 +3,7 @@ import com.microsoft.semantickernel.Kernel; import com.microsoft.semantickernel.orchestration.InvocationContext; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import javax.annotation.Nullable; import reactor.core.publisher.Mono; @@ -24,6 +24,6 @@ public interface CodeRendering { */ Mono renderCodeAsync( Kernel kernel, - @Nullable KernelFunctionArguments arguments, + @Nullable KernelArguments arguments, @Nullable InvocationContext context); } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/FunctionIdBlock.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/FunctionIdBlock.java index 58f8f534..69e5a63b 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/FunctionIdBlock.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/FunctionIdBlock.java @@ -2,7 +2,7 @@ package com.microsoft.semantickernel.implementation.templateengine.tokenizer.blocks; import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import javax.annotation.Nullable; /** @@ -49,7 +49,7 @@ private static boolean hasMoreThanOneDot(String value) { @Override @Nullable - public String render(ContextVariableTypes types, @Nullable KernelFunctionArguments variables) { + public String render(ContextVariableTypes types, @Nullable KernelArguments variables) { return this.getContent(); } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/NamedArgBlock.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/NamedArgBlock.java index 010ba44e..bed79502 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/NamedArgBlock.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/NamedArgBlock.java @@ -7,7 +7,7 @@ import com.microsoft.semantickernel.exceptions.SKException; import com.microsoft.semantickernel.implementation.Verify; import com.microsoft.semantickernel.localization.SemanticKernelResources; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -132,7 +132,7 @@ public boolean isValid() { } @Override - public String render(ContextVariableTypes types, @Nullable KernelFunctionArguments variables) { + public String render(ContextVariableTypes types, @Nullable KernelArguments variables) { return getContent(); } @@ -155,7 +155,7 @@ public String getName() { } @SuppressWarnings("NullAway") - public String getValue(ContextVariableTypes types, KernelFunctionArguments arguments) { + public String getValue(ContextVariableTypes types, KernelArguments arguments) { boolean valueIsValidValBlock = this.valBlock != null && this.valBlock.isValid(); if (valueIsValidValBlock) { return this.valBlock.render(types, arguments); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/TextBlock.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/TextBlock.java index 3b7d1c09..38128538 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/TextBlock.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/TextBlock.java @@ -2,7 +2,7 @@ package com.microsoft.semantickernel.implementation.templateengine.tokenizer.blocks; import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import javax.annotation.Nullable; public final class TextBlock extends Block implements TextRendering { @@ -21,7 +21,7 @@ public boolean isValid() { } @Override - public String render(ContextVariableTypes types, @Nullable KernelFunctionArguments variables) { + public String render(ContextVariableTypes types, @Nullable KernelArguments variables) { return super.getContent(); } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/TextRendering.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/TextRendering.java index 75119074..184b4ca4 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/TextRendering.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/TextRendering.java @@ -2,7 +2,7 @@ package com.microsoft.semantickernel.implementation.templateengine.tokenizer.blocks; import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import javax.annotation.Nullable; /** @@ -17,5 +17,5 @@ public interface TextRendering { * @return Rendered content */ @Nullable - String render(ContextVariableTypes types, @Nullable KernelFunctionArguments variables); + String render(ContextVariableTypes types, @Nullable KernelArguments variables); } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/ValBlock.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/ValBlock.java index 3d647840..d81c84b9 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/ValBlock.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/ValBlock.java @@ -3,7 +3,7 @@ import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; import com.microsoft.semantickernel.localization.SemanticKernelResources; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import reactor.util.annotation.Nullable; @@ -41,7 +41,7 @@ public static boolean hasValPrefix(@Nullable String text) { @Override @Nullable - public String render(ContextVariableTypes types, @Nullable KernelFunctionArguments variables) { + public String render(ContextVariableTypes types, @Nullable KernelArguments variables) { return value; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/VarBlock.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/VarBlock.java index 1aec228c..5ae70bb3 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/VarBlock.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/templateengine/tokenizer/blocks/VarBlock.java @@ -4,7 +4,7 @@ import com.microsoft.semantickernel.contextvariables.ContextVariable; import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; import com.microsoft.semantickernel.localization.SemanticKernelResources; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.templateengine.semantickernel.TemplateException; import javax.annotation.Nullable; import org.slf4j.Logger; @@ -26,7 +26,7 @@ public VarBlock(String content) { } @Override - public String render(ContextVariableTypes types, @Nullable KernelFunctionArguments variables) { + public String render(ContextVariableTypes types, @Nullable KernelArguments variables) { if (variables == null) { return ""; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java index 435d8538..0ae16e19 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java @@ -14,7 +14,7 @@ import com.microsoft.semantickernel.implementation.telemetry.SemanticKernelTelemetry; import com.microsoft.semantickernel.localization.SemanticKernelResources; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.NoSuchElementException; import java.util.function.BiConsumer; @@ -41,7 +41,7 @@ public class FunctionInvocation extends Mono> { protected final ContextVariableType resultType; protected final ContextVariableTypes contextVariableTypes = new ContextVariableTypes(); @Nullable - protected KernelFunctionArguments arguments; + protected KernelArguments arguments; @Nullable protected UnmodifiableKernelHooks hooks; @Nullable @@ -95,7 +95,7 @@ private static void performSubscribe( CoreSubscriber> coreSubscriber, Kernel kernel, KernelFunction function, - @Nullable KernelFunctionArguments arguments, + @Nullable KernelArguments arguments, @Nullable ContextVariableType variableType, @Nullable InvocationContext context) { if (variableType == null) { @@ -111,7 +111,7 @@ private static void performSubscribe( function .invokeAsync( kernel, - KernelFunctionArguments + KernelArguments .builder() .withVariables(arguments) .build(), @@ -174,9 +174,9 @@ private static UnmodifiableKernelHooks unmodifiableClone( * @return this {@code FunctionInvocation} for fluent chaining. */ public FunctionInvocation withArguments( - @Nullable KernelFunctionArguments arguments) { + @Nullable KernelArguments arguments) { logSubscribeWarning(); - this.arguments = KernelFunctionArguments.builder().withVariables(arguments).build(); + this.arguments = KernelArguments.builder().withVariables(arguments).build(); return this; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java new file mode 100644 index 00000000..b8100551 --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java @@ -0,0 +1,410 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.semanticfunctions; + +import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.contextvariables.CaseInsensitiveMap; +import com.microsoft.semantickernel.contextvariables.ContextVariable; +import com.microsoft.semantickernel.contextvariables.ContextVariableType; +import com.microsoft.semantickernel.contextvariables.ContextVariableTypeConverter; +import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; +import com.microsoft.semantickernel.exceptions.SKException; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; +import reactor.util.annotation.NonNull; + +/** + * Arguments to a kernel function. + */ +public class KernelArguments implements Map> { + + /** + * Default key for the main input. + */ + public static final String MAIN_KEY = "input"; + + protected final CaseInsensitiveMap> variables; + protected final Map promptExecutionSettings; + + /** + * Create a new instance of KernelArguments. + * + * @param variables The variables to use for the function invocation. + */ + protected KernelArguments( + @Nullable Map> variables, + @Nullable Map promptExecutionSettings) { + if (variables == null) { + this.variables = new CaseInsensitiveMap<>(); + } else { + this.variables = new CaseInsensitiveMap<>(variables); + } + + if (promptExecutionSettings == null) { + this.promptExecutionSettings = new HashMap<>(); + } else { + this.promptExecutionSettings = new HashMap<>(promptExecutionSettings); + } + } + + /** + * Create a new instance of KernelArguments. + * + * @param content The content to use for the function invocation. + */ + protected KernelArguments(@NonNull ContextVariable content) { + this.variables = new CaseInsensitiveMap<>(); + this.variables.put(MAIN_KEY, content); + + this.promptExecutionSettings = new HashMap<>(); + } + + /** + * Create a new instance of KernelArguments. + */ + protected KernelArguments() { + this.variables = new CaseInsensitiveMap<>(); + this.promptExecutionSettings = new HashMap<>(); + } + + /** + * Create a new instance of KernelArguments. + * + * @param arguments The arguments to copy. + */ + protected KernelArguments(@NonNull KernelArguments arguments) { + this.variables = new CaseInsensitiveMap<>(arguments.variables); + this.promptExecutionSettings = new HashMap<>(arguments.promptExecutionSettings); + } + + /** + * Get the prompt execution settings + * + * @return prompt execution settings + */ + @Nonnull + public Map getPromptExecutionSettings() { + return promptExecutionSettings; + } + + /** + * Get the input (entry in the MAIN_KEY slot) + * + * @return input + */ + @Nullable + public ContextVariable getInput() { + return get(MAIN_KEY); + } + + /** + * Create formatted string of the variables + * + * @return formatted string + */ + public String prettyPrint() { + return variables.entrySet().stream() + .reduce( + "", + (str, entry) -> str + + System.lineSeparator() + + entry.getKey() + + ": " + + entry.getValue().toPromptString(ContextVariableTypes.getGlobalTypes()), + (a, b) -> a + b); + } + + /** + * Return the variable with the given name + * + * @param key variable name + * @return content of the variable + */ + @Nullable + public ContextVariable get(String key) { + return variables.get(key); + } + + /** + * Return the variable with the given name + * + * @param key variable name + * @return content of the variable + */ + @Nullable + ContextVariable get(String key, Class clazz) { + ContextVariable value = variables.get(key); + if (value == null) { + return null; + } else if (clazz.isAssignableFrom(value.getType().getClazz())) { + return (ContextVariable) value; + } + + throw new SKException( + String.format( + "Variable %s is of type %s, but requested type is %s", + key, value.getType().getClazz(), clazz)); + } + + /** + * Return whether the variable with the given name is {@code null} or empty. + * + * @param key the key for the variable + * @return {@code true} if the variable is {@code null} or empty, {@code false} otherwise + */ + public boolean isNullOrEmpty(String key) { + return get(key) == null || get(key).isEmpty(); + } + + @Override + public int size() { + return variables.size(); + } + + @Override + public boolean isEmpty() { + return variables.isEmpty(); + } + + @Override + public boolean containsKey(Object key) { + return variables.containsKey(key); + } + + @Override + public boolean containsValue(Object value) { + return variables.containsValue(value); + } + + @Override + @Nullable + public ContextVariable get(Object key) { + return variables.get(key); + } + + @Override + public ContextVariable put(String key, ContextVariable value) { + return variables.put(key, value); + } + + @Override + public ContextVariable remove(Object key) { + return variables.remove(key); + } + + @Override + public void putAll(Map> m) { + variables.putAll(m); + } + + @Override + public void clear() { + variables.clear(); + } + + @Override + public Set keySet() { + return variables.keySet(); + } + + @Override + public Collection> values() { + return variables.values(); + } + + @Override + public Set>> entrySet() { + return variables.entrySet(); + } + + /** + * Create a copy of the current instance + * + * @return copy of the current instance + */ + public KernelArguments copy() { + return new KernelArguments(variables, promptExecutionSettings); + } + + /** + * Create a new instance of Builder. + * + * @return Builder + */ + public static Builder builder() { + return new Builder<>(KernelArguments::new); + } + + + /** + * Builder for ContextVariables + */ + public static class Builder implements SemanticKernelBuilder { + + private final Function constructor; + private final Map> variables; + private final Map promptExecutionSettings; + + + protected Builder(Function constructor) { + this.constructor = constructor; + this.variables = new HashMap<>(); + this.promptExecutionSettings = new HashMap<>(); + } + + /** + * Builds an instance with the given content in the default main key + * + * @param content Entry to place in the "input" slot + * @param Type of the value + * @return {$code this} Builder for fluent coding + */ + public Builder withInput(ContextVariable content) { + return withVariable(MAIN_KEY, content); + } + + /** + * Builds an instance with the given content in the default main key + * + * @param content Entry to place in the "input" slot + * @return {$code this} Builder for fluent coding + * @throws SKException if the content cannot be converted to a ContextVariable + */ + public Builder withInput(Object content) { + return withInput(ContextVariable.ofGlobalType(content)); + } + + /** + * Builds an instance with the given content in the default main key + * + * @param content Entry to place in the "input" slot + * @param typeConverter Type converter for the content + * @param Type of the value + * @return {$code this} Builder for fluent coding + * @throws SKException if the content cannot be converted to a ContextVariable + */ + public Builder withInput(T content, ContextVariableTypeConverter typeConverter) { + return withInput(new ContextVariable<>( + new ContextVariableType<>( + typeConverter, + typeConverter.getType()), + content)); + } + + /** + * Builds an instance with the given variables + * + * @param map Existing variables + * @return {$code this} Builder for fluent coding + */ + public Builder withVariables(@Nullable Map> map) { + if (map == null) { + return this; + } + variables.putAll(map); + return this; + } + + /** + * Set variable + * + * @param key variable name + * @param value variable value + * @param Type of the value + * @return {$code this} Builder for fluent coding + */ + public Builder withVariable(String key, ContextVariable value) { + variables.put(key, value); + return this; + } + + /** + * Set variable, uses the default type converters + * + * @param key variable name + * @param value variable value + * @return {$code this} Builder for fluent coding + * @throws SKException if the value cannot be converted to a ContextVariable + */ + public Builder withVariable(String key, Object value) { + if (value instanceof ContextVariable) { + return withVariable(key, (ContextVariable) value); + } + return withVariable(key, ContextVariable.ofGlobalType(value)); + } + + /** + * Set variable + * + * @param key variable name + * @param value variable value + * @param typeConverter Type converter for the value + * @param Type of the value + * @return {$code this} Builder for fluent coding + * @throws SKException if the value cannot be converted to a ContextVariable + */ + public Builder withVariable(String key, T value, + ContextVariableTypeConverter typeConverter) { + return withVariable(key, new ContextVariable<>( + new ContextVariableType<>( + typeConverter, + typeConverter.getType()), + value)); + } + + /** + * Set prompt execution settings + * + * @param promptExecutionSettings Prompt execution settings + * @return {$code this} Builder for fluent coding + */ + public Builder withPromptExecutionSettings(Map promptExecutionSettings) { + return withPromptExecutionSettings(new ArrayList<>(promptExecutionSettings.values())); + } + + /** + * Set prompt execution settings + * + * @param promptExecutionSettings Prompt execution settings + * @return {$code this} Builder for fluent coding + */ + public Builder withPromptExecutionSettings(List promptExecutionSettings) { + for (PromptExecutionSettings settings : promptExecutionSettings) { + String serviceId = settings.getServiceId(); + + if (this.promptExecutionSettings.containsKey(serviceId)) { + if (serviceId.equals(PromptExecutionSettings.DEFAULT_SERVICE_ID)) { + throw new SKException( + String.format( + "Multiple prompt execution settings with the default service id '%s' or no service id have been provided. Specify a single default prompt execution settings and provide a unique service id for all other instances.", + PromptExecutionSettings.DEFAULT_SERVICE_ID) + ); + } + + throw new SKException( + String.format( + "Multiple prompt execution settings with the service id '%s' have been provided. Specify a unique service id for all instances.", + serviceId) + ); + } + } + + return this; + } + + @Override + public U build() { + KernelArguments arguments = new KernelArguments(variables, promptExecutionSettings); + return constructor.apply(arguments); + } + } +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunction.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunction.java index 70f33bd7..742e8457 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunction.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunction.java @@ -189,7 +189,7 @@ public KernelFunctionMetadata getMetadata() { */ public abstract Mono> invokeAsync( Kernel kernel, - @Nullable KernelFunctionArguments arguments, + @Nullable KernelArguments arguments, @Nullable ContextVariableType variableType, @Nullable InvocationContext invocationContext); @@ -221,7 +221,7 @@ public abstract Mono> invokeAsync( */ public FunctionResult invoke( Kernel kernel, - @Nullable KernelFunctionArguments arguments, + @Nullable KernelArguments arguments, @Nullable ContextVariableType variableType, @Nullable InvocationContext invocationContext) { return invokeAsync(kernel, arguments, variableType, invocationContext).block(); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionArguments.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionArguments.java index f3b46c21..76936cf2 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionArguments.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionArguments.java @@ -17,16 +17,17 @@ /** * Arguments to a kernel function. + * + * @deprecated Use {@link KernelArguments} instead. */ -public class KernelFunctionArguments implements Map> { +@Deprecated +public class KernelFunctionArguments extends KernelArguments { /** * Default key for the main input. */ public static final String MAIN_KEY = "input"; - private final CaseInsensitiveMap> variables; - /** * Create a new instance of KernelFunctionArguments. * @@ -34,11 +35,7 @@ public class KernelFunctionArguments implements Map> */ protected KernelFunctionArguments( @Nullable Map> variables) { - if (variables == null) { - this.variables = new CaseInsensitiveMap<>(); - } else { - this.variables = new CaseInsensitiveMap<>(variables); - } + super(variables, null); } /** @@ -47,15 +44,23 @@ protected KernelFunctionArguments( * @param content The content to use for the function invocation. */ protected KernelFunctionArguments(@NonNull ContextVariable content) { - this.variables = new CaseInsensitiveMap<>(); - this.variables.put(MAIN_KEY, content); + super(content); + } + + /** + * Create a new instance of KernelArguments. + * + * @param arguments The arguments to copy. + */ + protected KernelFunctionArguments(@NonNull KernelArguments arguments) { + super(arguments); } /** * Create a new instance of KernelFunctionArguments. */ protected KernelFunctionArguments() { - this.variables = new CaseInsensitiveMap<>(); + super(); } /** @@ -208,121 +213,18 @@ public KernelFunctionArguments copy() { /** * Builder for ContextVariables + * + * @deprecated Use {@link KernelArguments} builder instead. */ - public static class Builder implements SemanticKernelBuilder { - - private final Map> variables; + @Deprecated + public static class Builder extends KernelArguments.Builder { /** * Create a new instance of Builder. */ + @Deprecated public Builder() { - variables = new HashMap<>(); - } - - /** - * Builds an instance with the given content in the default main key - * - * @param content Entry to place in the "input" slot - * @param Type of the value - * @return {$code this} Builder for fluent coding - */ - public Builder withInput(ContextVariable content) { - return withVariable(MAIN_KEY, content); - } - - /** - * Builds an instance with the given content in the default main key - * - * @param content Entry to place in the "input" slot - * @return {$code this} Builder for fluent coding - * @throws SKException if the content cannot be converted to a ContextVariable - */ - public Builder withInput(Object content) { - return withInput(ContextVariable.ofGlobalType(content)); - } - - /** - * Builds an instance with the given content in the default main key - * - * @param content Entry to place in the "input" slot - * @param typeConverter Type converter for the content - * @param Type of the value - * @return {$code this} Builder for fluent coding - * @throws SKException if the content cannot be converted to a ContextVariable - */ - public Builder withInput(T content, ContextVariableTypeConverter typeConverter) { - return withInput(new ContextVariable<>( - new ContextVariableType<>( - typeConverter, - typeConverter.getType()), - content)); - } - - /** - * Builds an instance with the given variables - * - * @param map Existing variables - * @return {$code this} Builder for fluent coding - */ - public Builder withVariables(@Nullable Map> map) { - if (map == null) { - return this; - } - variables.putAll(map); - return this; - } - - /** - * Set variable - * - * @param key variable name - * @param value variable value - * @param Type of the value - * @return {$code this} Builder for fluent coding - */ - public Builder withVariable(String key, ContextVariable value) { - variables.put(key, value); - return this; - } - - /** - * Set variable, uses the default type converters - * - * @param key variable name - * @param value variable value - * @return {$code this} Builder for fluent coding - * @throws SKException if the value cannot be converted to a ContextVariable - */ - public Builder withVariable(String key, Object value) { - if (value instanceof ContextVariable) { - return withVariable(key, (ContextVariable) value); - } - return withVariable(key, ContextVariable.ofGlobalType(value)); - } - - /** - * Set variable - * - * @param key variable name - * @param value variable value - * @param typeConverter Type converter for the value - * @param Type of the value - * @return {$code this} Builder for fluent coding - * @throws SKException if the value cannot be converted to a ContextVariable - */ - public Builder withVariable(String key, T value, - ContextVariableTypeConverter typeConverter) { - return withVariable(key, new ContextVariable<>( - new ContextVariableType<>( - typeConverter, - typeConverter.getType()), - value)); - } - - @Override - public KernelFunctionArguments build() { - return new KernelFunctionArguments(variables); + super(KernelFunctionArguments::new); } } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java index afda500a..6704255f 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java @@ -176,7 +176,7 @@ public static ImplementationFunc getFunction(Method method, Object instan FunctionInvokingEvent updatedState = kernelHooks .executeHooks( new FunctionInvokingEvent(function, arguments)); - KernelFunctionArguments updatedArguments = updatedState != null + KernelArguments updatedArguments = updatedState != null ? updatedState.getArguments() : arguments; @@ -344,11 +344,11 @@ private static Mono invokeAsyncFunction( @Nullable private static Function getParameters( Method method, - @Nullable KernelFunctionArguments context, + @Nullable KernelArguments context, Kernel kernel, InvocationContext invocationContext) { return parameter -> { - if (KernelFunctionArguments.class.isAssignableFrom(parameter.getType())) { + if (KernelArguments.class.isAssignableFrom(parameter.getType())) { return context; } else if (Kernel.class.isAssignableFrom(parameter.getType())) { return kernel; @@ -361,7 +361,7 @@ private static Function getParameters( @Nullable private static Object getArgumentValue( Method method, - @Nullable KernelFunctionArguments context, + @Nullable KernelArguments context, Parameter parameter, Kernel kernel, InvocationContext invocationContext) { @@ -525,15 +525,15 @@ private static Object toObjectType( @Nullable private static ContextVariable getVariableFromContext( Method method, - @Nullable KernelFunctionArguments context, + @Nullable KernelArguments context, String variableName) { ContextVariable variable = context == null ? null : context.get(variableName); // If there is 1 argument use "input" or the only argument if (variable == null && method.getParameters().length == 1) { if (context != null) { - if (context.containsKey(KernelFunctionArguments.MAIN_KEY)) { - variable = context.get(KernelFunctionArguments.MAIN_KEY); + if (context.containsKey(KernelArguments.MAIN_KEY)) { + variable = context.get(KernelArguments.MAIN_KEY); } else if (context.size() == 1) { variable = context.values().iterator().next(); } @@ -672,7 +672,7 @@ private static InputVariable toKernelParameterMetadata(Parameter parameter) { boolean isRequired = true; Class type = parameter.getType(); - if (Kernel.class.isAssignableFrom(type) || KernelFunctionArguments.class.isAssignableFrom( + if (Kernel.class.isAssignableFrom(type) || KernelArguments.class.isAssignableFrom( type)) { return null; } @@ -729,7 +729,7 @@ public static Builder builder() { @Override public Mono> invokeAsync( Kernel kernel, - @Nullable KernelFunctionArguments arguments, + @Nullable KernelArguments arguments, @Nullable ContextVariableType variableType, @Nullable InvocationContext invocationContext) { @@ -770,7 +770,7 @@ public interface ImplementationFunc { Mono> invokeAsync( Kernel kernel, KernelFunction function, - @Nullable KernelFunctionArguments arguments, + @Nullable KernelArguments arguments, @Nullable ContextVariableType variableType, @Nullable InvocationContext invocationContext); @@ -787,7 +787,7 @@ Mono> invokeAsync( default FunctionResult invoke( Kernel kernel, KernelFunction function, - @Nullable KernelFunctionArguments arguments, + @Nullable KernelArguments arguments, @Nullable ContextVariableType variableType, @Nullable InvocationContext invocationContext) { return invokeAsync(kernel, function, arguments, variableType, diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java index 5c2c5412..1d12f961 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java @@ -98,7 +98,7 @@ public static Builder builder(Class returnType) { private Flux> invokeInternalAsync( Kernel kernel, - @Nullable KernelFunctionArguments argumentsIn, + @Nullable KernelArguments argumentsIn, @Nullable ContextVariableType contextVariableType, @Nullable InvocationContext invocationContext) { @@ -113,7 +113,7 @@ private Flux> invokeInternalAsync( PromptRenderingEvent preRenderingHookResult = kernelHooks .executeHooks(new PromptRenderingEvent(this, argumentsIn)); - KernelFunctionArguments arguments = preRenderingHookResult.getArguments(); + KernelArguments arguments = preRenderingHookResult.getArguments(); // TODO: put in method, add catch for classcastexception, fallback to noopconverter ContextVariableType variableType = contextVariableType != null @@ -127,7 +127,7 @@ private Flux> invokeInternalAsync( PromptRenderedEvent promptHookResult = kernelHooks .executeHooks(new PromptRenderedEvent(this, arguments, prompt)); prompt = promptHookResult.getPrompt(); - KernelFunctionArguments args = promptHookResult.getArguments(); + KernelArguments args = promptHookResult.getArguments(); LOGGER.info(SemanticKernelResources.getString("rendered.prompt"), prompt); @@ -273,7 +273,7 @@ private Flux> invokeInternalAsync( @Override public Mono> invokeAsync( Kernel kernel, - @Nullable KernelFunctionArguments arguments, + @Nullable KernelArguments arguments, @Nullable ContextVariableType variableType, @Nullable InvocationContext invocationContext) { return Mono.deferContextual(contextView -> { diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplate.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplate.java index 04f0c643..58a5c6a8 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplate.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplate.java @@ -24,7 +24,7 @@ public interface PromptTemplate { */ Mono renderAsync( Kernel kernel, - @Nullable KernelFunctionArguments arguments, + @Nullable KernelArguments arguments, @Nullable InvocationContext context); } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIServiceSelector.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIServiceSelector.java index 3255d79e..c48fa475 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIServiceSelector.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIServiceSelector.java @@ -2,7 +2,7 @@ package com.microsoft.semantickernel.services; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import javax.annotation.Nullable; /** @@ -15,7 +15,7 @@ public interface AIServiceSelector { /** * Resolves an {@link AIService} and associated and * {@link com.microsoft.semantickernel.orchestration.PromptExecutionSettings} based on the - * associated {@link KernelFunction} and {@link KernelFunctionArguments}. + * associated {@link KernelFunction} and {@link KernelArguments}. * * @param serviceType The type of service to select. This must be the same type with which the * service was registered in the {@link AIServiceSelection} @@ -26,9 +26,10 @@ public interface AIServiceSelector { * @return An {@code AIServiceSelection} containing the selected service and associated * PromptExecutionSettings. */ +// @Deprecated @Nullable AIServiceSelection trySelectAIService( Class serviceType, @Nullable KernelFunction function, - @Nullable KernelFunctionArguments arguments); + @Nullable KernelArguments arguments); } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/BaseAIServiceSelector.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/BaseAIServiceSelector.java index 2eaa9404..c3e42b8e 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/BaseAIServiceSelector.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/BaseAIServiceSelector.java @@ -2,16 +2,16 @@ package com.microsoft.semantickernel.services; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import java.util.Map; import javax.annotation.Nullable; /** * Base class for {@link AIServiceSelector} implementations which provides a {@code Map} based * collection from which an {@link AIService} can be selected. The - * {@link #trySelectAIService(Class, KernelFunction, KernelFunctionArguments)} method has been + * {@link #trySelectAIService(Class, KernelFunction, KernelArguments)} method has been * implemented. Child classes must implement the method - * {@link #trySelectAIService(Class, KernelFunction, KernelFunctionArguments, Map)}. + * {@link #trySelectAIService(Class, KernelFunction, KernelArguments, Map)}. */ public abstract class BaseAIServiceSelector implements AIServiceSelector { @@ -31,7 +31,7 @@ protected BaseAIServiceSelector(AIServiceCollection services) { public AIServiceSelection trySelectAIService( Class serviceType, @Nullable KernelFunction function, - @Nullable KernelFunctionArguments arguments) { + @Nullable KernelArguments arguments) { return trySelectAIService(serviceType, function, arguments, services); } @@ -52,6 +52,6 @@ public AIServiceSelection trySelectAIService( protected abstract AIServiceSelection trySelectAIService( Class serviceType, @Nullable KernelFunction function, - @Nullable KernelFunctionArguments arguments, + @Nullable KernelArguments arguments, Map, AIService> services); } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/OrderedAIServiceSelector.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/OrderedAIServiceSelector.java index 8e607cdb..2be73543 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/OrderedAIServiceSelector.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/OrderedAIServiceSelector.java @@ -5,7 +5,7 @@ import com.microsoft.semantickernel.localization.SemanticKernelResources; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; import com.microsoft.semantickernel.services.textcompletion.TextGenerationService; import java.util.List; @@ -66,18 +66,34 @@ private static Map settingsFromFunctionSettings return null; } +// @Nullable +// @Override +// public AIServiceSelection trySelectAIService( +// Class serviceType, +// @Nullable KernelArguments arguments) { +// return selectAIService(serviceType, arguments.getPromptExecutionSettings()); +// } + @Nullable @Override public AIServiceSelection trySelectAIService( Class serviceType, @Nullable KernelFunction function, - @Nullable KernelFunctionArguments arguments, + @Nullable KernelArguments arguments, Map, AIService> services) { // Allow the execution settings from the kernel arguments to take precedence Map executionSettings = settingsFromFunctionSettings( function); + return selectAIService(serviceType, executionSettings); + } + + + private AIServiceSelection selectAIService( + Class serviceType, + @Nullable Map executionSettings) { + if (executionSettings == null || executionSettings.isEmpty()) { AIService service = getAnyService(serviceType); if (service != null) { @@ -85,50 +101,50 @@ public AIServiceSelection trySelectAIService( } } else { AIServiceSelection selection = executionSettings - .entrySet() - .stream() - .map(keyValue -> { - - PromptExecutionSettings settings = keyValue.getValue(); - String serviceId = keyValue.getKey(); - - if (!Verify.isNullOrEmpty(serviceId)) { - AIService service = getService(serviceId); - if (service != null) { - return castServiceSelection( - new AIServiceSelection<>(service, settings)); + .entrySet() + .stream() + .map(keyValue -> { + + PromptExecutionSettings settings = keyValue.getValue(); + String serviceId = keyValue.getKey(); + + if (!Verify.isNullOrEmpty(serviceId)) { + AIService service = getService(serviceId); + if (service != null) { + return castServiceSelection( + new AIServiceSelection<>(service, settings)); + } } - } - return null; - }) - .filter(Objects::nonNull) - .findFirst() - .orElseGet(() -> null); + return null; + }) + .filter(Objects::nonNull) + .findFirst() + .orElseGet(() -> null); if (selection != null) { return castServiceSelection(selection); } selection = executionSettings - .entrySet() - .stream() - .map(keyValue -> { - PromptExecutionSettings settings = keyValue.getValue(); - - if (!Verify.isNullOrEmpty(settings.getModelId())) { - AIService service = getServiceByModelId(settings.getModelId()); - if (service != null) { - return castServiceSelection( - new AIServiceSelection<>(service, settings)); + .entrySet() + .stream() + .map(keyValue -> { + PromptExecutionSettings settings = keyValue.getValue(); + + if (!Verify.isNullOrEmpty(settings.getModelId())) { + AIService service = getServiceByModelId(settings.getModelId()); + if (service != null) { + return castServiceSelection( + new AIServiceSelection<>(service, settings)); + } } - } - return null; - }) - .filter(Objects::nonNull) - .findFirst() - .orElseGet(() -> null); + return null; + }) + .filter(Objects::nonNull) + .findFirst() + .orElseGet(() -> null); if (selection != null) { return castServiceSelection(selection); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplate.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplate.java index aed37718..460e4815 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplate.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplate.java @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.templateengine.handlebars; -import static com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments.MAIN_KEY; +import static com.microsoft.semantickernel.semanticfunctions.KernelArguments.MAIN_KEY; import com.github.jknack.handlebars.Context; import com.github.jknack.handlebars.EscapingStrategy; @@ -19,7 +19,7 @@ import com.microsoft.semantickernel.orchestration.ToolCallBehavior; import com.microsoft.semantickernel.plugin.KernelPlugin; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.PromptTemplate; import com.microsoft.semantickernel.semanticfunctions.PromptTemplateConfig; import com.microsoft.semantickernel.semanticfunctions.PromptTemplateOption; @@ -59,7 +59,7 @@ public HandlebarsPromptTemplate( @Override public Mono renderAsync( Kernel kernel, - @Nullable KernelFunctionArguments arguments, + @Nullable KernelArguments arguments, @Nullable InvocationContext context) { String template = promptTemplate.getTemplate(); if (template == null) { @@ -75,7 +75,7 @@ public Mono renderAsync( template, context); if (arguments == null) { - arguments = KernelFunctionArguments.builder().build(); + arguments = KernelArguments.builder().build(); } return handler.render(arguments); } @@ -131,8 +131,8 @@ private static class ContextVariableResolver implements ValueResolver { public Object resolve(Object context, String name) { Object value = null; ContextVariable variable = null; - if (context instanceof KernelFunctionArguments) { - variable = ((KernelFunctionArguments) context).get(name); + if (context instanceof KernelArguments) { + variable = ((KernelArguments) context).get(name); } else if (context instanceof ContextVariable) { variable = ((ContextVariable) context); } @@ -174,9 +174,9 @@ private String promptString(ContextVariable context) { @Override public Set> propertySet(Object context) { - if (context instanceof KernelFunctionArguments) { + if (context instanceof KernelArguments) { HashMap result = new HashMap<>(); - result.putAll((KernelFunctionArguments) context); + result.putAll((KernelArguments) context); return result.entrySet(); } else if (context instanceof ContextVariable) { HashMap result = new HashMap<>(); @@ -270,7 +270,7 @@ private CharSequence handleMessage(Object context, Options options) return null; } - public Mono render(KernelFunctionArguments variables) { + public Mono render(KernelArguments variables) { try { ArrayList resolvers = new ArrayList<>(); resolvers.add(new MessageResolver()); @@ -319,9 +319,9 @@ private static Helper functionInvokeHelper( InvocationContext invocationContext) { return (context, options) -> { - KernelFunctionArguments.Builder builder = KernelFunctionArguments.builder(); - if (context instanceof KernelFunctionArguments) { - builder.withVariables((KernelFunctionArguments) context); + KernelArguments.Builder builder = KernelArguments.builder(); + if (context instanceof KernelArguments) { + builder.withVariables((KernelArguments) context); } else { builder.withInput(context); } diff --git a/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethodTest.java b/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethodTest.java index 589705f3..a2dae137 100644 --- a/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethodTest.java +++ b/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethodTest.java @@ -52,7 +52,7 @@ void typeConversionOnMethodCall() { .invokeAsync(kernel) .withResultType(ContextVariableTypes.getGlobalVariableTypeForClass(String.class)) .withArguments( - KernelFunctionArguments.builder() + KernelArguments.builder() .withVariable("number1", "12.0") .build()) .block(); @@ -182,7 +182,7 @@ interface InvocationTest { Method getMethod() throws NoSuchMethodException; - KernelFunctionArguments getArguments(); + KernelArguments getArguments(); void assertCalled(); } @@ -203,8 +203,8 @@ public Method getMethod() throws NoSuchMethodException { } @Override - public KernelFunctionArguments getArguments() { - return KernelFunctionArguments.builder() + public KernelArguments getArguments() { + return KernelArguments.builder() .withVariable("i", 123) .build(); } @@ -232,8 +232,8 @@ public Method getMethod() throws NoSuchMethodException { } @Override - public KernelFunctionArguments getArguments() { - return KernelFunctionArguments.builder() + public KernelArguments getArguments() { + return KernelArguments.builder() .withVariable("i", 123) .build(); } @@ -261,8 +261,8 @@ public Method getMethod() throws NoSuchMethodException { } @Override - public KernelFunctionArguments getArguments() { - return KernelFunctionArguments.builder() + public KernelArguments getArguments() { + return KernelArguments.builder() .withVariable("i", 123) .build(); } @@ -290,8 +290,8 @@ public Method getMethod() throws NoSuchMethodException { } @Override - public KernelFunctionArguments getArguments() { - return KernelFunctionArguments.builder() + public KernelArguments getArguments() { + return KernelArguments.builder() .withVariable("i", Arrays.asList(1, 2, 3)) .build(); } @@ -319,8 +319,8 @@ public Method getMethod() throws NoSuchMethodException { } @Override - public KernelFunctionArguments getArguments() { - return KernelFunctionArguments.builder() + public KernelArguments getArguments() { + return KernelArguments.builder() .build(); } @@ -347,7 +347,7 @@ public Method getMethod() throws NoSuchMethodException { } @Override - public KernelFunctionArguments getArguments() { + public KernelArguments getArguments() { ContextVariableTypeConverter dbConverter = ContextVariableTypeConverter .builder(BigDecimal.class) @@ -355,7 +355,7 @@ public KernelFunctionArguments getArguments() { .toPromptString(i -> null) .build(); - return KernelFunctionArguments.builder() + return KernelArguments.builder() .withVariable("i", new BigDecimal(123), dbConverter) .build(); } @@ -401,14 +401,14 @@ public Method getMethod() throws NoSuchMethodException { } @Override - public KernelFunctionArguments getArguments() { + public KernelArguments getArguments() { ContextVariableTypeConverter sourceConverter = ContextVariableTypeConverter .builder(SourceClass.class) .fromObject(i -> (SourceClass) i) .toPromptString(i -> null) .build(); - return KernelFunctionArguments.builder() + return KernelArguments.builder() .withVariable("i", new SourceClass(123), sourceConverter) .build(); } diff --git a/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateFactoryTest.java b/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateFactoryTest.java index f386084e..67e86d43 100644 --- a/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateFactoryTest.java +++ b/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateFactoryTest.java @@ -93,7 +93,7 @@ private void executeTest(String templateFormat) throws Exception { PromptTemplate promptTemplate = PromptTemplateFactory.build(config); - KernelFunctionArguments args = KernelFunctionArguments.builder() + KernelArguments args = KernelArguments.builder() .withInput(ContextVariable.of("input from args")).build(); String expected = String.format("A template for testing: %s", diff --git a/semantickernel-api/src/test/java/com/microsoft/semantickernel/services/AIServiceSelectorTest.java b/semantickernel-api/src/test/java/com/microsoft/semantickernel/services/AIServiceSelectorTest.java index f9be46cd..13ee076f 100644 --- a/semantickernel-api/src/test/java/com/microsoft/semantickernel/services/AIServiceSelectorTest.java +++ b/semantickernel-api/src/test/java/com/microsoft/semantickernel/services/AIServiceSelectorTest.java @@ -8,7 +8,7 @@ import com.microsoft.semantickernel.Kernel; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.PromptTemplateConfig; import org.junit.jupiter.api.Test; @@ -185,7 +185,7 @@ public String getServiceId() { AIServiceSelection expected = new AIServiceSelection<>(aService, null); - KernelFunctionArguments arguments = KernelFunctionArguments.builder().build(); + KernelArguments arguments = KernelArguments.builder().build(); Kernel kernel = Kernel.builder() .withAIService((Class) aService.getClass(), aService) diff --git a/semantickernel-api/src/test/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplateTest.java b/semantickernel-api/src/test/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplateTest.java index 228d64dc..3a31de30 100644 --- a/semantickernel-api/src/test/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplateTest.java +++ b/semantickernel-api/src/test/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplateTest.java @@ -13,7 +13,7 @@ import com.microsoft.semantickernel.orchestration.InvocationContext; import com.microsoft.semantickernel.plugin.KernelPlugin; import com.microsoft.semantickernel.plugin.KernelPluginFactory; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.PromptTemplateConfig; import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; import com.microsoft.semantickernel.semanticfunctions.annotations.KernelFunctionParameter; @@ -97,7 +97,7 @@ void testRenderAsync() { HandlebarsPromptTemplate instance = new HandlebarsPromptTemplate(promptTemplate); - KernelFunctionArguments arguments = KernelFunctionArguments.builder() + KernelArguments arguments = KernelArguments.builder() .withVariable("input", "Hello ") .withVariable("suffix", "World") .withVariable("choices", choices) @@ -146,7 +146,7 @@ public void testSerializesObject() { HandlebarsPromptTemplate instance = new HandlebarsPromptTemplate(promptTemplate); - KernelFunctionArguments arguments = KernelFunctionArguments.builder() + KernelArguments arguments = KernelArguments.builder() .withVariable("input", new Foo("bar"), ContextVariableJacksonConverter.create(Foo.class)) .build(); @@ -171,7 +171,7 @@ public void testMessageContent() { HandlebarsPromptTemplate instance = new HandlebarsPromptTemplate(promptTemplate); - KernelFunctionArguments arguments = KernelFunctionArguments.builder() + KernelArguments arguments = KernelArguments.builder() .withVariable("input", new ChatHistory() .addAssistantMessage("foo") .addUserMessage("bar\"<>&")) @@ -196,7 +196,7 @@ public void testMessageHandler() { HandlebarsPromptTemplate instance = new HandlebarsPromptTemplate(promptTemplate); - KernelFunctionArguments arguments = KernelFunctionArguments.builder() + KernelArguments arguments = KernelArguments.builder() .withVariable("input", "bar\"<>&") .build(); @@ -220,7 +220,7 @@ public void iterableWithContextVariable() { HandlebarsPromptTemplate instance = new HandlebarsPromptTemplate(promptTemplate); - KernelFunctionArguments arguments = KernelFunctionArguments.builder() + KernelArguments arguments = KernelArguments.builder() .withVariable("input", Arrays.asList(ContextVariable.of("foo\"<>&"))) .build(); @@ -245,7 +245,7 @@ public void withCustomConverter() { Foo.class) .toPromptString(Foo::getVal) .build(); - KernelFunctionArguments arguments = KernelFunctionArguments.builder() + KernelArguments arguments = KernelArguments.builder() .withVariable("input", ContextVariable.of(new Foo("bar\"<>&"), converter)) .build(); diff --git a/semantickernel-api/src/test/java/com/microsoft/semantickernel/templateengine/semantickernel/CodeTokenizerTest.java b/semantickernel-api/src/test/java/com/microsoft/semantickernel/templateengine/semantickernel/CodeTokenizerTest.java index 8a491680..d39c6aaf 100644 --- a/semantickernel-api/src/test/java/com/microsoft/semantickernel/templateengine/semantickernel/CodeTokenizerTest.java +++ b/semantickernel-api/src/test/java/com/microsoft/semantickernel/templateengine/semantickernel/CodeTokenizerTest.java @@ -6,7 +6,7 @@ import com.microsoft.semantickernel.implementation.templateengine.tokenizer.blocks.Block; import com.microsoft.semantickernel.implementation.templateengine.tokenizer.blocks.FunctionIdBlock; import com.microsoft.semantickernel.implementation.templateengine.tokenizer.blocks.NamedArgBlock; -import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import java.util.List; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -28,7 +28,7 @@ public void parseNamedArgs1() { Assertions.assertEquals("street", namedArgBlock.getName()); Assertions.assertEquals("123 Main St", namedArgBlock.getValue( new ContextVariableTypes(), - new KernelFunctionArguments.Builder() + KernelArguments.builder() .withVariable("street", "123 Main St") .build())); @@ -36,13 +36,13 @@ public void parseNamedArgs1() { Assertions.assertEquals("zip", namedArgBlock.getName()); Assertions.assertEquals("98123", namedArgBlock.getValue( new ContextVariableTypes(), - new KernelFunctionArguments.Builder().build())); + KernelArguments.builder().build())); namedArgBlock = (NamedArgBlock) tokens.get(3); Assertions.assertEquals("city", namedArgBlock.getName()); Assertions.assertEquals("Seattle", namedArgBlock.getValue( new ContextVariableTypes(), - new KernelFunctionArguments.Builder().build())); + KernelArguments.builder().build())); } @Test @@ -60,7 +60,6 @@ public void parseNamedArgs2() { Assertions.assertEquals("recall", namedArgBlock.getName()); Assertions.assertEquals("where did I grow up?", namedArgBlock.getValue( new ContextVariableTypes(), - new KernelFunctionArguments.Builder() - .build())); + KernelArguments.builder().build())); } } From b378f634ddbebf9ab5aec6baa6e793401d4563f3 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Mon, 14 Apr 2025 23:45:22 -0700 Subject: [PATCH 121/277] Update --- .../samples/syntaxexamples/Example61_MultipleLLMs.java | 7 ++++--- .../semantickernel/semanticfunctions/KernelArguments.java | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example61_MultipleLLMs.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example61_MultipleLLMs.java index 8348ee51..0396d575 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example61_MultipleLLMs.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example61_MultipleLLMs.java @@ -9,6 +9,7 @@ import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIChatCompletion; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionFromPrompt; import com.microsoft.semantickernel.semanticfunctions.PromptTemplateConfig; @@ -73,7 +74,7 @@ public static void runByServiceIdAsync(Kernel kernel, String serviceId) { var prompt = "Hello AI, what can you do for me?"; - KernelFunctionArguments arguments = KernelFunctionArguments.builder().build(); + KernelArguments arguments = KernelArguments.builder().build(); KernelFunction func = KernelFunctionFromPrompt .builder() @@ -104,7 +105,7 @@ public static void runByModelIdAsync(Kernel kernel, String modelId) { .build()) .withOutputVariable("result", "java.lang.String") .build()) - .withArguments(KernelFunctionArguments.builder().build()) + .withArguments(KernelArguments.builder().build()) .block(); System.out.println(result.getResult()); @@ -136,7 +137,7 @@ public static void runByFirstModelIdAsync(Kernel kernel, String... modelIds) { .build(); var result = kernel.invokeAsync(function) - .withArguments(KernelFunctionArguments.builder().build()) + .withArguments(KernelArguments.builder().build()) .block(); System.out.println(result.getResult()); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java index b8100551..5b41afc9 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java @@ -11,6 +11,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -93,7 +94,7 @@ protected KernelArguments(@NonNull KernelArguments arguments) { */ @Nonnull public Map getPromptExecutionSettings() { - return promptExecutionSettings; + return Collections.unmodifiableMap(promptExecutionSettings); } /** From c788b1ce7014cc0418e81a42eb95d3e1c88008a2 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Mon, 14 Apr 2025 23:52:11 -0700 Subject: [PATCH 122/277] Update --- .../samples/plugins/ConversationSummaryPlugin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/plugins/ConversationSummaryPlugin.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/plugins/ConversationSummaryPlugin.java index 1bd16371..0a224c2f 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/plugins/ConversationSummaryPlugin.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/plugins/ConversationSummaryPlugin.java @@ -71,7 +71,7 @@ private static Mono processAsync(KernelFunction func, String inp // The first parameter is the input text. return func.invokeAsync(kernel) .withArguments( - new KernelArguments.Builder() + KernelArguments.builder() .withInput(paragraph) .build()) .withResultType( From c98c39ac3147941aace5e3d6841241e30a81d84c Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 15 Apr 2025 11:44:38 -0700 Subject: [PATCH 123/277] Remove comment --- .../com/microsoft/semantickernel/services/AIServiceSelector.java | 1 - 1 file changed, 1 deletion(-) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIServiceSelector.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIServiceSelector.java index c48fa475..e68e6a69 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIServiceSelector.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIServiceSelector.java @@ -26,7 +26,6 @@ public interface AIServiceSelector { * @return An {@code AIServiceSelection} containing the selected service and associated * PromptExecutionSettings. */ -// @Deprecated @Nullable AIServiceSelection trySelectAIService( Class serviceType, From 7620749f7e912f8581762f9eebf60334d80eeaed Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 15 Apr 2025 16:53:04 -0700 Subject: [PATCH 124/277] Rename field to executionSettings to match old class field name --- .../semanticfunctions/KernelArguments.java | 48 ++++++++++--------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java index 5b41afc9..80ec5972 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java @@ -34,7 +34,7 @@ public class KernelArguments implements Map> { public static final String MAIN_KEY = "input"; protected final CaseInsensitiveMap> variables; - protected final Map promptExecutionSettings; + protected final Map executionSettings; /** * Create a new instance of KernelArguments. @@ -43,17 +43,17 @@ public class KernelArguments implements Map> { */ protected KernelArguments( @Nullable Map> variables, - @Nullable Map promptExecutionSettings) { + @Nullable Map executionSettings) { if (variables == null) { this.variables = new CaseInsensitiveMap<>(); } else { this.variables = new CaseInsensitiveMap<>(variables); } - if (promptExecutionSettings == null) { - this.promptExecutionSettings = new HashMap<>(); + if (executionSettings == null) { + this.executionSettings = new HashMap<>(); } else { - this.promptExecutionSettings = new HashMap<>(promptExecutionSettings); + this.executionSettings = new HashMap<>(executionSettings); } } @@ -63,10 +63,8 @@ protected KernelArguments( * @param content The content to use for the function invocation. */ protected KernelArguments(@NonNull ContextVariable content) { - this.variables = new CaseInsensitiveMap<>(); + this(); this.variables.put(MAIN_KEY, content); - - this.promptExecutionSettings = new HashMap<>(); } /** @@ -74,7 +72,7 @@ protected KernelArguments(@NonNull ContextVariable content) { */ protected KernelArguments() { this.variables = new CaseInsensitiveMap<>(); - this.promptExecutionSettings = new HashMap<>(); + this.executionSettings = new HashMap<>(); } /** @@ -84,7 +82,7 @@ protected KernelArguments() { */ protected KernelArguments(@NonNull KernelArguments arguments) { this.variables = new CaseInsensitiveMap<>(arguments.variables); - this.promptExecutionSettings = new HashMap<>(arguments.promptExecutionSettings); + this.executionSettings = new HashMap<>(arguments.executionSettings); } /** @@ -93,8 +91,8 @@ protected KernelArguments(@NonNull KernelArguments arguments) { * @return prompt execution settings */ @Nonnull - public Map getPromptExecutionSettings() { - return Collections.unmodifiableMap(promptExecutionSettings); + public Map getExecutionSettings() { + return Collections.unmodifiableMap(executionSettings); } /** @@ -233,7 +231,7 @@ public Set>> entrySet() { * @return copy of the current instance */ public KernelArguments copy() { - return new KernelArguments(variables, promptExecutionSettings); + return new KernelArguments(variables, executionSettings); } /** @@ -253,13 +251,13 @@ public static class Builder implements SemanticKernel private final Function constructor; private final Map> variables; - private final Map promptExecutionSettings; + private final Map executionSettings; protected Builder(Function constructor) { this.constructor = constructor; this.variables = new HashMap<>(); - this.promptExecutionSettings = new HashMap<>(); + this.executionSettings = new HashMap<>(); } /** @@ -365,24 +363,28 @@ public Builder withVariable(String key, T value, /** * Set prompt execution settings * - * @param promptExecutionSettings Prompt execution settings + * @param executionSettings Execution settings * @return {$code this} Builder for fluent coding */ - public Builder withPromptExecutionSettings(Map promptExecutionSettings) { - return withPromptExecutionSettings(new ArrayList<>(promptExecutionSettings.values())); + public Builder withExecutionSettings(Map executionSettings) { + return withExecutionSettings(new ArrayList<>(executionSettings.values())); } /** * Set prompt execution settings * - * @param promptExecutionSettings Prompt execution settings + * @param executionSettings Execution settings * @return {$code this} Builder for fluent coding */ - public Builder withPromptExecutionSettings(List promptExecutionSettings) { - for (PromptExecutionSettings settings : promptExecutionSettings) { + public Builder withExecutionSettings(List executionSettings) { + if (executionSettings == null) { + return this; + } + + for (PromptExecutionSettings settings : executionSettings) { String serviceId = settings.getServiceId(); - if (this.promptExecutionSettings.containsKey(serviceId)) { + if (this.executionSettings.containsKey(serviceId)) { if (serviceId.equals(PromptExecutionSettings.DEFAULT_SERVICE_ID)) { throw new SKException( String.format( @@ -404,7 +406,7 @@ public Builder withPromptExecutionSettings(List prom @Override public U build() { - KernelArguments arguments = new KernelArguments(variables, promptExecutionSettings); + KernelArguments arguments = new KernelArguments(variables, executionSettings); return constructor.apply(arguments); } } From 44b27cdf1c1af071ec2eba2ca30f0d48621afc80 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 17 Apr 2025 09:00:39 -0700 Subject: [PATCH 125/277] Add AIServiceSelector.trySelectAIService(Class,KernelArguments) and related updates --- .../com/microsoft/semantickernel/Kernel.java | 27 +++++++++++++-- .../semanticfunctions/KernelArguments.java | 10 ++++++ .../KernelFunctionFromPrompt.java | 8 +++-- .../services/AIServiceSelector.java | 26 ++++++++++++++- .../services/BaseAIServiceSelector.java | 33 +++++++++++++++++++ .../services/OrderedAIServiceSelector.java | 16 +++++---- 6 files changed, 107 insertions(+), 13 deletions(-) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/Kernel.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/Kernel.java index cf38760f..d8059ef7 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/Kernel.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/Kernel.java @@ -302,14 +302,12 @@ public AIServiceSelector getServiceSelector() { * @param clazz The class of the service to get. * @return The service of the specified type from the kernel. * @throws ServiceNotFoundException if the service is not found. - * @see com.microsoft.semantickernel.services.AIServiceSelector#trySelectAIService(Class, - * KernelFunction, KernelArguments) + * @see com.microsoft.semantickernel.services.AIServiceSelector#trySelectAIService(Class, KernelArguments) */ public T getService(Class clazz) throws ServiceNotFoundException { AIServiceSelection selector = serviceSelector .trySelectAIService( clazz, - null, null); if (selector == null) { @@ -319,6 +317,29 @@ public T getService(Class clazz) throws ServiceNotFound return selector.getService(); } + /** + * Get the service of the specified type from the kernel. + * + * @param The type of the service to get. + * @param clazz The class of the service to get. + * @param args The arguments to help select the service to get. + * @return The service of the specified type from the kernel. + * @throws ServiceNotFoundException if the service is not found. + * @see com.microsoft.semantickernel.services.AIServiceSelector#trySelectAIService(Class, KernelArguments) + */ + public T getService(Class clazz, KernelArguments args) throws ServiceNotFoundException { + AIServiceSelection selector = serviceSelector + .trySelectAIService( + clazz, + args); + + if (selector == null) { + throw new ServiceNotFoundException("Unable to find service of type " + clazz.getName()); + } + + return selector.getService(); + } + /** * A fluent builder for creating a new instance of {@code Kernel}. */ diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java index 80ec5972..8f731915 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java @@ -360,6 +360,16 @@ public Builder withVariable(String key, T value, value)); } + /** + * Set prompt execution settings + * + * @param executionSettings Execution settings + * @return {$code this} Builder for fluent coding + */ + public Builder withExecutionSettings(PromptExecutionSettings executionSettings) { + return withExecutionSettings(Collections.singletonList(executionSettings)); + } + /** * Set prompt execution settings * diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java index 1d12f961..7671e9e1 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java @@ -133,13 +133,17 @@ private Flux> invokeInternalAsync( FunctionInvokingEvent updateArguments = kernelHooks .executeHooks(new FunctionInvokingEvent(this, args)); - args = updateArguments.getArguments(); + + args = KernelArguments.builder() + .withVariables(updateArguments.getArguments()) + .withExecutionSettings( + this.getExecutionSettings()) + .build(); AIServiceSelection aiServiceSelection = kernel .getServiceSelector() .trySelectAIService( TextAIService.class, - this, args); AIService client = aiServiceSelection != null ? aiServiceSelection.getService() diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIServiceSelector.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIServiceSelector.java index e68e6a69..9116add4 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIServiceSelector.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIServiceSelector.java @@ -20,15 +20,39 @@ public interface AIServiceSelector { * @param serviceType The type of service to select. This must be the same type with which the * service was registered in the {@link AIServiceSelection} * @param function The KernelFunction to use to select the service, or {@code null}. - * @param arguments The KernelFunctionArguments to use to select the service, or + * @param arguments The KernelArguments to use to select the service, or * {@code null}. * @param The type of service to select. * @return An {@code AIServiceSelection} containing the selected service and associated * PromptExecutionSettings. + * + * @deprecated Use {@link #trySelectAIService(Class, KernelArguments)} instead. */ + @Deprecated @Nullable AIServiceSelection trySelectAIService( Class serviceType, @Nullable KernelFunction function, @Nullable KernelArguments arguments); + + /** + * Resolves an {@link AIService} and associated and + * {@link com.microsoft.semantickernel.orchestration.PromptExecutionSettings} based on the + * associated {@link KernelFunction} and {@link KernelArguments}. + * + * @param serviceType The type of service to select. This must be the same type with which the + * service was registered in the {@link AIServiceSelection} + * @param arguments The KernelArguments to use to select the service, or + * {@code null}. + * @param The type of service to select. + * @return An {@code AIServiceSelection} containing the selected service and associated + * PromptExecutionSettings. + */ + @Nullable + default AIServiceSelection trySelectAIService( + Class serviceType, + @Nullable KernelArguments arguments) { + throw new UnsupportedOperationException( + "This method is not implemented."); + } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/BaseAIServiceSelector.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/BaseAIServiceSelector.java index c3e42b8e..eb745894 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/BaseAIServiceSelector.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/BaseAIServiceSelector.java @@ -35,6 +35,14 @@ public AIServiceSelection trySelectAIService( return trySelectAIService(serviceType, function, arguments, services); } + @Override + @Nullable + public AIServiceSelection trySelectAIService( + Class serviceType, + @Nullable KernelArguments arguments) { + return trySelectAIService(serviceType, arguments, services); + } + /** * Resolves an {@link AIService} from the {@code services} argument using the specified * {@code function} and {@code arguments} for selection. @@ -47,11 +55,36 @@ public AIServiceSelection trySelectAIService( * @param services The services to select from. * @param The type of service to select. * @return The selected service, or {@code null} if no service could be selected. + * + * @deprecated Implement {@link #trySelectAIService(Class, KernelArguments)} */ + @Deprecated @Nullable protected abstract AIServiceSelection trySelectAIService( Class serviceType, @Nullable KernelFunction function, @Nullable KernelArguments arguments, Map, AIService> services); + + + /** + * Resolves an {@link AIService} from the {@code services} argument using the specified + * {@code function} and {@code arguments} for selection. + * + * @param serviceType The type of service to select. This must be the same type with which the + * service was registered in the {@link AIServiceSelection} + * @param arguments The KernelArguments to use to select the service, or + * {@code null}. + * @param services The services to select from. + * @param The type of service to select. + * @return The selected service, or {@code null} if no service could be selected. + */ + @Nullable + protected AIServiceSelection trySelectAIService( + Class serviceType, + @Nullable KernelArguments arguments, + Map, AIService> services) { + throw new UnsupportedOperationException( + "This method is not implemented."); + } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/OrderedAIServiceSelector.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/OrderedAIServiceSelector.java index 2be73543..d478a41f 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/OrderedAIServiceSelector.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/OrderedAIServiceSelector.java @@ -66,13 +66,15 @@ private static Map settingsFromFunctionSettings return null; } -// @Nullable -// @Override -// public AIServiceSelection trySelectAIService( -// Class serviceType, -// @Nullable KernelArguments arguments) { -// return selectAIService(serviceType, arguments.getPromptExecutionSettings()); -// } + @Nullable + @Override + public AIServiceSelection trySelectAIService( + Class serviceType, + @Nullable KernelArguments arguments, + Map, AIService> services) { + + return selectAIService(serviceType, arguments != null ? arguments.getExecutionSettings() : null); + } @Nullable @Override From bc2c366c83d428fce74114e12f55c827f3f544bd Mon Sep 17 00:00:00 2001 From: psilberk Date: Tue, 22 Apr 2025 09:31:27 -0700 Subject: [PATCH 126/277] Adding Oracle Vector Store --- .../OracleVectorStoreQueryProvider.java | 91 ++++++++ .../memory/VectorStoreWithOracle.java | 210 ++++++++++++++++++ 2 files changed, 301 insertions(+) create mode 100644 data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java create mode 100644 samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java new file mode 100644 index 00000000..f179d0be --- /dev/null +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -0,0 +1,91 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; +import com.microsoft.semantickernel.exceptions.SKException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +import javax.annotation.Nonnull; +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.List; + +public class OracleVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider { + + // This could be removed if super.collectionTable made protected + private final String collectionsTable; + + private OracleVectorStoreQueryProvider(@Nonnull DataSource dataSource, @Nonnull String collectionsTable, @Nonnull String prefixForCollectionTables) { + super(dataSource, collectionsTable, prefixForCollectionTables); + this.collectionsTable = collectionsTable; + } + + @Override + public void prepareVectorStore() { + String createCollectionsTable = formatQuery( + "CREATE TABLE IF NOT EXISTS %s (collectionId VARCHAR(255) PRIMARY KEY)", + validateSQLidentifier(collectionsTable)); + + try (Connection connection = dataSource.getConnection(); + PreparedStatement createTable = connection.prepareStatement(createCollectionsTable)) { + createTable.execute(); + } catch (SQLException e) { + throw new SKException("Failed to prepare vector store", e); + } + } + + @Override + public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { + + // Using hsqldb impl + + + super.upsertRecords(collectionName, records, recordDefinition, options); + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder + extends JDBCVectorStoreQueryProvider.Builder { + + private DataSource dataSource; + private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; + private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; + + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withDataSource(DataSource dataSource) { + this.dataSource = dataSource; + return this; + } + + /** + * Sets the collections table name. + * @param collectionsTable the collections table name + * @return the builder + */ + public Builder withCollectionsTable(String collectionsTable) { + this.collectionsTable = validateSQLidentifier(collectionsTable); + return this; + } + + /** + * Sets the prefix for collection tables. + * @param prefixForCollectionTables the prefix for collection tables + * @return the builder + */ + public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { + this.prefixForCollectionTables = validateSQLidentifier(prefixForCollectionTables); + return this; + } + + @Override + public OracleVectorStoreQueryProvider build() { + return new OracleVectorStoreQueryProvider(dataSource, collectionsTable, prefixForCollectionTables); + } + } +} \ No newline at end of file diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java new file mode 100644 index 00000000..9d142507 --- /dev/null +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java @@ -0,0 +1,210 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.syntaxexamples.memory; + +import com.azure.ai.openai.OpenAIAsyncClient; +import com.azure.ai.openai.OpenAIClientBuilder; +import com.azure.core.credential.AzureKeyCredential; +import com.azure.core.credential.KeyCredential; +import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; + +import java.nio.charset.StandardCharsets; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.postgresql.ds.PGSimpleDataSource; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +public class VectorStoreWithOracle { + + static class GitHubFile { + @VectorStoreRecordKey + private final String id; + @VectorStoreRecordData + private final String description; + @VectorStoreRecordData + private final String link; + @VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, distanceFunction = DistanceFunction.COSINE_DISTANCE) + private final List embedding; + + public GitHubFile() { + this(null, null, null, Collections.emptyList()); + } + + public GitHubFile( + String id, + String description, + String link, + List embedding) { + this.id = id; + this.description = description; + this.link = link; + this.embedding = embedding; + } + + public String getId() { + return id; + } + + public String getDescription() { + return description; + } + + public String getLink() { + return link; + } + + public List getEmbedding() { + return embedding; + } + + static String encodeId(String realId) { + byte[] bytes = Base64.getUrlEncoder().encode(realId.getBytes(StandardCharsets.UTF_8)); + return new String(bytes, StandardCharsets.UTF_8); + } + } + + // Run a PostgreSQL server with: + // docker run -d --name pgvector-container -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=root -e POSTGRES_DB=sk -p 5432:5432 pgvector/pgvector:pg17 + + public static void main(String[] args) throws SQLException { + System.out.println("=============================================================="); + System.out.println("============== Oracle Vector Store Example ==================="); + System.out.println("=============================================================="); + + OpenAIAsyncClient client; + + if (AZURE_CLIENT_KEY != null) { + client = new OpenAIClientBuilder() + .credential(new AzureKeyCredential(AZURE_CLIENT_KEY)) + .endpoint(CLIENT_ENDPOINT) + .buildAsyncClient(); + + } else { + client = new OpenAIClientBuilder() + .credential(new KeyCredential(CLIENT_KEY)) + .buildAsyncClient(); + } + + // Create an OpenAI text embedding generation service + var embeddingGeneration = OpenAITextEmbeddingGenerationService.builder() + .withOpenAIAsyncClient(client) + .withModelId(MODEL_ID) + .withDimensions(EMBEDDING_DIMENSIONS) + .build(); + + storeAndSearch(embeddingGeneration); + } + + public static void storeAndSearch(OpenAITextEmbeddingGenerationService embeddingGeneration) { + // Configure the data source + PGSimpleDataSource dataSource = new PGSimpleDataSource(); + dataSource.setUrl("jdbc:postgresql://localhost:5432/sk"); + dataSource.setUser("postgres"); + dataSource.setPassword("root"); + + // Build a query provider + // Other available query providers are PostgreSQLVectorStoreQueryProvider and SQLiteVectorStoreQueryProvider + var queryProvider = PostgreSQLVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build(); + + // Build a vector store + var jdbcVectorStore = JDBCVectorStore.builder() + .withDataSource(dataSource) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + // Set up the record collection to use + String collectionName = "skgithubfiles"; + var collection = jdbcVectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(GitHubFile.class) + .build()); + + // Create collection if it does not exist and store data + collection + .createCollectionIfNotExistsAsync() + .then(storeData(collection, embeddingGeneration, sampleData())) + .block(); + + // Search for results + var results = search("How to get started", collection, embeddingGeneration).block(); + + if (results == null || results.getTotalCount() == 0) { + System.out.println("No search results found."); + return; + } + var searchResult = results.getResults().get(0); + System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", + searchResult.getScore(), searchResult.getRecord().link, + searchResult.getRecord().description); + } + + private static Mono> search( + String searchText, + VectorStoreRecordCollection recordCollection, + OpenAITextEmbeddingGenerationService embeddingGeneration) { + // Generate embeddings for the search text and search for the closest records + return embeddingGeneration.generateEmbeddingAsync(searchText) + .flatMap(r -> recordCollection.searchAsync(r.getVector(), null)); + } + + private static Mono> storeData( + VectorStoreRecordCollection recordStore, + OpenAITextEmbeddingGenerationService embeddingGeneration, + Map data) { + + return Flux.fromIterable(data.entrySet()) + .flatMap(entry -> { + System.out.println("Save '" + entry.getKey() + "' to memory."); + + // Generate embeddings for the data and store it + return embeddingGeneration + .generateEmbeddingsAsync(Collections.singletonList(entry.getValue())) + .flatMap(embeddings -> { + GitHubFile gitHubFile = new GitHubFile( + GitHubFile.encodeId(entry.getKey()), + entry.getValue(), + entry.getKey(), + embeddings.get(0).getVector()); + return recordStore.upsertAsync(gitHubFile, null); + }); + }) + .collectList(); + } + + private static Map sampleData() { + return Arrays.stream(new String[][] { + { "https://github.com/microsoft/semantic-kernel/blob/main/README.md", + "README: Installation, getting started with Semantic Kernel, and how to contribute" }, + { "https://github.com/microsoft/semantic-kernel/blob/main/samples/notebooks/dotnet/02-running-prompts-from-file.ipynb", + "Jupyter notebook describing how to pass prompts from a file to a semantic skill or function" }, + { "https://github.com/microsoft/semantic-kernel/tree/main/samples/skills/ChatSkill/ChatGPT", + "Sample demonstrating how to create a chat skill interfacing with ChatGPT" }, + { "https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel/Memory/VolatileMemoryStore.cs", + "C# class that defines a volatile embedding store" }, + { "https://github.com/microsoft/semantic-kernel/blob/main/samples/dotnet/KernelHttpServer/README.md", + "README: How to set up a Semantic Kernel Service API using Azure Function Runtime v4" }, + { "https://github.com/microsoft/semantic-kernel/blob/main/samples/apps/chat-summary-webapp-react/README.md", + "README: README associated with a sample chat summary react-based webapp" }, + }).collect(Collectors.toMap(element -> element[0], element -> element[1])); + } +} From 5859394d4bbde27e2a0d97fc09f2e9de8e34d8b4 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 24 Apr 2025 00:46:49 -0700 Subject: [PATCH 127/277] Updates to service selector --- .../services/AIServiceSelector.java | 3 --- .../services/BaseAIServiceSelector.java | 5 +--- .../services/OrderedAIServiceSelector.java | 27 +++---------------- 3 files changed, 5 insertions(+), 30 deletions(-) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIServiceSelector.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIServiceSelector.java index 9116add4..ac95f7d3 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIServiceSelector.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIServiceSelector.java @@ -25,10 +25,7 @@ public interface AIServiceSelector { * @param The type of service to select. * @return An {@code AIServiceSelection} containing the selected service and associated * PromptExecutionSettings. - * - * @deprecated Use {@link #trySelectAIService(Class, KernelArguments)} instead. */ - @Deprecated @Nullable AIServiceSelection trySelectAIService( Class serviceType, diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/BaseAIServiceSelector.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/BaseAIServiceSelector.java index eb745894..03ecdd78 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/BaseAIServiceSelector.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/BaseAIServiceSelector.java @@ -56,9 +56,7 @@ public AIServiceSelection trySelectAIService( * @param The type of service to select. * @return The selected service, or {@code null} if no service could be selected. * - * @deprecated Implement {@link #trySelectAIService(Class, KernelArguments)} */ - @Deprecated @Nullable protected abstract AIServiceSelection trySelectAIService( Class serviceType, @@ -84,7 +82,6 @@ protected AIServiceSelection trySelectAIService( Class serviceType, @Nullable KernelArguments arguments, Map, AIService> services) { - throw new UnsupportedOperationException( - "This method is not implemented."); + return trySelectAIService(serviceType, null, arguments, services); } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/OrderedAIServiceSelector.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/OrderedAIServiceSelector.java index d478a41f..6b04fe2b 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/OrderedAIServiceSelector.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/OrderedAIServiceSelector.java @@ -57,25 +57,6 @@ private static AIServiceSelection castServiceSelection( } } - @Nullable - private static Map settingsFromFunctionSettings( - @Nullable KernelFunction function) { - if (function != null) { - return function.getExecutionSettings(); - } - return null; - } - - @Nullable - @Override - public AIServiceSelection trySelectAIService( - Class serviceType, - @Nullable KernelArguments arguments, - Map, AIService> services) { - - return selectAIService(serviceType, arguments != null ? arguments.getExecutionSettings() : null); - } - @Nullable @Override public AIServiceSelection trySelectAIService( @@ -84,11 +65,11 @@ public AIServiceSelection trySelectAIService( @Nullable KernelArguments arguments, Map, AIService> services) { - // Allow the execution settings from the kernel arguments to take precedence - Map executionSettings = settingsFromFunctionSettings( - function); + if (function == null) { + return selectAIService(serviceType, arguments != null ? arguments.getExecutionSettings() : null); + } - return selectAIService(serviceType, executionSettings); + return selectAIService(serviceType, function.getExecutionSettings()); } From 393d987f3b31093a4f86ab08738474480f3c1b64 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 24 Apr 2025 01:16:08 -0700 Subject: [PATCH 128/277] Remove unnecessary line --- .../com/microsoft/semantickernel/tests/ResponseSchemaTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ResponseSchemaTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ResponseSchemaTest.java index 77d4b4a3..ef792e2b 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ResponseSchemaTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ResponseSchemaTest.java @@ -160,8 +160,6 @@ private static void verifyCalled(OpenAIAsyncClient client, String expected) { writer, new JsonOptions() ); - JsonWriter format = chatCompletionsOptions.getResponseFormat() - .toJson(jsonWriter); jsonWriter.flush(); writer.flush(); From 17a920eb514ed50364b3b0bf442ffff63015f79b Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 24 Apr 2025 02:10:09 -0700 Subject: [PATCH 129/277] Fixes --- .../microsoft/semantickernel/tests/ResponseSchemaTest.java | 4 ++++ .../semantickernel/semanticfunctions/KernelArguments.java | 2 ++ .../semanticfunctions/KernelFunctionFromPrompt.java | 7 +++---- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ResponseSchemaTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ResponseSchemaTest.java index ef792e2b..4ec67774 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ResponseSchemaTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ResponseSchemaTest.java @@ -160,15 +160,19 @@ private static void verifyCalled(OpenAIAsyncClient client, String expected) { writer, new JsonOptions() ); + JsonWriter format = chatCompletionsOptions.getResponseFormat() + .toJson(jsonWriter); jsonWriter.flush(); writer.flush(); String json = String.valueOf(writer.getBuffer()) .replaceAll("\n", "") + .replaceAll("\r", "") .replaceAll(" +", ""); String expectedClean = expected .stripIndent() .replaceAll("\n", "") + .replaceAll("\r", "") .replaceAll(" +", ""); return json.equals(expectedClean); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java index 8f731915..e4fb8319 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java @@ -409,6 +409,8 @@ public Builder withExecutionSettings(List executionS serviceId) ); } + + this.executionSettings.put(serviceId, settings); } return this; diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java index 7671e9e1..2ea90974 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java @@ -131,13 +131,12 @@ private Flux> invokeInternalAsync( LOGGER.info(SemanticKernelResources.getString("rendered.prompt"), prompt); - FunctionInvokingEvent updateArguments = kernelHooks + FunctionInvokingEvent invokingEvent = kernelHooks .executeHooks(new FunctionInvokingEvent(this, args)); args = KernelArguments.builder() - .withVariables(updateArguments.getArguments()) - .withExecutionSettings( - this.getExecutionSettings()) + .withVariables(invokingEvent.getArguments()) + .withExecutionSettings(this.getExecutionSettings()) .build(); AIServiceSelection aiServiceSelection = kernel From ed651d61d498c61ae83f19f2d9bd1537b1c358ac Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 17 Apr 2025 09:05:00 -0700 Subject: [PATCH 130/277] Add draft changes for agents --- agents/semantickernel-agents-core/pom.xml | 25 ++ .../chatcompletion/ChatCompletionAgent.java | 238 ++++++++++++++++++ .../ChatHistoryAgentThread.java | 64 +++++ agents/semantickernel-agents-openai/pom.xml | 26 ++ pom.xml | 2 + .../semantickernel-syntax-examples/pom.xml | 5 + .../samples/plugins/github/GitHubModel.java | 219 ++++++++++++++++ .../samples/plugins/github/GitHubPlugin.java | 165 ++++++++++++ .../agents/CompletionsAgent.java | 145 +++++++++++ .../semantickernel/agents/Agent.java | 60 +++++ .../agents/AgentInvokeOptions.java | 126 ++++++++++ .../agents/AgentResponseItem.java | 19 ++ .../semantickernel/agents/AgentThread.java | 45 ++++ .../agents/BaseAgentThread.java | 23 ++ .../semantickernel/agents/KernelAgent.java | 139 ++++++++++ .../services/chatcompletion/ChatHistory.java | 7 + semantickernel-bom/pom.xml | 6 + 17 files changed, 1314 insertions(+) create mode 100644 agents/semantickernel-agents-core/pom.xml create mode 100644 agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java create mode 100644 agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatHistoryAgentThread.java create mode 100644 agents/semantickernel-agents-openai/pom.xml create mode 100644 samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubModel.java create mode 100644 samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubPlugin.java create mode 100644 samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionsAgent.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/Agent.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentResponseItem.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentThread.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/BaseAgentThread.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java diff --git a/agents/semantickernel-agents-core/pom.xml b/agents/semantickernel-agents-core/pom.xml new file mode 100644 index 00000000..4eec1b76 --- /dev/null +++ b/agents/semantickernel-agents-core/pom.xml @@ -0,0 +1,25 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-SNAPSHOT + ../../pom.xml + + + semantickernel-agents-core + + Semantic Kernel Chat Completion Agent + Chat Completion Agent for Semantic Kernel + + + + com.microsoft.semantic-kernel + semantickernel-api + + + + \ No newline at end of file diff --git a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java new file mode 100644 index 00000000..c8c29007 --- /dev/null +++ b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java @@ -0,0 +1,238 @@ +package com.microsoft.semantickernel.agents.chatcompletion; + +import com.microsoft.semantickernel.Kernel; +import com.microsoft.semantickernel.agents.AgentInvokeOptions; +import com.microsoft.semantickernel.agents.AgentResponseItem; +import com.microsoft.semantickernel.agents.AgentThread; +import com.microsoft.semantickernel.agents.KernelAgent; +import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.contextvariables.ContextVariable; +import com.microsoft.semantickernel.orchestration.InvocationContext; +import com.microsoft.semantickernel.orchestration.InvocationReturnMode; +import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; +import com.microsoft.semantickernel.orchestration.ToolCallBehavior; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; +import com.microsoft.semantickernel.semanticfunctions.PromptTemplate; +import com.microsoft.semantickernel.semanticfunctions.PromptTemplateConfig; +import com.microsoft.semantickernel.semanticfunctions.PromptTemplateFactory; +import com.microsoft.semantickernel.services.ServiceNotFoundException; +import com.microsoft.semantickernel.services.chatcompletion.AuthorRole; +import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; +import com.microsoft.semantickernel.services.chatcompletion.ChatHistory; +import com.microsoft.semantickernel.services.chatcompletion.ChatMessageContent; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class ChatCompletionAgent extends KernelAgent { + + ChatCompletionAgent( + String id, + String name, + String description, + Kernel kernel, + KernelArguments kernelArguments, + InvocationContext context, + String instructions, + PromptTemplate template + ) { + super( + id, + name, + description, + kernel, + kernelArguments, + context, + instructions, + template + ); + } + + /** + * Invoke the agent with the given chat history. + * + * @param messages The chat history to process + * @param thread The agent thread to use + * @param options The options for invoking the agent + * @return A Mono containing the agent response + */ + @Override + public Mono>>> invokeAsync(List> messages, AgentThread thread, AgentInvokeOptions options) { + + Mono chatHistoryFromThread = this.ensureThreadExistsAsync(messages, thread, ChatHistoryAgentThread::new) + .cast(ChatHistoryAgentThread.class) + .map(ChatHistoryAgentThread::getChatHistory) + .flatMap(threadChatHistory -> { + return Mono.just(new ChatHistory(threadChatHistory.getMessages())); + }); + + + Mono>> updatedChatHistory = chatHistoryFromThread.flatMap( + chatHistory -> internalInvokeAsync( + this.getName(), + chatHistory, + options + ) + ); + + return updatedChatHistory.flatMap(chatMessageContents -> { + return Flux.fromIterable(chatMessageContents) + .concatMap(chatMessageContent -> this.notifyThreadOfNewMessageAsync(thread, chatMessageContent)) + .then(Mono.just(chatMessageContents)); // return the original list + }).flatMap(chatMessageContents -> { + return Mono.just(chatMessageContents.stream() + .map(chatMessageContent -> { + return new AgentResponseItem>( + chatMessageContent, + thread); + }).collect(Collectors.toList())); + }); + } + + private Mono>> internalInvokeAsync( + String agentName, + ChatHistory history, + AgentInvokeOptions options + ) { + final Kernel kernel = options.getKernel() != null ? options.getKernel() : this.kernel; + final KernelArguments arguments = mergeArguments(options.getKernelArguments()); + final String additionalInstructions = options.getAdditionalInstructions(); + final InvocationContext invocationContext = options.getInvocationContext() != null ? options.getInvocationContext() : this.invocationContext; + + try { + ChatCompletionService chatCompletionService = kernel.getService(ChatCompletionService.class, arguments); + + PromptExecutionSettings executionSettings = invocationContext.getPromptExecutionSettings() != null + ? invocationContext.getPromptExecutionSettings() + : kernelArguments.getExecutionSettings().get(chatCompletionService.getServiceId()); + + final InvocationContext newMessagesContext = InvocationContext.builder() + .withPromptExecutionSettings(executionSettings) + .withToolCallBehavior(invocationContext.getToolCallBehavior()) + .withTelemetry(invocationContext.getTelemetry()) + .withContextVariableConverter(invocationContext.getContextVariableTypes()) + .withKernelHooks(invocationContext.getKernelHooks()) + .withReturnMode(InvocationReturnMode.FULL_HISTORY) + .build(); + + return formatInstructionsAsync(kernel, arguments, newMessagesContext).flatMap( + instructions -> { + // Create a new chat history with the instructions + ChatHistory chat = new ChatHistory( + instructions + ); + + // Add agent additional instructions + if (additionalInstructions != null) { + chat.addMessage(new ChatMessageContent<>( + AuthorRole.SYSTEM, + additionalInstructions + )); + } + + chat.addAll(history); + int previousHistorySize = chat.getMessages().size(); + + return chatCompletionService.getChatMessageContentsAsync(chat, kernel, newMessagesContext) + .map(chatMessageContents -> { + return chatMessageContents.subList( + previousHistorySize, + chatMessageContents.size()); + }); + } + ); + + + } catch (ServiceNotFoundException e) { + throw new RuntimeException(e); + } + } + + /** + * Builder for creating instances of ChatCompletionAgent. + */ + public static Builder builder() { + return new Builder(); + } + + public static class Builder implements SemanticKernelBuilder { + private String id; + private String name; + private String description; + private Kernel kernel; + private KernelArguments KernelArguments; + private InvocationContext invocationContext; + private String instructions; + private PromptTemplate template; + + public Builder withId(String id) { + this.id = id; + return this; + } + + public Builder withName(String name) { + this.name = name; + return this; + } + + public Builder withDescription(String description) { + this.description = description; + return this; + } + + public Builder withKernel(Kernel kernel) { + this.kernel = kernel; + return this; + } + + public Builder withKernelArguments(KernelArguments KernelArguments) { + this.KernelArguments = KernelArguments; + return this; + } + + public Builder withInstructions(String instructions) { + this.instructions = instructions; + return this; + } + + public Builder withInvocationContext(InvocationContext invocationContext) { + this.invocationContext = invocationContext; + return this; + } + + public Builder withTemplate(PromptTemplate template) { + this.template = template; + return this; + } + + public ChatCompletionAgent build() { + return new ChatCompletionAgent( + id, + name, + description, + kernel, + KernelArguments, + invocationContext, + instructions, + template + ); + } + + public ChatCompletionAgent build(PromptTemplateConfig promptTemplateConfig, PromptTemplateFactory promptTemplateFactory) { + return new ChatCompletionAgent( + id, + name, + description, + kernel, + KernelArguments, + invocationContext, + promptTemplateConfig.getTemplate(), + promptTemplateFactory.tryCreate(promptTemplateConfig) + ); + } + } +} diff --git a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatHistoryAgentThread.java b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatHistoryAgentThread.java new file mode 100644 index 00000000..b000ca9f --- /dev/null +++ b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatHistoryAgentThread.java @@ -0,0 +1,64 @@ +package com.microsoft.semantickernel.agents.chatcompletion; + +import com.microsoft.semantickernel.agents.AgentThread; +import com.microsoft.semantickernel.agents.BaseAgentThread; +import com.microsoft.semantickernel.services.chatcompletion.ChatHistory; +import com.microsoft.semantickernel.services.chatcompletion.ChatMessageContent; +import reactor.core.publisher.Mono; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; +import java.util.UUID; + +public class ChatHistoryAgentThread extends BaseAgentThread { + private ChatHistory chatHistory; + + public ChatHistoryAgentThread() { + } + + /** + * Constructor for com.microsoft.semantickernel.agents.chatcompletion.ChatHistoryAgentThread. + * + * @param id The ID of the thread. + * @param chatHistory The chat history. + */ + public ChatHistoryAgentThread(String id, @Nullable ChatHistory chatHistory) { + super(id); + this.chatHistory = chatHistory; + } + + /** + * Get the chat history. + * + * @return The chat history. + */ + public ChatHistory getChatHistory() { + return chatHistory; + } + + @Override + public Mono createAsync() { + if (this.id == null) { + this.id = UUID.randomUUID().toString(); + chatHistory = new ChatHistory(); + } + return Mono.just(id); + } + + @Override + public Mono deleteAsync() { + return Mono.fromRunnable(chatHistory::clear); + } + + @Override + public Mono onNewMessageAsync(ChatMessageContent newMessage) { + return Mono.fromRunnable(() -> { + chatHistory.addMessage(newMessage); + }); + } + + public List> getMessages() { + return chatHistory.getMessages(); + } +} diff --git a/agents/semantickernel-agents-openai/pom.xml b/agents/semantickernel-agents-openai/pom.xml new file mode 100644 index 00000000..815a0d5b --- /dev/null +++ b/agents/semantickernel-agents-openai/pom.xml @@ -0,0 +1,26 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-SNAPSHOT + + + semantickernel-agents-openai + + + 17 + 17 + UTF-8 + + + + com.microsoft.semantic-kernel + semantickernel-api + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 30963f54..f1536e87 100644 --- a/pom.xml +++ b/pom.xml @@ -77,6 +77,8 @@ data/semantickernel-data-azureaisearch data/semantickernel-data-jdbc data/semantickernel-data-redis + agents/semantickernel-agents-openai + agents/semantickernel-agents-core diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 1f6bc181..3ccc114c 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -49,6 +49,11 @@ semantickernel-data-redis + + com.microsoft.semantic-kernel + semantickernel-agents-core + + com.microsoft.semantic-kernel semantickernel-experimental diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubModel.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubModel.java new file mode 100644 index 00000000..180ec8ed --- /dev/null +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubModel.java @@ -0,0 +1,219 @@ +package com.microsoft.semantickernel.samples.plugins.github; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +public abstract class GitHubModel { + public final static ObjectMapper objectMapper = new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + @Override + public String toString() { + try { + return objectMapper.writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + public static class User extends GitHubModel { + @JsonProperty("login") + private String login; + @JsonProperty("id") + private long id; + @JsonProperty("name") + private String name; + @JsonProperty("company") + private String company; + @JsonProperty("html_url") + private String url; + @JsonCreator + public User(@JsonProperty("login") String login, + @JsonProperty("id") long id, + @JsonProperty("name") String name, + @JsonProperty("company") String company, + @JsonProperty("html_url") String url) { + this.login = login; + this.id = id; + this.name = name; + this.company = company; + this.url = url; + } + + public String getLogin() { + return login; + } + public long getId() { + return id; + } + public String getName() { + return name; + } + public String getCompany() { + return company; + } + public String getUrl() { + return url; + } + } + + public static class Repository extends GitHubModel { + @JsonProperty("id") + private long id; + @JsonProperty("full_name") + private String name; + @JsonProperty("description") + private String description; + @JsonProperty("html_url") + private String url; + @JsonCreator + public Repository(@JsonProperty("id") long id, + @JsonProperty("full_name") String name, + @JsonProperty("description") String description, + @JsonProperty("html_url") String url) { + this.id = id; + this.name = name; + this.description = description; + this.url = url; + } + + public long getId() { + return id; + } + public String getName() { + return name; + } + public String getDescription() { + return description; + } + public String getUrl() { + return url; + } + + @Override + public String toString() { + try { + return objectMapper.writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + } + + public static class Issue extends GitHubModel { + @JsonProperty("id") + private long id; + @JsonProperty("number") + private long number; + @JsonProperty("title") + private String title; + @JsonProperty("state") + private String state; + @JsonProperty("html_url") + private String url; + @JsonProperty("labels") + private Label[] labels; + @JsonProperty("created_at") + private String createdAt; + @JsonProperty("closed_at") + private String closedAt; + + @JsonCreator + public Issue(@JsonProperty("id") long id, + @JsonProperty("number") long number, + @JsonProperty("title") String title, + @JsonProperty("state") String state, + @JsonProperty("html_url") String url, + @JsonProperty("labels") Label[] labels, + @JsonProperty("created_at") String createdAt, + @JsonProperty("closed_at") String closedAt) { + this.id = id; + this.number = number; + this.title = title; + this.state = state; + this.url = url; + this.labels = labels; + this.createdAt = createdAt; + this.closedAt = closedAt; + } + + public long getId() { + return id; + } + public long getNumber() { + return number; + } + public String getTitle() { + return title; + } + public String getState() { + return state; + } + public String getUrl() { + return url; + } + public Label[] getLabels() { + return labels; + } + public String getCreatedAt() { + return createdAt; + } + public String getClosedAt() { + return closedAt; + } + } + + public static class IssueDetail extends Issue { + @JsonProperty("body") + private String body; + + @JsonCreator + public IssueDetail(@JsonProperty("id") long id, + @JsonProperty("number") long number, + @JsonProperty("title") String title, + @JsonProperty("state") String state, + @JsonProperty("html_url") String url, + @JsonProperty("labels") Label[] labels, + @JsonProperty("created_at") String createdAt, + @JsonProperty("closed_at") String closedAt, + @JsonProperty("body") String body) { + super(id, number, title, state, url, labels, createdAt, closedAt); + this.body = body; + } + + public String getBody() { + return body; + } + } + + public static class Label extends GitHubModel { + @JsonProperty("id") + private long id; + @JsonProperty("name") + private String name; + @JsonProperty("description") + private String description; + + @JsonCreator + public Label(@JsonProperty("id") long id, + @JsonProperty("name") String name, + @JsonProperty("description") String description) { + this.id = id; + this.name = name; + this.description = description; + } + + public long getId() { + return id; + } + public String getName() { + return name; + } + public String getDescription() { + return description; + } + } +} diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubPlugin.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubPlugin.java new file mode 100644 index 00000000..2d1fb444 --- /dev/null +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubPlugin.java @@ -0,0 +1,165 @@ +package com.microsoft.semantickernel.samples.plugins.github; + +import reactor.core.publisher.Mono; +import reactor.netty.http.client.HttpClient; +import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; +import com.microsoft.semantickernel.semanticfunctions.annotations.KernelFunctionParameter; + +import java.io.IOException; +import java.util.List; + +public class GitHubPlugin { + public static final String baseUrl = "https://api.github.com"; + private final String token; + + public GitHubPlugin(String token) { + this.token = token; + } + + @DefineKernelFunction(name = "get_user_info", description = "Get user information from GitHub", + returnType = "com.microsoft.semantickernel.samples.plugins.github.GitHubModel$User") + public Mono getUserProfileAsync() { + HttpClient client = createClient(); + + return makeRequestAsync(client, "/user") + .map(json -> { + try { + return GitHubModel.objectMapper.readValue(json, GitHubModel.User.class); + } catch (IOException e) { + throw new IllegalStateException("Failed to deserialize GitHubUser", e); + } + }); + } + + @DefineKernelFunction(name = "get_repo_info", description = "Get repository information from GitHub", + returnType = "com.microsoft.semantickernel.samples.plugins.github.GitHubModel$Repository") + public Mono getRepositoryAsync( + @KernelFunctionParameter( + name = "organization", + description = "The name of the repository to retrieve information for" + ) String organization, + @KernelFunctionParameter( + name = "repo_name", + description = "The name of the repository to retrieve information for" + ) String repoName + ) { + HttpClient client = createClient(); + + return makeRequestAsync(client, String.format("/repos/%s/%s", organization, repoName)) + .map(json -> { + try { + return GitHubModel.objectMapper.readValue(json, GitHubModel.Repository.class); + } catch (IOException e) { + throw new IllegalStateException("Failed to deserialize GitHubRepository", e); + } + }); + } + + @DefineKernelFunction(name = "get_issues", description = "Get issues from GitHub", + returnType = "java.util.List") + public Mono> getIssuesAsync( + @KernelFunctionParameter( + name = "organization", + description = "The name of the organization to retrieve issues for" + ) String organization, + @KernelFunctionParameter( + name = "repo_name", + description = "The name of the repository to retrieve issues for" + ) String repoName, + @KernelFunctionParameter( + name = "max_results", + description = "The maximum number of issues to retrieve", + required = false, + defaultValue = "10", + type = int.class + ) int maxResults, + @KernelFunctionParameter( + name = "state", + description = "The state of the issues to retrieve", + required = false, + defaultValue = "open" + ) String state, + @KernelFunctionParameter( + name = "assignee", + description = "The assignee of the issues to retrieve", + required = false + ) String assignee + ) { + HttpClient client = createClient(); + + String query = String.format("/repos/%s/%s/issues", organization, repoName); + query = buildQueryString(query, "state", state); + query = buildQueryString(query, "assignee", assignee); + query = buildQueryString(query, "per_page", String.valueOf(maxResults)); + + return makeRequestAsync(client, query) + .flatMap(json -> { + try { + GitHubModel.Issue[] issues = GitHubModel.objectMapper.readValue(json, GitHubModel.Issue[].class); + return Mono.just(List.of(issues)); + } catch (IOException e) { + throw new IllegalStateException("Failed to deserialize GitHubIssues", e); + } + }); + } + + @DefineKernelFunction(name = "get_issue_detail_info", description = "Get issue detail information from GitHub", + returnType = "com.microsoft.semantickernel.samples.plugins.github.GitHubModel$IssueDetail") + public GitHubModel.IssueDetail getIssueDetailAsync( + @KernelFunctionParameter( + name = "organization", + description = "The name of the repository to retrieve information for" + ) String organization, + @KernelFunctionParameter( + name = "repo_name", + description = "The name of the repository to retrieve information for" + ) String repoName, + @KernelFunctionParameter( + name = "issue_number", + description = "The issue number to retrieve information for", + type = int.class + ) int issueNumber + ) { + HttpClient client = createClient(); + + return makeRequestAsync(client, String.format("/repos/%s/%s/issues/%d", organization, repoName, issueNumber)) + .map(json -> { + try { + return GitHubModel.objectMapper.readValue(json, GitHubModel.IssueDetail.class); + } catch (IOException e) { + throw new IllegalStateException("Failed to deserialize GitHubIssue", e); + } + }).block(); + } + + private HttpClient createClient() { + return HttpClient.create() + .baseUrl(baseUrl) + .headers(headers -> { + headers.add("User-Agent", "request"); + headers.add("Accept", "application/vnd.github+json"); + headers.add("Authorization", "Bearer " + token); + headers.add("X-GitHub-Api-Version", "2022-11-28"); + }); + } + + private static String buildQueryString(String path, String param, String value) { + if (value == null || value.isEmpty() || value.equals(KernelFunctionParameter.NO_DEFAULT_VALUE)) { + return path; + } + + return path + (path.contains("?") ? "&" : "?") + param + "=" + value; + } + + private Mono makeRequestAsync(HttpClient client, String path) { + return client + .get() + .uri(path) + .responseSingle((res, content) -> { + if (res.status().code() != 200) { + return Mono.error(new IllegalStateException("Request failed: " + res.status())); + } + return content.asString(); + }); + } +} diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionsAgent.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionsAgent.java new file mode 100644 index 00000000..38e9f1e8 --- /dev/null +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionsAgent.java @@ -0,0 +1,145 @@ +package com.microsoft.semantickernel.samples.syntaxexamples.agents; + +import com.azure.ai.openai.OpenAIAsyncClient; +import com.azure.ai.openai.OpenAIClientBuilder; +import com.azure.core.credential.AzureKeyCredential; +import com.azure.core.credential.KeyCredential; +import com.microsoft.semantickernel.Kernel; +import com.microsoft.semantickernel.agents.AgentInvokeOptions; +import com.microsoft.semantickernel.agents.chatcompletion.ChatCompletionAgent; +import com.microsoft.semantickernel.agents.chatcompletion.ChatHistoryAgentThread; +import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIChatCompletion; +import com.microsoft.semantickernel.contextvariables.ContextVariableTypeConverter; +import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; +import com.microsoft.semantickernel.implementation.templateengine.tokenizer.DefaultPromptTemplate; +import com.microsoft.semantickernel.orchestration.InvocationContext; +import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; +import com.microsoft.semantickernel.orchestration.ToolCallBehavior; +import com.microsoft.semantickernel.plugin.KernelPluginFactory; +import com.microsoft.semantickernel.samples.plugins.github.GitHubModel; +import com.microsoft.semantickernel.samples.plugins.github.GitHubPlugin; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; +import com.microsoft.semantickernel.semanticfunctions.PromptTemplateConfig; +import com.microsoft.semantickernel.services.chatcompletion.AuthorRole; +import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; +import com.microsoft.semantickernel.services.chatcompletion.ChatMessageContent; + +import java.util.List; +import java.util.Scanner; + +public class CompletionsAgent { + private static final String CLIENT_KEY = System.getenv("CLIENT_KEY"); + private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY"); + + // Only required if AZURE_CLIENT_KEY is set + private static final String CLIENT_ENDPOINT = System.getenv("CLIENT_ENDPOINT"); + private static final String MODEL_ID = System.getenv() + .getOrDefault("MODEL_ID", "gpt-35-turbo"); + + private static final String GITHUB_PAT = System.getenv("GITHUB_PAT"); + public static void main(String[] args) { + System.out.println("======== ChatCompletion Agent ========"); + + OpenAIAsyncClient client; + + if (AZURE_CLIENT_KEY != null) { + client = new OpenAIClientBuilder() + .credential(new AzureKeyCredential(AZURE_CLIENT_KEY)) + .endpoint(CLIENT_ENDPOINT) + .buildAsyncClient(); + + } else { + client = new OpenAIClientBuilder() + .credential(new KeyCredential(CLIENT_KEY)) + .buildAsyncClient(); + } + + ChatCompletionService chatCompletion = OpenAIChatCompletion.builder() + .withModelId(MODEL_ID) + .withOpenAIAsyncClient(client) + .build(); + + System.out.println("------------------------"); + + ContextVariableTypes.addGlobalConverter( + new ContextVariableTypeConverter<>( + GitHubModel.Issue.class, + o -> (GitHubModel.Issue) o, + o -> o.toString(), + s -> null + ) + ); + + Kernel kernel = Kernel.builder() + .withAIService(ChatCompletionService.class, chatCompletion) + .withPlugin(KernelPluginFactory.createFromObject(new GitHubPlugin(GITHUB_PAT), + "GitHubPlugin")) + .build(); + + InvocationContext invocationContext = InvocationContext.builder() + .withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(true)) + .build(); + + ChatCompletionAgent agent = ChatCompletionAgent.builder() + .withKernel(kernel) + .withKernelArguments( + KernelArguments.builder() + .withVariable("repository", "microsoft/semantic-kernel-java") + .withExecutionSettings(PromptExecutionSettings.builder() + .build()) + .build() + ) + .withInvocationContext(invocationContext) + .withTemplate( + DefaultPromptTemplate.build( + PromptTemplateConfig.builder() + .withTemplate( + """ + You are an agent designed to query and retrieve information from a single GitHub repository in a read-only manner. + You are also able to access the profile of the active user. + + Use the current date and time to provide up-to-date details or time-sensitive responses. + + The repository you are querying is a public repository with the following name: {{$repository}} + + The current date and time is: {{$now}}. + """ + ) + .build() + ) + ).build(); + + ChatHistoryAgentThread agentThread = new ChatHistoryAgentThread(); + agentThread.createAsync().block(); + + Scanner scanner = new Scanner(System.in); + + while (true) { + System.out.print("> "); + + String input = scanner.nextLine(); + if (input.equalsIgnoreCase("exit")) { + break; + } + + var message = new ChatMessageContent<>(AuthorRole.USER, input); + KernelArguments arguments = KernelArguments.builder() + .withVariable("now", System.currentTimeMillis()) + .build(); + + var responses = agent.invokeAsync( + List.of(message), + agentThread, + AgentInvokeOptions.builder() + .withKernel(kernel) + .withKernelArguments(arguments) + .build() + ).block(); + + for (var response : responses) { + System.out.println("> " + response.getMessage()); + } + } + } + +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/Agent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/Agent.java new file mode 100644 index 00000000..314201e9 --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/Agent.java @@ -0,0 +1,60 @@ +package com.microsoft.semantickernel.agents; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Function; +import java.util.function.Supplier; + +import com.microsoft.semantickernel.Kernel; +import com.microsoft.semantickernel.orchestration.InvocationContext; +import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; +import com.microsoft.semantickernel.semanticfunctions.PromptTemplate; +import com.microsoft.semantickernel.services.chatcompletion.ChatHistory; +import com.microsoft.semantickernel.services.chatcompletion.ChatMessageContent; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +/** + * Interface for a semantic kernel agent. + */ +public interface Agent { + + /** + * Gets the agent's ID. + * + * @return The agent's ID + */ + String getId(); + + /** + * Gets the agent's name. + * + * @return The agent's name + */ + String getName(); + + /** + * Gets the agent's description. + * + * @return The agent's description + */ + String getDescription(); + + /** + * Invoke the agent with the given chat history. + * + * @param messages The chat history to process + * @param thread The agent thread to use + * @param options The options for invoking the agent + * @return A Mono containing the agent response + */ + Mono>>> invokeAsync(List> messages, AgentThread thread, AgentInvokeOptions options); + + + + + Mono notifyThreadOfNewMessageAsync(AgentThread thread, ChatMessageContent newMessage); +} \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java new file mode 100644 index 00000000..a542c40a --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java @@ -0,0 +1,126 @@ +package com.microsoft.semantickernel.agents; + +import com.microsoft.semantickernel.Kernel; +import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.orchestration.InvocationContext; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; + +/** + * Options for invoking an agent. + */ +public class AgentInvokeOptions { + + private final KernelArguments KernelArguments; + private final Kernel kernel; + private final String additionalInstructions; + private final InvocationContext invocationContext; + + + /** + * Constructor for AgentInvokeOptions. + * + * @param KernelArguments The arguments for the kernel function. + * @param kernel The kernel to use. + * @param additionalInstructions Additional instructions for the agent. + * @param invocationContext The invocation context. + */ + public AgentInvokeOptions(KernelArguments KernelArguments, Kernel kernel, String additionalInstructions, + InvocationContext invocationContext) { + this.KernelArguments = KernelArguments; + this.kernel = kernel; + this.additionalInstructions = additionalInstructions; + this.invocationContext = invocationContext; + } + + public KernelArguments getKernelArguments() { + return KernelArguments; + } + + public Kernel getKernel() { + return kernel; + } + + public String getAdditionalInstructions() { + return additionalInstructions; + } + + public InvocationContext getInvocationContext() { + return invocationContext; + } + + + + /** + * Builder for AgentInvokeOptions. + */ + public static Builder builder() { + return new Builder(); + } + + public static class Builder implements SemanticKernelBuilder { + + private KernelArguments kernelArguments; + private Kernel kernel; + private String additionalInstructions; + private InvocationContext invocationContext; + + /** + * Set the kernel arguments. + * + * @param kernelArguments The kernel arguments. + * @return The builder. + */ + public Builder withKernelArguments(KernelArguments kernelArguments) { + this.kernelArguments = kernelArguments; + return this; + } + + /** + * Set the kernel. + * + * @param kernel The kernel. + * @return The builder. + */ + public Builder withKernel(Kernel kernel) { + this.kernel = kernel; + return this; + } + + /** + * Set additional instructions. + * + * @param additionalInstructions The additional instructions. + * @return The builder. + */ + public Builder withAdditionalInstructions(String additionalInstructions) { + this.additionalInstructions = additionalInstructions; + return this; + } + + /** + * Set the invocation context. + * + * @param invocationContext The invocation context. + * @return The builder. + */ + public Builder withInvocationContext(InvocationContext invocationContext) { + this.invocationContext = invocationContext; + return this; + } + + /** + * Build the object. + * + * @return a constructed object. + */ + @Override + public AgentInvokeOptions build() { + return new AgentInvokeOptions( + kernelArguments != null ? kernelArguments : com.microsoft.semantickernel.semanticfunctions.KernelArguments.builder().build(), + kernel != null ? kernel : Kernel.builder().build(), + additionalInstructions, + invocationContext + ); + } + } +} \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentResponseItem.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentResponseItem.java new file mode 100644 index 00000000..61ffdd6d --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentResponseItem.java @@ -0,0 +1,19 @@ +package com.microsoft.semantickernel.agents; + +public class AgentResponseItem { + private final T message; + private final AgentThread thread; + + public AgentResponseItem(T message, AgentThread thread) { + this.message = message; + this.thread = thread; + } + + public T getMessage() { + return message; + } + + public AgentThread getThread() { + return thread; + } +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentThread.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentThread.java new file mode 100644 index 00000000..0c83751a --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentThread.java @@ -0,0 +1,45 @@ +package com.microsoft.semantickernel.agents; + +import com.microsoft.semantickernel.services.chatcompletion.ChatMessageContent; +import reactor.core.publisher.Mono; + +/** + * Interface for an agent thread. + */ +public interface AgentThread { + /** + * Get the thread ID. + * + * @return The thread ID. + */ + String getId(); + + /** + * Create a new thread. + * + * @return A Mono containing the thread ID. + */ + Mono createAsync(); + + /** + * Delete the thread. + * + * @return A Mono indicating completion. + */ + Mono deleteAsync(); + + /** + * Check if the thread is deleted. + * + * @return A Mono containing true if the thread is deleted, false otherwise. + */ + boolean isDeleted(); + + /** + * Handle a new message in the thread. + * + * @param newMessage The new message to handle. + * @return A Mono indicating completion. + */ + Mono onNewMessageAsync(ChatMessageContent newMessage); +} \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/BaseAgentThread.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/BaseAgentThread.java new file mode 100644 index 00000000..b7c97eea --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/BaseAgentThread.java @@ -0,0 +1,23 @@ +package com.microsoft.semantickernel.agents; + +public abstract class BaseAgentThread implements AgentThread { + + protected String id; + protected boolean isDeleted; + + public BaseAgentThread() { + } + + public BaseAgentThread(String id) { + this.id = id; + } + + @Override + public String getId() { + return id; + } + @Override + public boolean isDeleted() { + return isDeleted; + } +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java new file mode 100644 index 00000000..8aaa432e --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java @@ -0,0 +1,139 @@ +package com.microsoft.semantickernel.agents; + +import com.microsoft.semantickernel.Kernel; +import com.microsoft.semantickernel.orchestration.InvocationContext; +import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; +import com.microsoft.semantickernel.semanticfunctions.PromptTemplate; +import com.microsoft.semantickernel.services.chatcompletion.ChatMessageContent; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Supplier; + +public abstract class KernelAgent implements Agent { + + protected final String id; + + protected final String name; + + protected final String description; + + protected final Kernel kernel; + protected final KernelArguments kernelArguments; + protected final InvocationContext invocationContext; + protected final String instructions; + + protected final PromptTemplate template; + + protected KernelAgent( + String id, + String name, + String description, + Kernel kernel, + KernelArguments kernelArguments, + InvocationContext invocationContext, + String instructions, + PromptTemplate template + ) { + this.id = id != null ? id : UUID.randomUUID().toString(); + this.name = name; + this.description = description; + this.kernel = kernel; + this.kernelArguments = kernelArguments; + this.invocationContext = invocationContext; + this.instructions = instructions; + this.template = template; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public Kernel getKernel() { + return kernel; + } + + public KernelArguments getKernelArguments() { + return kernelArguments; + } + + public String getInstructions() { + return instructions; + } + + public PromptTemplate getTemplate() { + return template; + } + + protected Mono ensureThreadExistsAsync(List> messages, AgentThread thread, Supplier constructor) { + return Mono.defer(() -> { + T newThread = thread != null ? (T) thread : constructor.get(); + + return newThread.createAsync() + .thenMany(Flux.fromIterable(messages)) + .concatMap(message -> { + return notifyThreadOfNewMessageAsync(newThread, message) + .then(Mono.just(message)); + }) + .then(Mono.just(newThread)); + }); + } + + @Override + public Mono notifyThreadOfNewMessageAsync(AgentThread thread, ChatMessageContent message) { + return Mono.defer(() -> { + return thread.onNewMessageAsync(message); + }); + } + + + /** + * Merges the provided arguments with the current arguments. + * Provided arguments will override the current arguments. + * + * @param arguments The arguments to merge with the current arguments. + */ + protected KernelArguments mergeArguments(KernelArguments arguments) { + if (arguments == null) { + return kernelArguments; + } + + Map executionSettings = new HashMap<>(kernelArguments.getExecutionSettings()); + executionSettings.putAll(arguments.getExecutionSettings()); + + return KernelArguments.builder() + .withVariables(kernelArguments) + .withVariables(arguments) + .withExecutionSettings(executionSettings) + .build(); + } + + /** + * Formats the instructions using the provided kernel, arguments, and context. + * + * @param kernel The kernel to use for formatting. + * @param arguments The arguments to use for formatting. + * @param context The context to use for formatting. + * @return A Mono that resolves to the formatted instructions. + */ + protected Mono formatInstructionsAsync(Kernel kernel, KernelArguments arguments, InvocationContext context) { + if (template != null) { + return template.renderAsync(kernel, arguments, context); + } else { + return Mono.just(instructions); + } + } +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java index 25cd8ea8..df5f1832 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java @@ -187,6 +187,13 @@ public ChatHistory addSystemMessage(String content) { return addMessage(AuthorRole.SYSTEM, content); } + /** + * Clear the chat history + */ + public void clear() { + chatMessageContents.clear(); + } + /** * Add all messages to the chat history * @param messages The messages to add to the chat history diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index bb4d766f..32ca102b 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -112,6 +112,12 @@ ${project.version} + + com.microsoft.semantic-kernel + semantickernel-agents-core + ${project.version} + + com.azure azure-ai-openai From e242d99f517383055184609488dedf4c3cbb9c1b Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Mon, 21 Apr 2025 11:00:31 -0700 Subject: [PATCH 131/277] Updates --- .../agents/chatcompletion/ChatCompletionAgent.java | 6 +++--- .../samples/plugins/github/GitHubPlugin.java | 2 +- .../syntaxexamples/agents/CompletionsAgent.java | 11 ++++------- .../com/microsoft/semantickernel/agents/Agent.java | 8 +++++--- .../microsoft/semantickernel/agents/KernelAgent.java | 8 +++----- 5 files changed, 16 insertions(+), 19 deletions(-) diff --git a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java index c8c29007..413abbc2 100644 --- a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java +++ b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java @@ -110,7 +110,7 @@ private Mono>> internalInvokeAsync( ? invocationContext.getPromptExecutionSettings() : kernelArguments.getExecutionSettings().get(chatCompletionService.getServiceId()); - final InvocationContext newMessagesContext = InvocationContext.builder() + final InvocationContext updatedInvocationContext = InvocationContext.builder() .withPromptExecutionSettings(executionSettings) .withToolCallBehavior(invocationContext.getToolCallBehavior()) .withTelemetry(invocationContext.getTelemetry()) @@ -119,7 +119,7 @@ private Mono>> internalInvokeAsync( .withReturnMode(InvocationReturnMode.FULL_HISTORY) .build(); - return formatInstructionsAsync(kernel, arguments, newMessagesContext).flatMap( + return formatInstructionsAsync(kernel, arguments, updatedInvocationContext).flatMap( instructions -> { // Create a new chat history with the instructions ChatHistory chat = new ChatHistory( @@ -137,7 +137,7 @@ private Mono>> internalInvokeAsync( chat.addAll(history); int previousHistorySize = chat.getMessages().size(); - return chatCompletionService.getChatMessageContentsAsync(chat, kernel, newMessagesContext) + return chatCompletionService.getChatMessageContentsAsync(chat, kernel, updatedInvocationContext) .map(chatMessageContents -> { return chatMessageContents.subList( previousHistorySize, diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubPlugin.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubPlugin.java index 2d1fb444..d3c59a15 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubPlugin.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubPlugin.java @@ -103,7 +103,7 @@ public Mono> getIssuesAsync( }); } - @DefineKernelFunction(name = "get_issue_detail_info", description = "Get issue detail information from GitHub", + @DefineKernelFunction(name = "get_issue_detail_info", description = "Get detail information of a single issue from GitHub", returnType = "com.microsoft.semantickernel.samples.plugins.github.GitHubModel$IssueDetail") public GitHubModel.IssueDetail getIssueDetailAsync( @KernelFunctionParameter( diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionsAgent.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionsAgent.java index 38e9f1e8..9e2d82c7 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionsAgent.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionsAgent.java @@ -34,7 +34,7 @@ public class CompletionsAgent { // Only required if AZURE_CLIENT_KEY is set private static final String CLIENT_ENDPOINT = System.getenv("CLIENT_ENDPOINT"); private static final String MODEL_ID = System.getenv() - .getOrDefault("MODEL_ID", "gpt-35-turbo"); + .getOrDefault("MODEL_ID", "gpt-4o"); private static final String GITHUB_PAT = System.getenv("GITHUB_PAT"); public static void main(String[] args) { @@ -77,7 +77,7 @@ public static void main(String[] args) { .build(); InvocationContext invocationContext = InvocationContext.builder() - .withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(true)) + .withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(false)) .build(); ChatCompletionAgent agent = ChatCompletionAgent.builder() @@ -127,7 +127,7 @@ public static void main(String[] args) { .withVariable("now", System.currentTimeMillis()) .build(); - var responses = agent.invokeAsync( + var response = agent.invokeAsync( List.of(message), agentThread, AgentInvokeOptions.builder() @@ -136,10 +136,7 @@ public static void main(String[] args) { .build() ).block(); - for (var response : responses) { - System.out.println("> " + response.getMessage()); - } + System.out.println("> " + response.get(response.size() - 1).getMessage()); } } - } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/Agent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/Agent.java index 314201e9..3a82550c 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/Agent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/Agent.java @@ -53,8 +53,10 @@ public interface Agent { */ Mono>>> invokeAsync(List> messages, AgentThread thread, AgentInvokeOptions options); - - - + /** + * Notifies the agent of a new message. + * + * @param thread The agent thread to use + */ Mono notifyThreadOfNewMessageAsync(AgentThread thread, ChatMessageContent newMessage); } \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java index 8aaa432e..a9af6f87 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java @@ -2,6 +2,7 @@ import com.microsoft.semantickernel.Kernel; import com.microsoft.semantickernel.orchestration.InvocationContext; +import com.microsoft.semantickernel.orchestration.InvocationReturnMode; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.PromptTemplate; @@ -18,16 +19,12 @@ public abstract class KernelAgent implements Agent { protected final String id; - protected final String name; - protected final String description; - protected final Kernel kernel; protected final KernelArguments kernelArguments; protected final InvocationContext invocationContext; protected final String instructions; - protected final PromptTemplate template; protected KernelAgent( @@ -45,7 +42,8 @@ protected KernelAgent( this.description = description; this.kernel = kernel; this.kernelArguments = kernelArguments; - this.invocationContext = invocationContext; + this.invocationContext = invocationContext != null + ? invocationContext : InvocationContext.builder().withReturnMode(InvocationReturnMode.FULL_HISTORY).build(); this.instructions = instructions; this.template = template; } From 83daec2a7de3a75b1f06f2f59489daab2dd56054 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Mon, 21 Apr 2025 22:30:01 -0700 Subject: [PATCH 132/277] Renaming sample and adjust invocation context --- .../chatcompletion/ChatCompletionAgent.java | 32 ++++++++++++------- ...letionsAgent.java => CompletionAgent.java} | 4 +-- 2 files changed, 23 insertions(+), 13 deletions(-) rename samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/{CompletionsAgent.java => CompletionAgent.java} (99%) diff --git a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java index 413abbc2..b521ce59 100644 --- a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java +++ b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java @@ -106,20 +106,30 @@ private Mono>> internalInvokeAsync( try { ChatCompletionService chatCompletionService = kernel.getService(ChatCompletionService.class, arguments); - PromptExecutionSettings executionSettings = invocationContext.getPromptExecutionSettings() != null + PromptExecutionSettings executionSettings = invocationContext != null && invocationContext.getPromptExecutionSettings() != null ? invocationContext.getPromptExecutionSettings() : kernelArguments.getExecutionSettings().get(chatCompletionService.getServiceId()); - final InvocationContext updatedInvocationContext = InvocationContext.builder() + ToolCallBehavior toolCallBehavior = invocationContext != null + ? invocationContext.getToolCallBehavior() + : ToolCallBehavior.allowAllKernelFunctions(false); + + // Build base invocation context + InvocationContext.Builder builder = InvocationContext.builder() .withPromptExecutionSettings(executionSettings) - .withToolCallBehavior(invocationContext.getToolCallBehavior()) - .withTelemetry(invocationContext.getTelemetry()) - .withContextVariableConverter(invocationContext.getContextVariableTypes()) - .withKernelHooks(invocationContext.getKernelHooks()) - .withReturnMode(InvocationReturnMode.FULL_HISTORY) - .build(); - - return formatInstructionsAsync(kernel, arguments, updatedInvocationContext).flatMap( + .withToolCallBehavior(toolCallBehavior) + .withReturnMode(InvocationReturnMode.FULL_HISTORY); + + if (invocationContext != null) { + builder = builder + .withTelemetry(invocationContext.getTelemetry()) + .withContextVariableConverter(invocationContext.getContextVariableTypes()) + .withKernelHooks(invocationContext.getKernelHooks()); + } + + InvocationContext agentInvocationContext = builder.build(); + + return formatInstructionsAsync(kernel, arguments, agentInvocationContext).flatMap( instructions -> { // Create a new chat history with the instructions ChatHistory chat = new ChatHistory( @@ -137,7 +147,7 @@ private Mono>> internalInvokeAsync( chat.addAll(history); int previousHistorySize = chat.getMessages().size(); - return chatCompletionService.getChatMessageContentsAsync(chat, kernel, updatedInvocationContext) + return chatCompletionService.getChatMessageContentsAsync(chat, kernel, agentInvocationContext) .map(chatMessageContents -> { return chatMessageContents.subList( previousHistorySize, diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionsAgent.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionAgent.java similarity index 99% rename from samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionsAgent.java rename to samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionAgent.java index 9e2d82c7..180b0c80 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionsAgent.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionAgent.java @@ -27,7 +27,7 @@ import java.util.List; import java.util.Scanner; -public class CompletionsAgent { +public class CompletionAgent { private static final String CLIENT_KEY = System.getenv("CLIENT_KEY"); private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY"); @@ -77,7 +77,7 @@ public static void main(String[] args) { .build(); InvocationContext invocationContext = InvocationContext.builder() - .withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(false)) + .withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(true)) .build(); ChatCompletionAgent agent = ChatCompletionAgent.builder() From 4664b2b835cf364b305c880ca4fe0ab752142b2c Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 24 Apr 2025 22:35:58 -0700 Subject: [PATCH 133/277] Updates --- .../chatcompletion/ChatCompletionAgent.java | 160 ++++++++++++------ .../ChatHistoryAgentThread.java | 51 +++++- .../agents/CompletionAgent.java | 26 ++- .../agents/AgentInvokeOptions.java | 20 ++- .../semantickernel/agents/AgentThread.java | 7 + .../semantickernel/agents/KernelAgent.java | 74 +++++--- .../chatcompletion/ChatMessageContent.java | 44 +++++ 7 files changed, 289 insertions(+), 93 deletions(-) diff --git a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java index b521ce59..2e9a0d7d 100644 --- a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java +++ b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java @@ -6,7 +6,6 @@ import com.microsoft.semantickernel.agents.AgentThread; import com.microsoft.semantickernel.agents.KernelAgent; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.contextvariables.ContextVariable; import com.microsoft.semantickernel.orchestration.InvocationContext; import com.microsoft.semantickernel.orchestration.InvocationReturnMode; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; @@ -23,14 +22,13 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import javax.annotation.Nullable; import java.util.List; -import java.util.Map; -import java.util.function.Function; import java.util.stream.Collectors; public class ChatCompletionAgent extends KernelAgent { - ChatCompletionAgent( + private ChatCompletionAgent( String id, String name, String description, @@ -61,43 +59,49 @@ public class ChatCompletionAgent extends KernelAgent { * @return A Mono containing the agent response */ @Override - public Mono>>> invokeAsync(List> messages, AgentThread thread, AgentInvokeOptions options) { - - Mono chatHistoryFromThread = this.ensureThreadExistsAsync(messages, thread, ChatHistoryAgentThread::new) - .cast(ChatHistoryAgentThread.class) - .map(ChatHistoryAgentThread::getChatHistory) - .flatMap(threadChatHistory -> { - return Mono.just(new ChatHistory(threadChatHistory.getMessages())); - }); - - - Mono>> updatedChatHistory = chatHistoryFromThread.flatMap( - chatHistory -> internalInvokeAsync( - this.getName(), - chatHistory, - options - ) - ); + public Mono>>> invokeAsync( + List> messages, + AgentThread thread, + @Nullable AgentInvokeOptions options + ) { + return ensureThreadExistsWithMessagesAsync(messages, thread, ChatHistoryAgentThread::new) + .cast(ChatHistoryAgentThread.class) + .flatMap(agentThread -> { + // Extract the chat history from the thread + ChatHistory history = new ChatHistory( + agentThread.getChatHistory().getMessages() + ); - return updatedChatHistory.flatMap(chatMessageContents -> { - return Flux.fromIterable(chatMessageContents) - .concatMap(chatMessageContent -> this.notifyThreadOfNewMessageAsync(thread, chatMessageContent)) - .then(Mono.just(chatMessageContents)); // return the original list - }).flatMap(chatMessageContents -> { - return Mono.just(chatMessageContents.stream() - .map(chatMessageContent -> { - return new AgentResponseItem>( - chatMessageContent, - thread); - }).collect(Collectors.toList())); - }); + // Invoke the agent with the chat history + return internalInvokeAsync( + history, + options + ) + .flatMapMany(Flux::fromIterable) + // notify on the new thread instance + .concatMap(agentMessage -> { + // Set the author name for the message + agentMessage.setAuthorName(this.name); + + return this.notifyThreadOfNewMessageAsync(agentThread, agentMessage).thenReturn(agentMessage); + }) + .collectList() + .map(chatMessageContents -> + chatMessageContents.stream() + .map(message -> new AgentResponseItem>(message, agentThread)) + .collect(Collectors.toList()) + ); + }); } private Mono>> internalInvokeAsync( - String agentName, ChatHistory history, - AgentInvokeOptions options + @Nullable AgentInvokeOptions options ) { + if (options == null) { + options = new AgentInvokeOptions(); + } + final Kernel kernel = options.getKernel() != null ? options.getKernel() : this.kernel; final KernelArguments arguments = mergeArguments(options.getKernelArguments()); final String additionalInstructions = options.getAdditionalInstructions(); @@ -112,13 +116,13 @@ private Mono>> internalInvokeAsync( ToolCallBehavior toolCallBehavior = invocationContext != null ? invocationContext.getToolCallBehavior() - : ToolCallBehavior.allowAllKernelFunctions(false); + : ToolCallBehavior.allowAllKernelFunctions(true); // Build base invocation context InvocationContext.Builder builder = InvocationContext.builder() .withPromptExecutionSettings(executionSettings) .withToolCallBehavior(toolCallBehavior) - .withReturnMode(InvocationReturnMode.FULL_HISTORY); + .withReturnMode(InvocationReturnMode.NEW_MESSAGES_ONLY); if (invocationContext != null) { builder = builder @@ -129,7 +133,7 @@ private Mono>> internalInvokeAsync( InvocationContext agentInvocationContext = builder.build(); - return formatInstructionsAsync(kernel, arguments, agentInvocationContext).flatMap( + return renderInstructionsAsync(kernel, arguments, agentInvocationContext).flatMap( instructions -> { // Create a new chat history with the instructions ChatHistory chat = new ChatHistory( @@ -144,24 +148,26 @@ private Mono>> internalInvokeAsync( )); } + // Add the chat history to the new chat chat.addAll(history); - int previousHistorySize = chat.getMessages().size(); - - return chatCompletionService.getChatMessageContentsAsync(chat, kernel, agentInvocationContext) - .map(chatMessageContents -> { - return chatMessageContents.subList( - previousHistorySize, - chatMessageContents.size()); - }); + + return chatCompletionService.getChatMessageContentsAsync(chat, kernel, agentInvocationContext); } ); - } catch (ServiceNotFoundException e) { throw new RuntimeException(e); } } + + @Override + public Mono notifyThreadOfNewMessageAsync(AgentThread thread, ChatMessageContent message) { + return Mono.defer(() -> { + return thread.onNewMessageAsync(message); + }); + } + /** * Builder for creating instances of ChatCompletionAgent. */ @@ -174,71 +180,123 @@ public static class Builder implements SemanticKernelBuilder deleteAsync() { return Mono.fromRunnable(chatHistory::clear); } + /** + * Create a copy of the thread. + * + * @return A new instance of the thread. + */ + @Override + public ChatHistoryAgentThread copy() { + return new ChatHistoryAgentThread(this.id, new ChatHistory(chatHistory.getMessages())); + } + @Override public Mono onNewMessageAsync(ChatMessageContent newMessage) { return Mono.fromRunnable(() -> { @@ -61,4 +73,41 @@ public Mono onNewMessageAsync(ChatMessageContent newMessage) { public List> getMessages() { return chatHistory.getMessages(); } + + + public static Builder builder() { + return new Builder(); + } + + public static class Builder implements SemanticKernelBuilder { + private String id; + private ChatHistory chatHistory; + + /** + * Set the ID of the thread. + * + * @param id The ID of the thread. + * @return The builder instance. + */ + public Builder withId(String id) { + this.id = id; + return this; + } + + /** + * Set the chat history. + * + * @param chatHistory The chat history. + * @return The builder instance. + */ + public Builder withChatHistory(ChatHistory chatHistory) { + this.chatHistory = chatHistory; + return this; + } + + @Override + public ChatHistoryAgentThread build() { + return new ChatHistoryAgentThread(id, chatHistory); + } + } } diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionAgent.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionAgent.java index 180b0c80..1e5a665a 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionAgent.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionAgent.java @@ -54,22 +54,13 @@ public static void main(String[] args) { .buildAsyncClient(); } + System.out.println("------------------------"); + ChatCompletionService chatCompletion = OpenAIChatCompletion.builder() .withModelId(MODEL_ID) .withOpenAIAsyncClient(client) .build(); - System.out.println("------------------------"); - - ContextVariableTypes.addGlobalConverter( - new ContextVariableTypeConverter<>( - GitHubModel.Issue.class, - o -> (GitHubModel.Issue) o, - o -> o.toString(), - s -> null - ) - ); - Kernel kernel = Kernel.builder() .withAIService(ChatCompletionService.class, chatCompletion) .withPlugin(KernelPluginFactory.createFromObject(new GitHubPlugin(GITHUB_PAT), @@ -78,6 +69,12 @@ public static void main(String[] args) { InvocationContext invocationContext = InvocationContext.builder() .withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(true)) + .withContextVariableConverter(new ContextVariableTypeConverter<>( + GitHubModel.Issue.class, + o -> (GitHubModel.Issue) o, + o -> o.toString(), + s -> null + )) .build(); ChatCompletionAgent agent = ChatCompletionAgent.builder() @@ -110,8 +107,6 @@ public static void main(String[] args) { ).build(); ChatHistoryAgentThread agentThread = new ChatHistoryAgentThread(); - agentThread.createAsync().block(); - Scanner scanner = new Scanner(System.in); while (true) { @@ -136,7 +131,10 @@ public static void main(String[] args) { .build() ).block(); - System.out.println("> " + response.get(response.size() - 1).getMessage()); + var lastResponse = response.get(response.size() - 1); + + System.out.println("> " + lastResponse.getMessage()); + agentThread = (ChatHistoryAgentThread) lastResponse.getThread(); } } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java index a542c40a..d5a99b5a 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java @@ -5,6 +5,8 @@ import com.microsoft.semantickernel.orchestration.InvocationContext; import com.microsoft.semantickernel.semanticfunctions.KernelArguments; +import javax.annotation.Nullable; + /** * Options for invoking an agent. */ @@ -15,6 +17,12 @@ public class AgentInvokeOptions { private final String additionalInstructions; private final InvocationContext invocationContext; + /** + * Default constructor for AgentInvokeOptions. + */ + public AgentInvokeOptions() { + this(null, null, null, null); + } /** * Constructor for AgentInvokeOptions. @@ -24,8 +32,10 @@ public class AgentInvokeOptions { * @param additionalInstructions Additional instructions for the agent. * @param invocationContext The invocation context. */ - public AgentInvokeOptions(KernelArguments KernelArguments, Kernel kernel, String additionalInstructions, - InvocationContext invocationContext) { + public AgentInvokeOptions(@Nullable KernelArguments KernelArguments, + @Nullable Kernel kernel, + @Nullable String additionalInstructions, + @Nullable InvocationContext invocationContext) { this.KernelArguments = KernelArguments; this.kernel = kernel; this.additionalInstructions = additionalInstructions; @@ -116,10 +126,10 @@ public Builder withInvocationContext(InvocationContext invocationContext) { @Override public AgentInvokeOptions build() { return new AgentInvokeOptions( - kernelArguments != null ? kernelArguments : com.microsoft.semantickernel.semanticfunctions.KernelArguments.builder().build(), - kernel != null ? kernel : Kernel.builder().build(), + kernelArguments, + kernel, additionalInstructions, - invocationContext + invocationContext ); } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentThread.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentThread.java index 0c83751a..d369d999 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentThread.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentThread.java @@ -35,6 +35,13 @@ public interface AgentThread { */ boolean isDeleted(); + /** + * Create a copy of the thread. + * + * @return A new instance of the thread. + */ + AgentThread copy(); + /** * Handle a new message in the thread. * diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java index a9af6f87..e5f54a6d 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java @@ -48,55 +48,69 @@ protected KernelAgent( this.template = template; } + /** + * Gets the agent's ID. + * + * @return The agent's ID + */ public String getId() { return id; } + /** + * Gets the agent's name. + * + * @return The agent's name + */ public String getName() { return name; } + /** + * Gets the agent's description. + * + * @return The agent's description + */ public String getDescription() { return description; } + /** + * Gets the kernel used by the agent. + * + * @return The kernel used by the agent + */ public Kernel getKernel() { return kernel; } + /** + * Gets the invocation context used by the agent. + * + * @return The invocation context used by the agent + */ public KernelArguments getKernelArguments() { return kernelArguments; } + /** + * Gets the invocation context used by the agent. + * + * @return The invocation context used by the agent + */ public String getInstructions() { return instructions; } + /** + * Gets the invocation context used by the agent. + * + * @return The invocation context used by the agent + */ public PromptTemplate getTemplate() { return template; } - protected Mono ensureThreadExistsAsync(List> messages, AgentThread thread, Supplier constructor) { - return Mono.defer(() -> { - T newThread = thread != null ? (T) thread : constructor.get(); - - return newThread.createAsync() - .thenMany(Flux.fromIterable(messages)) - .concatMap(message -> { - return notifyThreadOfNewMessageAsync(newThread, message) - .then(Mono.just(message)); - }) - .then(Mono.just(newThread)); - }); - } - - @Override - public Mono notifyThreadOfNewMessageAsync(AgentThread thread, ChatMessageContent message) { - return Mono.defer(() -> { - return thread.onNewMessageAsync(message); - }); - } - /** * Merges the provided arguments with the current arguments. @@ -127,11 +141,27 @@ protected KernelArguments mergeArguments(KernelArguments arguments) { * @param context The context to use for formatting. * @return A Mono that resolves to the formatted instructions. */ - protected Mono formatInstructionsAsync(Kernel kernel, KernelArguments arguments, InvocationContext context) { + protected Mono renderInstructionsAsync(Kernel kernel, KernelArguments arguments, InvocationContext context) { if (template != null) { return template.renderAsync(kernel, arguments, context); } else { return Mono.just(instructions); } } + + protected Mono ensureThreadExistsWithMessagesAsync(List> messages, AgentThread thread, Supplier threadSupplier) { + return Mono.defer(() -> { + // Check if the thread already exists + // If it does, we can work with a copy of it + AgentThread newThread = thread == null ? threadSupplier.get() : thread.copy(); + + return newThread.createAsync() + .thenMany(Flux.fromIterable(messages)) + .concatMap(message -> { + return notifyThreadOfNewMessageAsync(newThread, message) + .then(Mono.just(message)); + }) + .then(Mono.just((T) newThread)); + }); + } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java index 9784648b..5769b5f1 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java @@ -24,6 +24,7 @@ */ public class ChatMessageContent extends KernelContentImpl { + private String authorName; private final AuthorRole authorRole; @Nullable private final String content; @@ -52,6 +53,28 @@ public ChatMessageContent( null); } + /** + * Creates a new instance of the {@link ChatMessageContent} class. Defaults to + * {@link ChatMessageContentType#TEXT} content type. + * + * @param authorRole the author role that generated the content + * @param authorName the author name + * @param content the content + */ + public ChatMessageContent( + AuthorRole authorRole, + String authorName, + String content) { + this( + authorRole, + authorName, + content, + null, + null, + null, + null); + } + /** * Creates a new instance of the {@link ChatMessageContent} class. Defaults to * {@link ChatMessageContentType#TEXT} content type. @@ -132,6 +155,27 @@ public ChatMessageContent( this.contentType = contentType; } + + /** + * Gets the author name that generated the content + * + * @return the author name that generated the content + */ + @Nullable + public String getAuthorName() { + return authorName; + } + + /** + * Sets the author name that generated the content + * + * @param authorName the author name that generated the content + */ + public void setAuthorName(String authorName) { + this.authorName = authorName; + } + + /** * Gets the author role that generated the content * From 9cfab020f575128f790cdaecf6591cfdba1a1583 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 24 Apr 2025 23:07:19 -0700 Subject: [PATCH 134/277] Remove openai package --- agents/semantickernel-agents-openai/pom.xml | 26 --------------------- 1 file changed, 26 deletions(-) delete mode 100644 agents/semantickernel-agents-openai/pom.xml diff --git a/agents/semantickernel-agents-openai/pom.xml b/agents/semantickernel-agents-openai/pom.xml deleted file mode 100644 index 815a0d5b..00000000 --- a/agents/semantickernel-agents-openai/pom.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - 4.0.0 - - com.microsoft.semantic-kernel - semantickernel-parent - 1.4.4-SNAPSHOT - - - semantickernel-agents-openai - - - 17 - 17 - UTF-8 - - - - com.microsoft.semantic-kernel - semantickernel-api - - - - \ No newline at end of file From afaa50ce8e018e2cbbf508d1335ee74f866c943f Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 24 Apr 2025 23:08:56 -0700 Subject: [PATCH 135/277] Remove openai package --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index f1536e87..0d31715d 100644 --- a/pom.xml +++ b/pom.xml @@ -77,7 +77,6 @@ data/semantickernel-data-azureaisearch data/semantickernel-data-jdbc data/semantickernel-data-redis - agents/semantickernel-agents-openai agents/semantickernel-agents-core From 3dcbb890cba4e23bec24c83aa4f3d7f1a425736c Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 24 Apr 2025 23:27:05 -0700 Subject: [PATCH 136/277] Add EI_EXPOSE_REP updates --- .../agents/AgentInvokeOptions.java | 32 ++++++++++++++++--- .../agents/AgentResponseItem.java | 14 ++++++++ .../semantickernel/agents/KernelAgent.java | 2 +- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java index d5a99b5a..00847d15 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java @@ -4,6 +4,7 @@ import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.orchestration.InvocationContext; import com.microsoft.semantickernel.semanticfunctions.KernelArguments; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.annotation.Nullable; @@ -12,7 +13,7 @@ */ public class AgentInvokeOptions { - private final KernelArguments KernelArguments; + private final KernelArguments kernelArguments; private final Kernel kernel; private final String additionalInstructions; private final InvocationContext invocationContext; @@ -27,33 +28,53 @@ public AgentInvokeOptions() { /** * Constructor for AgentInvokeOptions. * - * @param KernelArguments The arguments for the kernel function. + * @param kernelArguments The arguments for the kernel function. * @param kernel The kernel to use. * @param additionalInstructions Additional instructions for the agent. * @param invocationContext The invocation context. */ - public AgentInvokeOptions(@Nullable KernelArguments KernelArguments, + public AgentInvokeOptions(@Nullable KernelArguments kernelArguments, @Nullable Kernel kernel, @Nullable String additionalInstructions, @Nullable InvocationContext invocationContext) { - this.KernelArguments = KernelArguments; + this.kernelArguments = kernelArguments != null ? kernelArguments.copy() : null; this.kernel = kernel; this.additionalInstructions = additionalInstructions; this.invocationContext = invocationContext; } + /** + * Get the kernel arguments. + * + * @return The kernel arguments. + */ public KernelArguments getKernelArguments() { - return KernelArguments; + return kernelArguments; } + /** + * Get the kernel. + * + * @return The kernel. + */ public Kernel getKernel() { return kernel; } + /** + * Get additional instructions. + * + * @return The additional instructions. + */ public String getAdditionalInstructions() { return additionalInstructions; } + /** + * Get the invocation context. + * + * @return The invocation context. + */ public InvocationContext getInvocationContext() { return invocationContext; } @@ -80,6 +101,7 @@ public static class Builder implements SemanticKernelBuilder * @param kernelArguments The kernel arguments. * @return The builder. */ + @SuppressFBWarnings("EI_EXPOSE_REP2") public Builder withKernelArguments(KernelArguments kernelArguments) { this.kernelArguments = kernelArguments; return this; diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentResponseItem.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentResponseItem.java index 61ffdd6d..f585bfea 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentResponseItem.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentResponseItem.java @@ -1,18 +1,32 @@ package com.microsoft.semantickernel.agents; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + public class AgentResponseItem { private final T message; private final AgentThread thread; + @SuppressFBWarnings("EI_EXPOSE_REP2") public AgentResponseItem(T message, AgentThread thread) { this.message = message; this.thread = thread; } + /** + * Gets the agent response message. + * + * @return The message. + */ public T getMessage() { return message; } + /** + * Gets the thread. + * + * @return The thread. + */ + @SuppressFBWarnings("EI_EXPOSE_REP") public AgentThread getThread() { return thread; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java index e5f54a6d..d7051432 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java @@ -41,7 +41,7 @@ protected KernelAgent( this.name = name; this.description = description; this.kernel = kernel; - this.kernelArguments = kernelArguments; + this.kernelArguments = kernelArguments != null ? kernelArguments.copy() : null; this.invocationContext = invocationContext != null ? invocationContext : InvocationContext.builder().withReturnMode(InvocationReturnMode.FULL_HISTORY).build(); this.instructions = instructions; From 3c0f2557cdc96d1a289af7b7954cb7fdca508fb0 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 24 Apr 2025 23:29:27 -0700 Subject: [PATCH 137/277] Add EI_EXPOSE_REP updates --- .../com/microsoft/semantickernel/agents/AgentInvokeOptions.java | 1 + .../java/com/microsoft/semantickernel/agents/KernelAgent.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java index 00847d15..3fb4cba1 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java @@ -48,6 +48,7 @@ public AgentInvokeOptions(@Nullable KernelArguments kernelArguments, * * @return The kernel arguments. */ + @SuppressFBWarnings("EI_EXPOSE_REP") public KernelArguments getKernelArguments() { return kernelArguments; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java index d7051432..6731a927 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java @@ -7,6 +7,7 @@ import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.PromptTemplate; import com.microsoft.semantickernel.services.chatcompletion.ChatMessageContent; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -89,6 +90,7 @@ public Kernel getKernel() { * * @return The invocation context used by the agent */ + @SuppressFBWarnings("EI_EXPOSE_REP") public KernelArguments getKernelArguments() { return kernelArguments; } From fff6b90828cb15682496176e58da5082b3ffe1e7 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 24 Apr 2025 23:35:11 -0700 Subject: [PATCH 138/277] Add EI_EXPOSE_REP updates --- .../agents/chatcompletion/ChatCompletionAgent.java | 2 ++ .../agents/chatcompletion/ChatHistoryAgentThread.java | 3 +++ 2 files changed, 5 insertions(+) diff --git a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java index 2e9a0d7d..ba09bff5 100644 --- a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java +++ b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java @@ -19,6 +19,7 @@ import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; import com.microsoft.semantickernel.services.chatcompletion.ChatHistory; import com.microsoft.semantickernel.services.chatcompletion.ChatMessageContent; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; @@ -230,6 +231,7 @@ public Builder withKernel(Kernel kernel) { * * @param KernelArguments The kernel arguments to use. */ + @SuppressFBWarnings("EI_EXPOSE_REP2") public Builder withKernelArguments(KernelArguments KernelArguments) { this.kernelArguments = KernelArguments; return this; diff --git a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatHistoryAgentThread.java b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatHistoryAgentThread.java index 3aa68ea1..1a68f8c4 100644 --- a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatHistoryAgentThread.java +++ b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatHistoryAgentThread.java @@ -5,6 +5,7 @@ import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.services.chatcompletion.ChatHistory; import com.microsoft.semantickernel.services.chatcompletion.ChatMessageContent; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import reactor.core.publisher.Mono; import javax.annotation.Nonnull; @@ -35,6 +36,7 @@ public ChatHistoryAgentThread(String id, @Nullable ChatHistory chatHistory) { * * @return The chat history. */ + @SuppressFBWarnings("EI_EXPOSE_REP") public ChatHistory getChatHistory() { return chatHistory; } @@ -100,6 +102,7 @@ public Builder withId(String id) { * @param chatHistory The chat history. * @return The builder instance. */ + @SuppressFBWarnings("EI_EXPOSE_REP2") public Builder withChatHistory(ChatHistory chatHistory) { this.chatHistory = chatHistory; return this; From ae917651980f0397dd62bca64754d00fffcbfb36 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Fri, 25 Apr 2025 11:16:35 -0700 Subject: [PATCH 139/277] Make NEW_MESSAGES_ONLY the default for kernel agent invocation context --- .../com/microsoft/semantickernel/agents/KernelAgent.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java index 6731a927..71e81951 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java @@ -42,9 +42,10 @@ protected KernelAgent( this.name = name; this.description = description; this.kernel = kernel; - this.kernelArguments = kernelArguments != null ? kernelArguments.copy() : null; + this.kernelArguments = kernelArguments != null + ? kernelArguments.copy() : KernelArguments.builder().build(); this.invocationContext = invocationContext != null - ? invocationContext : InvocationContext.builder().withReturnMode(InvocationReturnMode.FULL_HISTORY).build(); + ? invocationContext : InvocationContext.builder().build(); this.instructions = instructions; this.template = template; } From ba1aa96d690d59933fa0f5b86557de221a73c69a Mon Sep 17 00:00:00 2001 From: psilberk Date: Fri, 25 Apr 2025 16:20:54 -0700 Subject: [PATCH 140/277] Upsert and Main sample --- data/semantickernel-data-jdbc/pom.xml | 5 + .../OracleVectorStoreQueryProvider.java | 61 +++++- pom.xml | 1 + .../semantickernel-syntax-examples/pom.xml | 11 + .../memory/VectorStoreWithOracle.java | 200 +++--------------- .../data/vectorstores/oracle/Main.java | 72 +++++++ 6 files changed, 171 insertions(+), 179 deletions(-) create mode 100644 samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 624779e9..1f2e1a28 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -63,5 +63,10 @@ sqlite-jdbc 3.47.0.0 + + com.oracle.database.jdbc + ojdbc11 + 23.7.0.25.01 + \ No newline at end of file diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index f179d0be..10af511c 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -1,6 +1,9 @@ package com.microsoft.semantickernel.data.jdbc.oracle; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; @@ -18,9 +21,14 @@ public class OracleVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider // This could be removed if super.collectionTable made protected private final String collectionsTable; - private OracleVectorStoreQueryProvider(@Nonnull DataSource dataSource, @Nonnull String collectionsTable, @Nonnull String prefixForCollectionTables) { + // This could be common to all query providers + private final ObjectMapper objectMapper; + + private OracleVectorStoreQueryProvider(@Nonnull DataSource dataSource, @Nonnull String collectionsTable, @Nonnull String prefixForCollectionTables, + ObjectMapper objectMapper) { super(dataSource, collectionsTable, prefixForCollectionTables); this.collectionsTable = collectionsTable; + this.objectMapper = objectMapper; } @Override @@ -37,13 +45,50 @@ public void prepareVectorStore() { } } + @Override + public void createCollection(String collectionName, + VectorStoreRecordDefinition recordDefinition) { + // TODO Override implementation. Eg: mapping TEXT to VARCHAR + super.createCollection(collectionName, recordDefinition); + } + @Override public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { - // Using hsqldb impl + // TODO look for public void createCollection(String collectionName, VectorStoreRecordDefinition recordDefinition) { + + // TODO Make this a MERGE query + +// String upsertStatemente = formatQuery(""" +// MERGE INTO %s EXIST_REC USING (SELECT ? AS ID) NEW_REC ON (EXIST_REC.%s = NEW_REC.ID) +// WHEN MATACHED THEN UPDATE SET EXISTING REC +// """, +// getCollectionTableName(collectionName), +// recordDefinition.getKeyField().getName(), +// getQueryColumnsFromFields(fields), +// getWildcardString(fields.size()), +// onDuplicateKeyUpdate);super.upsertRecords(collectionName, records, recordDefinition, options); + String query = formatQuery("INSERT INTO %s (%s, %s, %s) values (?, ?, ?)", + getCollectionTableName(collectionName), + recordDefinition.getAllFields().get(0).getStorageName(), + recordDefinition.getAllFields().get(1).getStorageName(), + recordDefinition.getAllFields().get(2).getStorageName()); - super.upsertRecords(collectionName, records, recordDefinition, options); + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(query)) { + for (Object record : records) { + JsonNode jsonNode = objectMapper.valueToTree(record); + for (int i = 0; i < 3; i++) { + statement.setObject(i + 1, jsonNode + .get(recordDefinition.getAllFields().get(i).getStorageName()).asText()); + } + statement.addBatch(); + } + statement.executeBatch(); + } catch (SQLException e) { + throw new SKException("Failed to upsert records", e); + } } public static Builder builder() { @@ -56,6 +101,7 @@ public static class Builder private DataSource dataSource; private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; + private ObjectMapper objectMapper = new ObjectMapper(); @SuppressFBWarnings("EI_EXPOSE_REP2") public Builder withDataSource(DataSource dataSource) { @@ -83,9 +129,16 @@ public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { return this; } + public Builder withObjectMapper( + ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + return this; + } + @Override public OracleVectorStoreQueryProvider build() { - return new OracleVectorStoreQueryProvider(dataSource, collectionsTable, prefixForCollectionTables); + return new OracleVectorStoreQueryProvider(dataSource, collectionsTable, + prefixForCollectionTables, objectMapper); } } } \ No newline at end of file diff --git a/pom.xml b/pom.xml index 30963f54..e12980a2 100644 --- a/pom.xml +++ b/pom.xml @@ -71,6 +71,7 @@ semantickernel-bom semantickernel-api semantickernel-experimental + samples aiservices/openai aiservices/google aiservices/huggingface diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 1f6bc181..1abf1174 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -131,6 +131,17 @@ com.github.victools jsonschema-module-jackson + + com.microsoft.semantic-kernel + semantickernel-data-jdbc + 1.4.4-SNAPSHOT + + + com.microsoft.semantic-kernel + semantickernel-learn-resources + 1.4.4-SNAPSHOT + compile + diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java index 9d142507..3d037c70 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java @@ -1,210 +1,60 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.samples.syntaxexamples.memory; -import com.azure.ai.openai.OpenAIAsyncClient; -import com.azure.ai.openai.OpenAIClientBuilder; -import com.azure.core.credential.AzureKeyCredential; -import com.azure.core.credential.KeyCredential; -import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollection; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; +import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; -import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; -import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; -import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; -import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; - -import java.nio.charset.StandardCharsets; +import com.microsoft.semantickernel.samples.documentationexamples.data.index.Hotel; import java.sql.SQLException; -import java.util.Arrays; -import java.util.Base64; import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import org.postgresql.ds.PGSimpleDataSource; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; +import oracle.jdbc.datasource.impl.OracleDataSource; public class VectorStoreWithOracle { - static class GitHubFile { - @VectorStoreRecordKey - private final String id; - @VectorStoreRecordData - private final String description; - @VectorStoreRecordData - private final String link; - @VectorStoreRecordVector(dimensions = EMBEDDING_DIMENSIONS, distanceFunction = DistanceFunction.COSINE_DISTANCE) - private final List embedding; - - public GitHubFile() { - this(null, null, null, Collections.emptyList()); - } - - public GitHubFile( - String id, - String description, - String link, - List embedding) { - this.id = id; - this.description = description; - this.link = link; - this.embedding = embedding; - } - - public String getId() { - return id; - } - - public String getDescription() { - return description; - } - - public String getLink() { - return link; - } - - public List getEmbedding() { - return embedding; - } - - static String encodeId(String realId) { - byte[] bytes = Base64.getUrlEncoder().encode(realId.getBytes(StandardCharsets.UTF_8)); - return new String(bytes, StandardCharsets.UTF_8); - } - } - - // Run a PostgreSQL server with: - // docker run -d --name pgvector-container -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=root -e POSTGRES_DB=sk -p 5432:5432 pgvector/pgvector:pg17 - public static void main(String[] args) throws SQLException { System.out.println("=============================================================="); System.out.println("============== Oracle Vector Store Example ==================="); System.out.println("=============================================================="); - OpenAIAsyncClient client; - - if (AZURE_CLIENT_KEY != null) { - client = new OpenAIClientBuilder() - .credential(new AzureKeyCredential(AZURE_CLIENT_KEY)) - .endpoint(CLIENT_ENDPOINT) - .buildAsyncClient(); - - } else { - client = new OpenAIClientBuilder() - .credential(new KeyCredential(CLIENT_KEY)) - .buildAsyncClient(); - } - - // Create an OpenAI text embedding generation service - var embeddingGeneration = OpenAITextEmbeddingGenerationService.builder() - .withOpenAIAsyncClient(client) - .withModelId(MODEL_ID) - .withDimensions(EMBEDDING_DIMENSIONS) - .build(); - - storeAndSearch(embeddingGeneration); - } - - public static void storeAndSearch(OpenAITextEmbeddingGenerationService embeddingGeneration) { // Configure the data source - PGSimpleDataSource dataSource = new PGSimpleDataSource(); - dataSource.setUrl("jdbc:postgresql://localhost:5432/sk"); - dataSource.setUser("postgres"); - dataSource.setPassword("root"); + OracleDataSource dataSource = new OracleDataSource(); + dataSource.setURL("jdbc:oracle:thin:@localhost:1521/FREEPDB1"); + dataSource.setUser("scott"); + dataSource.setPassword("tiger"); // Build a query provider - // Other available query providers are PostgreSQLVectorStoreQueryProvider and SQLiteVectorStoreQueryProvider - var queryProvider = PostgreSQLVectorStoreQueryProvider.builder() + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() .withDataSource(dataSource) .build(); // Build a vector store - var jdbcVectorStore = JDBCVectorStore.builder() + JDBCVectorStore vectorStore = JDBCVectorStore.builder() .withDataSource(dataSource) .withOptions(JDBCVectorStoreOptions.builder() .withQueryProvider(queryProvider) .build()) .build(); - // Set up the record collection to use - String collectionName = "skgithubfiles"; - var collection = jdbcVectorStore.getCollection(collectionName, - JDBCVectorStoreRecordCollectionOptions.builder() - .withRecordClass(GitHubFile.class) - .build()); - - // Create collection if it does not exist and store data - collection - .createCollectionIfNotExistsAsync() - .then(storeData(collection, embeddingGeneration, sampleData())) + // Get a collection from the vector store + VectorStoreRecordCollection collection = + vectorStore.getCollection("skhotels", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); + + // Create the collection if it doesn't exist yet. + collection.createCollectionAsync().block(); + + collection.upsertAsync(new Hotel("1", + "HotelOne", + "Desc for HotelOne", + Collections.emptyList(), Collections.emptyList()), + null) .block(); - // Search for results - var results = search("How to get started", collection, embeddingGeneration).block(); - - if (results == null || results.getTotalCount() == 0) { - System.out.println("No search results found."); - return; - } - var searchResult = results.getResults().get(0); - System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n", - searchResult.getScore(), searchResult.getRecord().link, - searchResult.getRecord().description); } - private static Mono> search( - String searchText, - VectorStoreRecordCollection recordCollection, - OpenAITextEmbeddingGenerationService embeddingGeneration) { - // Generate embeddings for the search text and search for the closest records - return embeddingGeneration.generateEmbeddingAsync(searchText) - .flatMap(r -> recordCollection.searchAsync(r.getVector(), null)); - } - - private static Mono> storeData( - VectorStoreRecordCollection recordStore, - OpenAITextEmbeddingGenerationService embeddingGeneration, - Map data) { - - return Flux.fromIterable(data.entrySet()) - .flatMap(entry -> { - System.out.println("Save '" + entry.getKey() + "' to memory."); - - // Generate embeddings for the data and store it - return embeddingGeneration - .generateEmbeddingsAsync(Collections.singletonList(entry.getValue())) - .flatMap(embeddings -> { - GitHubFile gitHubFile = new GitHubFile( - GitHubFile.encodeId(entry.getKey()), - entry.getValue(), - entry.getKey(), - embeddings.get(0).getVector()); - return recordStore.upsertAsync(gitHubFile, null); - }); - }) - .collectList(); - } - - private static Map sampleData() { - return Arrays.stream(new String[][] { - { "https://github.com/microsoft/semantic-kernel/blob/main/README.md", - "README: Installation, getting started with Semantic Kernel, and how to contribute" }, - { "https://github.com/microsoft/semantic-kernel/blob/main/samples/notebooks/dotnet/02-running-prompts-from-file.ipynb", - "Jupyter notebook describing how to pass prompts from a file to a semantic skill or function" }, - { "https://github.com/microsoft/semantic-kernel/tree/main/samples/skills/ChatSkill/ChatGPT", - "Sample demonstrating how to create a chat skill interfacing with ChatGPT" }, - { "https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel/Memory/VolatileMemoryStore.cs", - "C# class that defines a volatile embedding store" }, - { "https://github.com/microsoft/semantic-kernel/blob/main/samples/dotnet/KernelHttpServer/README.md", - "README: How to set up a Semantic Kernel Service API using Azure Function Runtime v4" }, - { "https://github.com/microsoft/semantic-kernel/blob/main/samples/apps/chat-summary-webapp-react/README.md", - "README: README associated with a sample chat summary react-based webapp" }, - }).collect(Collectors.toMap(element -> element[0], element -> element[1])); - } } diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java new file mode 100644 index 00000000..7a7b2b8f --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.documentationexamples.data.vectorstores.oracle; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.samples.documentationexamples.data.index.Hotel; +import oracle.jdbc.datasource.impl.OracleDataSource; +import java.sql.SQLException; +import java.util.Collections; + +public class Main { + public static void main(String[] args) throws SQLException { + + // Configure the data source + OracleDataSource dataSource = new OracleDataSource(); + dataSource.setURL("jdbc:oracle:thin:@localhost:1521/FREEPDB1"); + dataSource.setUser("scott"); + dataSource.setPassword("tiger"); + + // Build a query provider + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build(); + + // Build a vector store + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(dataSource) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + // Get a collection from the vector store + VectorStoreRecordCollection collection = + vectorStore.getCollection("skhotels", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); + + // Create the collection if it doesn't exist yet. + // TODO Override implementation to map TEXT to VARCHAR + // Craeted manually for the moment + //collection.createCollectionAsync().block(); + + // Upsert a record. + collection.upsertAsync(new Hotel("1", + "HotelOne", + "My Description for HotelOne", + Collections.emptyList(), Collections.emptyList()), + null) + .block(); + + // Retrieve the upserted record. +// var retrievedHotel = collection.getAsync("1", null).block(); + + // Generate a vector for your search text, using your chosen embedding generation implementation. + // Just showing a placeholder method here for brevity. +// var searchVector = generateEmbeddingsAsync( +// "I'm looking for a hotel where customer happiness is the priority.").block(); + + // Do the search. +// var searchResult = collection.searchAsync(searchVector, VectorSearchOptions.builder() +// .withTop(1).build()).block(); + +// Hotel record = searchResult.getResults().get(0).getRecord(); +// System.out.printf("Found hotel description: %s\n", record.getDescription()); + + } +} From c004ae05fb658f035b77848e28072cbad59db829 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Mon, 28 Apr 2025 09:54:47 -0700 Subject: [PATCH 141/277] Return Mono.error instead of runtime exception --- .../agents/chatcompletion/ChatCompletionAgent.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java index ba09bff5..82a64334 100644 --- a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java +++ b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java @@ -157,7 +157,7 @@ private Mono>> internalInvokeAsync( ); } catch (ServiceNotFoundException e) { - throw new RuntimeException(e); + return Mono.error(e); } } From 03bd7ed72dc51a6173f0ec8375807a746132c3e1 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Mon, 28 Apr 2025 11:36:09 -0700 Subject: [PATCH 142/277] Remove authorName from ChatMessageContent --- .../chatcompletion/ChatCompletionAgent.java | 7 +----- .../chatcompletion/ChatMessageContent.java | 23 ------------------- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java index 82a64334..f8423f3d 100644 --- a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java +++ b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java @@ -80,12 +80,7 @@ public Mono>>> invokeAsync( ) .flatMapMany(Flux::fromIterable) // notify on the new thread instance - .concatMap(agentMessage -> { - // Set the author name for the message - agentMessage.setAuthorName(this.name); - - return this.notifyThreadOfNewMessageAsync(agentThread, agentMessage).thenReturn(agentMessage); - }) + .concatMap(agentMessage -> this.notifyThreadOfNewMessageAsync(agentThread, agentMessage).thenReturn(agentMessage)) .collectList() .map(chatMessageContents -> chatMessageContents.stream() diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java index 5769b5f1..0408860e 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java @@ -23,8 +23,6 @@ * @param the type of the inner content within the messages */ public class ChatMessageContent extends KernelContentImpl { - - private String authorName; private final AuthorRole authorRole; @Nullable private final String content; @@ -155,27 +153,6 @@ public ChatMessageContent( this.contentType = contentType; } - - /** - * Gets the author name that generated the content - * - * @return the author name that generated the content - */ - @Nullable - public String getAuthorName() { - return authorName; - } - - /** - * Sets the author name that generated the content - * - * @param authorName the author name that generated the content - */ - public void setAuthorName(String authorName) { - this.authorName = authorName; - } - - /** * Gets the author role that generated the content * From a19b09fddd661f4d43c8f868be9c009e2d7ec512 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Mon, 28 Apr 2025 11:36:09 -0700 Subject: [PATCH 143/277] Remove authorName from ChatMessageContent --- .../chatcompletion/ChatCompletionAgent.java | 7 +----- .../chatcompletion/ChatMessageContent.java | 23 ------------------- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java index 82a64334..f8423f3d 100644 --- a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java +++ b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java @@ -80,12 +80,7 @@ public Mono>>> invokeAsync( ) .flatMapMany(Flux::fromIterable) // notify on the new thread instance - .concatMap(agentMessage -> { - // Set the author name for the message - agentMessage.setAuthorName(this.name); - - return this.notifyThreadOfNewMessageAsync(agentThread, agentMessage).thenReturn(agentMessage); - }) + .concatMap(agentMessage -> this.notifyThreadOfNewMessageAsync(agentThread, agentMessage).thenReturn(agentMessage)) .collectList() .map(chatMessageContents -> chatMessageContents.stream() diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java index 5769b5f1..0408860e 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java @@ -23,8 +23,6 @@ * @param the type of the inner content within the messages */ public class ChatMessageContent extends KernelContentImpl { - - private String authorName; private final AuthorRole authorRole; @Nullable private final String content; @@ -155,27 +153,6 @@ public ChatMessageContent( this.contentType = contentType; } - - /** - * Gets the author name that generated the content - * - * @return the author name that generated the content - */ - @Nullable - public String getAuthorName() { - return authorName; - } - - /** - * Sets the author name that generated the content - * - * @param authorName the author name that generated the content - */ - public void setAuthorName(String authorName) { - this.authorName = authorName; - } - - /** * Gets the author role that generated the content * From f1980c2f6b56a9d1b1327f3d766dc0c6fea74f5c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Apr 2025 02:06:55 +0000 Subject: [PATCH 144/277] Bump org.springframework.boot:spring-boot Bumps [org.springframework.boot:spring-boot](https://github.com/spring-projects/spring-boot) from 3.3.1 to 3.3.11. - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v3.3.1...v3.3.11) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot dependency-version: 3.3.11 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- .../semantickernel-demos/semantickernel-spring-starter/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index fd9822eb..bea07af9 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -56,7 +56,7 @@ org.springframework.boot spring-boot - 3.3.1 + 3.3.11 org.springframework From 9b821db80e71b751a50b9880087fe73acdeaed56 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 29 Apr 2025 09:46:34 -0700 Subject: [PATCH 145/277] Add FunctionChoiceBehavior implementation --- .../chatcompletion/ChatCompletionAgent.java | 7 +- .../chatcompletion/OpenAIChatCompletion.java | 241 +++++++++++++----- .../chatcompletion/OpenAIToolCallConfig.java | 70 +++++ .../agents/CompletionAgent.java | 5 +- .../Example59_OpenAIFunctionCalling.java | 11 +- .../AutoFunctionChoiceBehavior.java | 39 +++ .../FunctionChoiceBehavior.java | 162 ++++++++++++ .../FunctionChoiceBehaviorOptions.java | 49 ++++ .../NoneFunctionChoiceBehavior.java | 15 ++ .../RequiredFunctionChoiceBehavior.java | 19 ++ .../orchestration/FunctionInvocation.java | 29 +++ .../orchestration/InvocationContext.java | 45 +++- .../services/TextAIService.java | 2 +- 13 files changed, 611 insertions(+), 83 deletions(-) create mode 100644 aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIToolCallConfig.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/AutoFunctionChoiceBehavior.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehaviorOptions.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/NoneFunctionChoiceBehavior.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/RequiredFunctionChoiceBehavior.java diff --git a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java index f8423f3d..7985f5ad 100644 --- a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java +++ b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java @@ -110,19 +110,16 @@ private Mono>> internalInvokeAsync( ? invocationContext.getPromptExecutionSettings() : kernelArguments.getExecutionSettings().get(chatCompletionService.getServiceId()); - ToolCallBehavior toolCallBehavior = invocationContext != null - ? invocationContext.getToolCallBehavior() - : ToolCallBehavior.allowAllKernelFunctions(true); - // Build base invocation context InvocationContext.Builder builder = InvocationContext.builder() .withPromptExecutionSettings(executionSettings) - .withToolCallBehavior(toolCallBehavior) .withReturnMode(InvocationReturnMode.NEW_MESSAGES_ONLY); if (invocationContext != null) { builder = builder .withTelemetry(invocationContext.getTelemetry()) + .withFunctionChoiceBehavior(invocationContext.getFunctionChoiceBehavior()) + .withToolCallBehavior(invocationContext.getToolCallBehavior()) .withContextVariableConverter(invocationContext.getContextVariableTypes()) .withKernelHooks(invocationContext.getKernelHooks()); } diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java index 33f46fc2..0d0dc304 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java @@ -44,6 +44,10 @@ import com.microsoft.semantickernel.exceptions.AIException.ErrorCodes; import com.microsoft.semantickernel.exceptions.SKCheckedException; import com.microsoft.semantickernel.exceptions.SKException; +import com.microsoft.semantickernel.functionchoice.AutoFunctionChoiceBehavior; +import com.microsoft.semantickernel.functionchoice.FunctionChoiceBehavior; +import com.microsoft.semantickernel.functionchoice.NoneFunctionChoiceBehavior; +import com.microsoft.semantickernel.functionchoice.RequiredFunctionChoiceBehavior; import com.microsoft.semantickernel.hooks.KernelHookEvent; import com.microsoft.semantickernel.hooks.KernelHooks; import com.microsoft.semantickernel.hooks.PostChatCompletionEvent; @@ -196,10 +200,20 @@ public Flux> getStreamingChatMessageContentsAsync( ChatHistory chatHistory, @Nullable Kernel kernel, @Nullable InvocationContext invocationContext) { - if (invocationContext != null && invocationContext.getToolCallBehavior() - .isAutoInvokeAllowed()) { + if (invocationContext != null && + invocationContext.getToolCallBehavior() != null && + invocationContext.getToolCallBehavior().isAutoInvokeAllowed()) { throw new SKException( - "Auto invoke is not supported for streaming chat message contents"); + "ToolCallBehavior auto-invoke is not supported for streaming chat message contents"); + } + + if (invocationContext != null && + invocationContext.getFunctionChoiceBehavior() != null && + invocationContext.getFunctionChoiceBehavior() instanceof AutoFunctionChoiceBehavior && + ((AutoFunctionChoiceBehavior) invocationContext.getFunctionChoiceBehavior()).isAutoInvoke() + ) { + throw new SKException( + "FunctionChoiceBehavior auto-invoke is not supported for streaming chat message contents"); } if (invocationContext != null @@ -219,6 +233,12 @@ public Flux> getStreamingChatMessageContentsAsync( .add(OpenAIFunction.build(function.getMetadata(), plugin.getName())))); } + OpenAIToolCallConfig toolCallConfig = getToolCallConfig( + invocationContext, + functions, + messages.allMessages, + 0); + ChatCompletionsOptions options = executeHook( invocationContext, kernel, @@ -226,8 +246,8 @@ public Flux> getStreamingChatMessageContentsAsync( getCompletionsOptions( this, messages.allMessages, - functions, - invocationContext))) + invocationContext, + toolCallConfig))) .getOptions(); return getClient() @@ -389,16 +409,12 @@ private Mono internalChatMessageContentsAsync( .add(OpenAIFunction.build(function.getMetadata(), plugin.getName())))); } - // Create copy to avoid reactor exceptions when updating request messages internally return internalChatMessageContentsAsync( messages, kernel, functions, invocationContext, - Math.min(MAXIMUM_INFLIGHT_AUTO_INVOKES, - invocationContext != null && invocationContext.getToolCallBehavior() != null - ? invocationContext.getToolCallBehavior().getMaximumAutoInvokeAttempts() - : 0)); + 0); } private Mono internalChatMessageContentsAsync( @@ -406,7 +422,13 @@ private Mono internalChatMessageContentsAsync( @Nullable Kernel kernel, List functions, @Nullable InvocationContext invocationContext, - int autoInvokeAttempts) { + int requestIndex) { + + OpenAIToolCallConfig toolCallConfig = getToolCallConfig( + invocationContext, + functions, + messages.allMessages, + requestIndex); ChatCompletionsOptions options = executeHook( invocationContext, @@ -415,8 +437,8 @@ private Mono internalChatMessageContentsAsync( getCompletionsOptions( this, messages.allMessages, - functions, - invocationContext))) + invocationContext, + toolCallConfig))) .getOptions(); return Mono.deferContextual(contextView -> { @@ -458,9 +480,9 @@ private Mono internalChatMessageContentsAsync( executeHook(invocationContext, kernel, new PostChatCompletionEvent(completions)); // Just return the result: - // If we don't want to attempt to invoke any functions + // If auto-invoking is not enabled // Or if we are auto-invoking, but we somehow end up with other than 1 choice even though only 1 was requested - if (autoInvokeAttempts == 0 || responseMessages.size() != 1) { + if (toolCallConfig == null || !toolCallConfig.isAutoInvoke() || responseMessages.size() != 1) { List> chatMessageContents = getChatMessageContentsAsync( completions); return Mono.just(messages.addChatMessage(chatMessageContents)); @@ -497,14 +519,14 @@ private Mono internalChatMessageContentsAsync( .flatMap(it -> it) .flatMap(msgs -> { return internalChatMessageContentsAsync(msgs, kernel, functions, - invocationContext, autoInvokeAttempts - 1); + invocationContext, requestIndex + 1); }) .onErrorResume(e -> { LOGGER.warn("Tool invocation attempt failed: ", e); // If FunctionInvocationError occurred and there are still attempts left, retry, else exit - if (autoInvokeAttempts > 0) { + if (requestIndex > 0) { ChatMessages currentMessages = messages; if (e instanceof FunctionInvocationError) { currentMessages.assertCommonHistory( @@ -518,7 +540,7 @@ private Mono internalChatMessageContentsAsync( kernel, functions, invocationContext, - autoInvokeAttempts - 1); + requestIndex + 1); } else { return Mono.error(e); } @@ -860,8 +882,8 @@ private List formOpenAiToolCalls( private static ChatCompletionsOptions getCompletionsOptions( ChatCompletionService chatCompletionService, List chatRequestMessages, - @Nullable List functions, - @Nullable InvocationContext invocationContext) { + @Nullable InvocationContext invocationContext, + @Nullable OpenAIToolCallConfig toolCallConfig) { chatRequestMessages = chatRequestMessages .stream() @@ -871,12 +893,13 @@ private static ChatCompletionsOptions getCompletionsOptions( ChatCompletionsOptions options = new ChatCompletionsOptions(chatRequestMessages) .setModel(chatCompletionService.getModelId()); - if (invocationContext != null && invocationContext.getToolCallBehavior() != null) { - configureToolCallBehaviorOptions( - options, - invocationContext.getToolCallBehavior(), - functions, - chatRequestMessages); + if (toolCallConfig != null) { + options.setTools(toolCallConfig.getTools()); + options.setToolChoice(toolCallConfig.getToolChoice()); + + if (toolCallConfig.getOptions() != null) { + options.setParallelToolCalls(toolCallConfig.getOptions().isParallelCallsAllowed()); + } } PromptExecutionSettings promptExecutionSettings = invocationContext != null @@ -946,92 +969,176 @@ private static ChatCompletionsOptions getCompletionsOptions( return options; } - private static void configureToolCallBehaviorOptions( - ChatCompletionsOptions options, + private static OpenAIToolCallConfig getToolCallConfig( + @Nullable InvocationContext invocationContext, + @Nullable List functions, + List chatRequestMessages, + int requestIndex) { + + if (invocationContext == null || functions == null || functions.isEmpty()) { + return null; + } + + if (invocationContext.getFunctionChoiceBehavior() == null && invocationContext.getToolCallBehavior() == null) { + return null; + } + + + if (invocationContext.getFunctionChoiceBehavior() != null) { + return getFunctionChoiceBehaviorConfig( + invocationContext.getFunctionChoiceBehavior(), + functions, + requestIndex); + } else { + return getToolCallBehaviorConfig( + invocationContext.getToolCallBehavior(), + functions, + chatRequestMessages, + requestIndex); + } + } + + private static OpenAIToolCallConfig getFunctionChoiceBehaviorConfig( + @Nullable FunctionChoiceBehavior functionChoiceBehavior, + @Nullable List functions, + int requestIndex) { + if (functionChoiceBehavior == null) { + return null; + } + + if (functions == null || functions.isEmpty()) { + return null; + } + + ChatCompletionsToolSelection toolChoice; + boolean autoInvoke; + + if (functionChoiceBehavior instanceof RequiredFunctionChoiceBehavior) { + // After first request a required function must have been called already + if (requestIndex >= 1) { + return null; + } + + toolChoice = new ChatCompletionsToolSelection(ChatCompletionsToolSelectionPreset.REQUIRED); + autoInvoke = ((RequiredFunctionChoiceBehavior) functionChoiceBehavior).isAutoInvoke(); + } else if (functionChoiceBehavior instanceof AutoFunctionChoiceBehavior) { + toolChoice = new ChatCompletionsToolSelection(ChatCompletionsToolSelectionPreset.AUTO); + autoInvoke = ((AutoFunctionChoiceBehavior) functionChoiceBehavior).isAutoInvoke() && requestIndex < MAXIMUM_INFLIGHT_AUTO_INVOKES; + } else if (functionChoiceBehavior instanceof NoneFunctionChoiceBehavior) { + toolChoice = new ChatCompletionsToolSelection(ChatCompletionsToolSelectionPreset.NONE); + autoInvoke = false; + } else { + throw new SKException( + "Unsupported function choice behavior: " + functionChoiceBehavior); + } + + // List of functions advertised to the model + List toolDefinitions = functions.stream() + .filter(function -> functionChoiceBehavior.isFunctionAllowed(function.getPluginName(), function.getName())) + .map(OpenAIFunction::getFunctionDefinition) + .map(it -> new ChatCompletionsFunctionToolDefinitionFunction(it.getName()) + .setDescription(it.getDescription()) + .setParameters(it.getParameters())) + .map(ChatCompletionsFunctionToolDefinition::new) + .collect(Collectors.toList()); + + return new OpenAIToolCallConfig( + toolDefinitions, + toolChoice, + autoInvoke, + functionChoiceBehavior.getOptions()); + } + + private static OpenAIToolCallConfig getToolCallBehaviorConfig( @Nullable ToolCallBehavior toolCallBehavior, @Nullable List functions, - List chatRequestMessages) { + List chatRequestMessages, + int requestIndex) { if (toolCallBehavior == null) { - return; + return null; } if (functions == null || functions.isEmpty()) { - return; + return null; } + List toolDefinitions; + ChatCompletionsToolSelection toolChoice; + // If a specific function is required to be called if (toolCallBehavior instanceof ToolCallBehavior.RequiredKernelFunction) { - KernelFunction toolChoice = ((ToolCallBehavior.RequiredKernelFunction) toolCallBehavior) + KernelFunction requiredFunction = ((ToolCallBehavior.RequiredKernelFunction) toolCallBehavior) .getRequiredFunction(); String toolChoiceName = String.format("%s%s%s", - toolChoice.getPluginName(), + requiredFunction.getPluginName(), OpenAIFunction.getNameSeparator(), - toolChoice.getName()); + requiredFunction.getName()); // If required tool call has already been called dont ask for it again boolean hasBeenExecuted = hasToolCallBeenExecuted(chatRequestMessages, toolChoiceName); if (hasBeenExecuted) { - return; + return null; } - List toolDefinitions = new ArrayList<>(); - FunctionDefinition function = OpenAIFunction.toFunctionDefinition( - toolChoice.getMetadata(), - toolChoice.getPluginName()); + requiredFunction.getMetadata(), + requiredFunction.getPluginName()); + toolDefinitions = new ArrayList<>(); toolDefinitions.add(new ChatCompletionsFunctionToolDefinition( new ChatCompletionsFunctionToolDefinitionFunction(function.getName()) .setDescription(function.getDescription()) .setParameters(function.getParameters()))); - options.setTools(toolDefinitions); try { String json = String.format( "{\"type\":\"function\",\"function\":{\"name\":\"%s\"}}", toolChoiceName); - options.setToolChoice( - new ChatCompletionsToolSelection( + toolChoice = new ChatCompletionsToolSelection( ChatCompletionsNamedToolSelection.fromJson( DefaultJsonReader.fromString( json, - new JsonOptions())))); + new JsonOptions()))); } catch (JsonProcessingException e) { throw SKException.build("Failed to parse tool choice", e); } catch (IOException e) { throw new SKException(e); } - return; } - // If a set of functions are enabled to be called - ToolCallBehavior.AllowedKernelFunctions enabledKernelFunctions = (ToolCallBehavior.AllowedKernelFunctions) toolCallBehavior; - List toolDefinitions = functions.stream() - .filter(function -> { - // check if all kernel functions are enabled - if (enabledKernelFunctions.isAllKernelFunctionsAllowed()) { - return true; - } - // otherwise, check for the specific function - return enabledKernelFunctions.isFunctionAllowed(function.getPluginName(), - function.getName()); - }) - .map(OpenAIFunction::getFunctionDefinition) - .map(it -> new ChatCompletionsFunctionToolDefinitionFunction(it.getName()) - .setDescription(it.getDescription()) - .setParameters(it.getParameters())) - .map(it -> new ChatCompletionsFunctionToolDefinition(it)) - .collect(Collectors.toList()); + else { + toolChoice = new ChatCompletionsToolSelection(ChatCompletionsToolSelectionPreset.AUTO); + + ToolCallBehavior.AllowedKernelFunctions enabledKernelFunctions = (ToolCallBehavior.AllowedKernelFunctions) toolCallBehavior; + toolDefinitions = functions.stream() + .filter(function -> { + // check if all kernel functions are enabled + if (enabledKernelFunctions.isAllKernelFunctionsAllowed()) { + return true; + } + // otherwise, check for the specific function + return enabledKernelFunctions.isFunctionAllowed(function.getPluginName(), + function.getName()); + }) + .map(OpenAIFunction::getFunctionDefinition) + .map(it -> new ChatCompletionsFunctionToolDefinitionFunction(it.getName()) + .setDescription(it.getDescription()) + .setParameters(it.getParameters())) + .map(ChatCompletionsFunctionToolDefinition::new) + .collect(Collectors.toList()); - if (toolDefinitions.isEmpty()) { - return; + if (toolDefinitions.isEmpty()) { + return null; + } } - options.setTools(toolDefinitions); - options.setToolChoice( - new ChatCompletionsToolSelection(ChatCompletionsToolSelectionPreset.AUTO)); + return new OpenAIToolCallConfig( + toolDefinitions, + toolChoice, + toolCallBehavior.isAutoInvokeAllowed() && requestIndex < Math.min(MAXIMUM_INFLIGHT_AUTO_INVOKES, toolCallBehavior.getMaximumAutoInvokeAttempts()), + null); } private static boolean hasToolCallBeenExecuted(List chatRequestMessages, diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIToolCallConfig.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIToolCallConfig.java new file mode 100644 index 00000000..895252d3 --- /dev/null +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIToolCallConfig.java @@ -0,0 +1,70 @@ +package com.microsoft.semantickernel.aiservices.openai.chatcompletion; + +import com.azure.ai.openai.models.ChatCompletionsToolDefinition; +import com.azure.ai.openai.models.ChatCompletionsToolSelection; +import com.microsoft.semantickernel.functionchoice.FunctionChoiceBehaviorOptions; + +import javax.annotation.Nullable; +import java.util.List; + +public class OpenAIToolCallConfig { + private final List tools; + private final ChatCompletionsToolSelection toolChoice; + private final boolean autoInvoke; + private final FunctionChoiceBehaviorOptions options; + + /** + * Creates a new instance of the {@link OpenAIToolCallConfig} class. + * + * @param tools The list of tools available for the call. + * @param toolChoice The tool selection strategy. + * @param autoInvoke Indicates whether to automatically invoke the tool. + * @param options Additional options for function choice behavior. + */ + public OpenAIToolCallConfig( + List tools, + ChatCompletionsToolSelection toolChoice, + boolean autoInvoke, + @Nullable FunctionChoiceBehaviorOptions options) { + this.tools = tools; + this.toolChoice = toolChoice; + this.autoInvoke = autoInvoke; + this.options = options; + } + + /** + * Gets the list of tools available for the call. + * + * @return The list of tools. + */ + public List getTools() { + return tools; + } + + /** + * Gets the tool selection strategy. + * + * @return The tool selection strategy. + */ + public ChatCompletionsToolSelection getToolChoice() { + return toolChoice; + } + + /** + * Indicates whether to automatically invoke the tool. + * + * @return True if auto-invocation is enabled; otherwise, false. + */ + public boolean isAutoInvoke() { + return autoInvoke; + } + + /** + * Gets additional options for function choice behavior. + * + * @return The function choice behavior options. + */ + public FunctionChoiceBehaviorOptions getOptions() { + return options; + } +} diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionAgent.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionAgent.java index 1e5a665a..16941ee4 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionAgent.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionAgent.java @@ -10,11 +10,10 @@ import com.microsoft.semantickernel.agents.chatcompletion.ChatHistoryAgentThread; import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIChatCompletion; import com.microsoft.semantickernel.contextvariables.ContextVariableTypeConverter; -import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; +import com.microsoft.semantickernel.functionchoice.FunctionChoiceBehavior; import com.microsoft.semantickernel.implementation.templateengine.tokenizer.DefaultPromptTemplate; import com.microsoft.semantickernel.orchestration.InvocationContext; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; -import com.microsoft.semantickernel.orchestration.ToolCallBehavior; import com.microsoft.semantickernel.plugin.KernelPluginFactory; import com.microsoft.semantickernel.samples.plugins.github.GitHubModel; import com.microsoft.semantickernel.samples.plugins.github.GitHubPlugin; @@ -68,7 +67,7 @@ public static void main(String[] args) { .build(); InvocationContext invocationContext = InvocationContext.builder() - .withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(true)) + .withFunctionChoiceBehavior(FunctionChoiceBehavior.auto(true)) .withContextVariableConverter(new ContextVariableTypeConverter<>( GitHubModel.Issue.class, o -> (GitHubModel.Issue) o, diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example59_OpenAIFunctionCalling.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example59_OpenAIFunctionCalling.java index e52c2be7..e921bb78 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example59_OpenAIFunctionCalling.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example59_OpenAIFunctionCalling.java @@ -10,6 +10,7 @@ import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIChatMessageContent; import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIFunctionToolCall; import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; +import com.microsoft.semantickernel.functionchoice.FunctionChoiceBehavior; import com.microsoft.semantickernel.implementation.CollectionUtil; import com.microsoft.semantickernel.orchestration.FunctionResult; import com.microsoft.semantickernel.orchestration.FunctionResultMetadata; @@ -38,7 +39,7 @@ public class Example59_OpenAIFunctionCalling { // Only required if AZURE_CLIENT_KEY is set private static final String CLIENT_ENDPOINT = System.getenv("CLIENT_ENDPOINT"); private static final String MODEL_ID = System.getenv() - .getOrDefault("MODEL_ID", "gpt-35-turbo-2"); + .getOrDefault("MODEL_ID", "gpt-4o"); // Define functions that can be called by the model public static class HelperFunctions { @@ -118,7 +119,7 @@ public static void main(String[] args) throws NoSuchMethodException { var result = kernel .invokeAsync(function) - .withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(true)) + .withFunctionChoiceBehavior(FunctionChoiceBehavior.auto(true)) .withResultType(ContextVariableTypes.getGlobalVariableTypeForClass(String.class)) .block(); System.out.println(result.getResult()); @@ -134,7 +135,7 @@ public static void main(String[] args) throws NoSuchMethodException { chatHistory, kernel, InvocationContext.builder() - .withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(false)) + .withFunctionChoiceBehavior(FunctionChoiceBehavior.auto(false)) .withReturnMode(InvocationReturnMode.FULL_HISTORY) .build()) .block(); @@ -243,7 +244,7 @@ public static void multiTurnaroundCall() { chatHistory, kernel, InvocationContext.builder() - .withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(true)) + .withFunctionChoiceBehavior(FunctionChoiceBehavior.auto(true)) .withReturnMode(InvocationReturnMode.FULL_HISTORY) .build()) .block(); @@ -258,7 +259,7 @@ public static void multiTurnaroundCall() { chatHistory, kernel, InvocationContext.builder() - .withToolCallBehavior(ToolCallBehavior.allowAllKernelFunctions(true)) + .withFunctionChoiceBehavior(FunctionChoiceBehavior.auto(true)) .withReturnMode(InvocationReturnMode.FULL_HISTORY) .build()) .block(); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/AutoFunctionChoiceBehavior.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/AutoFunctionChoiceBehavior.java new file mode 100644 index 00000000..3e3878b7 --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/AutoFunctionChoiceBehavior.java @@ -0,0 +1,39 @@ +package com.microsoft.semantickernel.functionchoice; + +import com.microsoft.semantickernel.semanticfunctions.KernelFunction; + +import javax.annotation.Nullable; +import java.util.List; + +/** + * A set of allowed kernel functions. All kernel functions are allowed if allKernelFunctionsAllowed is true. + * Otherwise, only the functions in allowedFunctions are allowed. + *

+ * If a function is allowed, it may be called. If it is not allowed, it will not be called. + */ +public class AutoFunctionChoiceBehavior extends FunctionChoiceBehavior { + private final boolean autoInvoke; + + /** + * Create a new instance of AutoFunctionChoiceBehavior. + * + * @param autoInvoke Whether auto-invocation is enabled. + * @param functions A set of functions to advertise to the model. + * @param options Options for the function choice behavior. + */ + public AutoFunctionChoiceBehavior(boolean autoInvoke, + @Nullable List> functions, + @Nullable FunctionChoiceBehaviorOptions options) { + super(functions, options); + this.autoInvoke = autoInvoke; + } + + /** + * Check whether the given function is allowed. + * + * @return Whether the function is allowed. + */ + public boolean isAutoInvoke() { + return autoInvoke; + } +} \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java new file mode 100644 index 00000000..5e4c50cd --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java @@ -0,0 +1,162 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.functionchoice; + +import com.microsoft.semantickernel.semanticfunctions.KernelFunction; + +import javax.annotation.Nullable; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +/** + * Defines the behavior of a tool call. Currently, the only tool available is function calling. + */ +public abstract class FunctionChoiceBehavior { + private final Set fullFunctionNames; + + protected final List> functions; + protected final FunctionChoiceBehaviorOptions options; + + protected FunctionChoiceBehavior(List> functions, + @Nullable FunctionChoiceBehaviorOptions options) { + this.functions = functions; + this.fullFunctionNames = new HashSet<>(); + + if (functions != null) { + functions.stream().filter(Objects::nonNull).forEach( + f -> this.fullFunctionNames + .add(formFullFunctionName(f.getPluginName(), f.getName()))); + } + + if (options != null) { + this.options = options; + } else { + this.options = FunctionChoiceBehaviorOptions.builder().build(); + } + } + + /** + * Gets the functions that are allowed. + * + * @return The functions that are allowed. + */ + public List> getFunctions() { + return functions; + } + + /** + * Gets the options for the function choice behavior. + * + * @return The options for the function choice behavior. + */ + public FunctionChoiceBehaviorOptions getOptions() { + return options; + } + + /** + * Gets an instance of the FunctionChoiceBehavior that provides all the Kernel's plugins functions to the AI model to call. + * + * @param autoInvoke Indicates whether the functions should be automatically invoked by AI connectors + * + * @return A new ToolCallBehavior instance with all kernel functions allowed. + */ + public static FunctionChoiceBehavior auto(boolean autoInvoke) { + return new AutoFunctionChoiceBehavior(autoInvoke, null, null); + } + + /** + * Gets an instance of the FunctionChoiceBehavior that provides either all the Kernel's plugins functions to the AI model to call or specific functions. + * + * @param autoInvoke Enable or disable auto-invocation. + * If auto-invocation is enabled, the model may request that the Semantic Kernel + * invoke the kernel functions and return the value to the model. + * @param functions Functions to provide to the model. If null, all the Kernel's plugins' functions are provided to the model. + * If empty, no functions are provided to the model, which is equivalent to disabling function calling. + * @param options Options for the function choice behavior. + * + * @return A new FunctionChoiceBehavior instance with all kernel functions allowed. + */ + public static FunctionChoiceBehavior auto(boolean autoInvoke, + List> functions, + @Nullable FunctionChoiceBehaviorOptions options) { + return new AutoFunctionChoiceBehavior(autoInvoke, functions, options); + } + + /** + * Gets an instance of the FunctionChoiceBehavior that provides either all the Kernel's plugins functions to the AI model to call or specific functions. + *

+ * This behavior forces the model to call the provided functions. + * SK connectors will invoke a requested function or multiple requested functions if the model requests multiple ones in one request, + * while handling the first request, and stop advertising the functions for the following requests to prevent the model from repeatedly calling the same function(s). + * + * @param functions Functions to provide to the model. If null, all the Kernel's plugins' functions are provided to the model. + * If empty, no functions are provided to the model, which is equivalent to disabling function calling. + * @return A new FunctionChoiceBehavior instance with the required function. + */ + public static FunctionChoiceBehavior required(boolean autoInvoke, + List> functions, + @Nullable FunctionChoiceBehaviorOptions options) { + return new RequiredFunctionChoiceBehavior(autoInvoke, functions, options); + } + + /** + * Gets an instance of the FunctionChoiceBehavior that provides either all the Kernel's plugins functions to the AI model to call or specific functions. + *

+ * This behavior is useful if the user should first validate what functions the model will use. + * + * @param functions Functions to provide to the model. If null, all the Kernel's plugins' functions are provided to the model. + * If empty, no functions are provided to the model, which is equivalent to disabling function calling. + */ + public static FunctionChoiceBehavior none(List> functions, + @Nullable FunctionChoiceBehaviorOptions options) { + return new NoneFunctionChoiceBehavior(functions, options); + } + + + /** + * The separator between the plugin name and the function name. + */ + public static final String FUNCTION_NAME_SEPARATOR = "-"; + + /** + * Form the full function name. + * + * @param pluginName The name of the plugin that the function is in. + * @param functionName The name of the function. + * @return The key for the function. + */ + public static String formFullFunctionName(@Nullable String pluginName, String functionName) { + if (pluginName == null) { + pluginName = ""; + } + return String.format("%s%s%s", pluginName, FUNCTION_NAME_SEPARATOR, functionName); + } + + /** + * Check whether the given function is allowed. + * + * @param function The function to check. + * @return Whether the function is allowed. + */ + public boolean isFunctionAllowed(KernelFunction function) { + return isFunctionAllowed(function.getPluginName(), function.getName()); + } + + /** + * Check whether the given function is allowed. + * + * @param pluginName The name of the plugin that the function is in. + * @param functionName The name of the function. + * @return Whether the function is allowed. + */ + public boolean isFunctionAllowed(@Nullable String pluginName, String functionName) { + // If no functions are provided, all functions are allowed. + if (functions == null || functions.isEmpty()) { + return true; + } + + String key = formFullFunctionName(pluginName, functionName); + return fullFunctionNames.contains(key); + } +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehaviorOptions.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehaviorOptions.java new file mode 100644 index 00000000..bbde7c57 --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehaviorOptions.java @@ -0,0 +1,49 @@ +package com.microsoft.semantickernel.functionchoice; + +import com.microsoft.semantickernel.builders.SemanticKernelBuilder; + +public class FunctionChoiceBehaviorOptions { + private final boolean parallelCallsAllowed; + + private FunctionChoiceBehaviorOptions(boolean parallelCallsAllowed) { + this.parallelCallsAllowed = parallelCallsAllowed; + } + + /** + * Returns a new builder for {@link FunctionChoiceBehaviorOptions}. + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Indicates whether parallel calls to functions are allowed. + * + * @return True if parallel calls are allowed; otherwise, false. + */ + public boolean isParallelCallsAllowed() { + return parallelCallsAllowed; + } + + /** + * Builder for {@link FunctionChoiceBehaviorOptions}. + */ + public static class Builder implements SemanticKernelBuilder { + private boolean allowParallelCalls = false; + + /** + * Sets whether parallel calls to functions are allowed. + * + * @param allowParallelCalls True if parallel calls are allowed; otherwise, false. + * @return The builder instance. + */ + public Builder withParallelCallsAllowed(boolean allowParallelCalls) { + this.allowParallelCalls = allowParallelCalls; + return this; + } + + public FunctionChoiceBehaviorOptions build() { + return new FunctionChoiceBehaviorOptions(allowParallelCalls); + } + } +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/NoneFunctionChoiceBehavior.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/NoneFunctionChoiceBehavior.java new file mode 100644 index 00000000..0ee7f1a0 --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/NoneFunctionChoiceBehavior.java @@ -0,0 +1,15 @@ +package com.microsoft.semantickernel.functionchoice; + +import com.microsoft.semantickernel.semanticfunctions.KernelFunction; + +import java.util.List; + +public class NoneFunctionChoiceBehavior extends FunctionChoiceBehavior { + + /** + * Create a new instance of NoneFunctionChoiceBehavior. + */ + public NoneFunctionChoiceBehavior(List> functions, FunctionChoiceBehaviorOptions options) { + super(functions, options); + } +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/RequiredFunctionChoiceBehavior.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/RequiredFunctionChoiceBehavior.java new file mode 100644 index 00000000..57312b0f --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/RequiredFunctionChoiceBehavior.java @@ -0,0 +1,19 @@ +package com.microsoft.semantickernel.functionchoice; + +import com.microsoft.semantickernel.semanticfunctions.KernelFunction; + +import java.util.List; + +public class RequiredFunctionChoiceBehavior extends AutoFunctionChoiceBehavior { + + /** + * Create a new instance of RequiredFunctionChoiceBehavior. + * + * @param autoInvoke Whether auto-invocation is enabled. + * @param functions A set of functions to advertise to the model. + * @param options Options for the function choice behavior. + */ + public RequiredFunctionChoiceBehavior(boolean autoInvoke, List> functions, FunctionChoiceBehaviorOptions options) { + super(autoInvoke, functions, options); + } +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java index 0ae16e19..9b8a518c 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java @@ -8,6 +8,7 @@ import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; import com.microsoft.semantickernel.contextvariables.converters.ContextVariableJacksonConverter; import com.microsoft.semantickernel.exceptions.SKException; +import com.microsoft.semantickernel.functionchoice.FunctionChoiceBehavior; import com.microsoft.semantickernel.hooks.KernelHook; import com.microsoft.semantickernel.hooks.KernelHooks; import com.microsoft.semantickernel.hooks.KernelHooks.UnmodifiableKernelHooks; @@ -48,6 +49,9 @@ public class FunctionInvocation extends Mono> { protected PromptExecutionSettings promptExecutionSettings; @Nullable protected ToolCallBehavior toolCallBehavior; + @Nullable + protected FunctionChoiceBehavior functionChoiceBehavior; + @Nullable protected SemanticKernelTelemetry telemetry; @@ -196,6 +200,7 @@ public FunctionInvocation withResultType(ContextVariableType resultTyp .withArguments(arguments) .addKernelHooks(hooks) .withPromptExecutionSettings(promptExecutionSettings) + .withFunctionChoiceBehavior(functionChoiceBehavior) .withToolCallBehavior(toolCallBehavior) .withTypes(contextVariableTypes); } @@ -287,10 +292,32 @@ public FunctionInvocation withPromptExecutionSettings( */ public FunctionInvocation withToolCallBehavior(@Nullable ToolCallBehavior toolCallBehavior) { logSubscribeWarning(); + if (toolCallBehavior != null && functionChoiceBehavior != null) { + throw new SKException( + "ToolCallBehavior cannot be set when FunctionChoiceBehavior is set."); + } this.toolCallBehavior = toolCallBehavior; return this; } + /** + * Supply function choice behavior to the function invocation. + * + * @param functionChoiceBehavior The function choice behavior to supply to the function + * invocation. + * @return this {@code FunctionInvocation} for fluent chaining. + */ + public FunctionInvocation withFunctionChoiceBehavior( + @Nullable FunctionChoiceBehavior functionChoiceBehavior) { + if (functionChoiceBehavior != null && toolCallBehavior != null) { + throw new SKException( + "FunctionChoiceBehavior cannot be set when ToolCallBehavior is set."); + } + logSubscribeWarning(); + this.functionChoiceBehavior = functionChoiceBehavior; + return this; + } + /** * Supply a type converter to the function invocation. * @@ -340,6 +367,7 @@ public FunctionInvocation withInvocationContext( } logSubscribeWarning(); withTypes(invocationContext.getContextVariableTypes()); + withFunctionChoiceBehavior(invocationContext.getFunctionChoiceBehavior()); withToolCallBehavior(invocationContext.getToolCallBehavior()); withPromptExecutionSettings(invocationContext.getPromptExecutionSettings()); addKernelHooks(invocationContext.getKernelHooks()); @@ -387,6 +415,7 @@ public void subscribe(CoreSubscriber> coreSubscriber) hooks, promptExecutionSettings, toolCallBehavior, + functionChoiceBehavior, contextVariableTypes, InvocationReturnMode.NEW_MESSAGES_ONLY, telemetry)); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/InvocationContext.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/InvocationContext.java index 6fd3f0d2..48c10949 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/InvocationContext.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/InvocationContext.java @@ -4,6 +4,8 @@ import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.contextvariables.ContextVariableTypeConverter; import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; +import com.microsoft.semantickernel.exceptions.SKException; +import com.microsoft.semantickernel.functionchoice.FunctionChoiceBehavior; import com.microsoft.semantickernel.hooks.KernelHooks; import com.microsoft.semantickernel.hooks.KernelHooks.UnmodifiableKernelHooks; import com.microsoft.semantickernel.implementation.telemetry.SemanticKernelTelemetry; @@ -23,6 +25,8 @@ public class InvocationContext { private final PromptExecutionSettings promptExecutionSettings; @Nullable private final ToolCallBehavior toolCallBehavior; + @Nullable + private final FunctionChoiceBehavior functionChoiceBehavior; private final ContextVariableTypes contextVariableTypes; private final InvocationReturnMode invocationReturnMode; private final SemanticKernelTelemetry telemetry; @@ -39,12 +43,14 @@ protected InvocationContext( @Nullable KernelHooks hooks, @Nullable PromptExecutionSettings promptExecutionSettings, @Nullable ToolCallBehavior toolCallBehavior, + @Nullable FunctionChoiceBehavior functionChoiceBehavior, @Nullable ContextVariableTypes contextVariableTypes, InvocationReturnMode invocationReturnMode, SemanticKernelTelemetry telemetry) { this.hooks = unmodifiableClone(hooks); this.promptExecutionSettings = promptExecutionSettings; this.toolCallBehavior = toolCallBehavior; + this.functionChoiceBehavior = functionChoiceBehavior; this.invocationReturnMode = invocationReturnMode; if (contextVariableTypes == null) { this.contextVariableTypes = new ContextVariableTypes(); @@ -61,6 +67,7 @@ protected InvocationContext() { this.hooks = null; this.promptExecutionSettings = null; this.toolCallBehavior = null; + this.functionChoiceBehavior = null; this.contextVariableTypes = new ContextVariableTypes(); this.invocationReturnMode = InvocationReturnMode.NEW_MESSAGES_ONLY; this.telemetry = null; @@ -76,6 +83,7 @@ protected InvocationContext(@Nullable InvocationContext context) { this.hooks = null; this.promptExecutionSettings = null; this.toolCallBehavior = null; + this.functionChoiceBehavior = null; this.contextVariableTypes = new ContextVariableTypes(); this.invocationReturnMode = InvocationReturnMode.NEW_MESSAGES_ONLY; this.telemetry = null; @@ -83,6 +91,7 @@ protected InvocationContext(@Nullable InvocationContext context) { this.hooks = context.hooks; this.promptExecutionSettings = context.promptExecutionSettings; this.toolCallBehavior = context.toolCallBehavior; + this.functionChoiceBehavior = context.functionChoiceBehavior; this.contextVariableTypes = context.contextVariableTypes; this.invocationReturnMode = context.invocationReturnMode; this.telemetry = context.telemetry; @@ -156,6 +165,16 @@ public ToolCallBehavior getToolCallBehavior() { return toolCallBehavior; } + /** + * Get the behavior for function choice. + * + * @return The behavior for function choice. + */ + @Nullable + public FunctionChoiceBehavior getFunctionChoiceBehavior() { + return functionChoiceBehavior; + } + /** * Get the types of context variables. * @@ -190,6 +209,8 @@ public static class Builder implements SemanticKernelBuilder private PromptExecutionSettings promptExecutionSettings; @Nullable private ToolCallBehavior toolCallBehavior; + @Nullable + private FunctionChoiceBehavior functionChoiceBehavior; private InvocationReturnMode invocationReturnMode = InvocationReturnMode.NEW_MESSAGES_ONLY; @Nullable private SemanticKernelTelemetry telemetry; @@ -226,10 +247,30 @@ public Builder withPromptExecutionSettings( */ public Builder withToolCallBehavior( @Nullable ToolCallBehavior toolCallBehavior) { + if (toolCallBehavior != null && functionChoiceBehavior != null) { + throw new SKException( + "ToolCallBehavior cannot be set when FunctionChoiceBehavior is set."); + } this.toolCallBehavior = toolCallBehavior; return this; } + /** + * Add function choice behavior to the builder. + * + * @param functionChoiceBehavior the behavior to add. + * @return this {@link Builder} + */ + public Builder withFunctionChoiceBehavior( + @Nullable FunctionChoiceBehavior functionChoiceBehavior) { + if (functionChoiceBehavior != null && toolCallBehavior != null) { + throw new SKException( + "FunctionChoiceBehavior cannot be set when ToolCallBehavior is set."); + } + this.functionChoiceBehavior = functionChoiceBehavior; + return this; + } + /** * Add a context variable type converter to the builder. * @@ -269,7 +310,7 @@ public Builder withReturnMode(InvocationReturnMode invocationReturnMode) { /** * Add a tracer to the builder. * - * @param tracer the tracer to add. + * @param telemetry the tracer to add. * @return this {@link Builder} */ public Builder withTelemetry(@Nullable SemanticKernelTelemetry telemetry) { @@ -282,7 +323,7 @@ public InvocationContext build() { if (telemetry == null) { telemetry = new SemanticKernelTelemetry(); } - return new InvocationContext(hooks, promptExecutionSettings, toolCallBehavior, + return new InvocationContext(hooks, promptExecutionSettings, toolCallBehavior, functionChoiceBehavior, contextVariableTypes, invocationReturnMode, telemetry); } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/TextAIService.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/TextAIService.java index 09b3ea1f..3eee32d3 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/TextAIService.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/TextAIService.java @@ -29,5 +29,5 @@ public interface TextAIService extends AIService { * future and/or made configurable should need arise. *

*/ - int MAXIMUM_INFLIGHT_AUTO_INVOKES = 5; + int MAXIMUM_INFLIGHT_AUTO_INVOKES = 128; } From bc618869e76dec1927b0a9624a6a8737644f89e9 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 29 Apr 2025 10:04:55 -0700 Subject: [PATCH 146/277] return Collections.unmodifiableList(functions) --- .../functionchoice/FunctionChoiceBehavior.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java index 5e4c50cd..844e66a5 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java @@ -4,6 +4,7 @@ import com.microsoft.semantickernel.semanticfunctions.KernelFunction; import javax.annotation.Nullable; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -20,7 +21,7 @@ public abstract class FunctionChoiceBehavior { protected FunctionChoiceBehavior(List> functions, @Nullable FunctionChoiceBehaviorOptions options) { - this.functions = functions; + this.functions = functions != null ? Collections.unmodifiableList(functions) : null; this.fullFunctionNames = new HashSet<>(); if (functions != null) { @@ -42,7 +43,7 @@ protected FunctionChoiceBehavior(List> functions, * @return The functions that are allowed. */ public List> getFunctions() { - return functions; + return Collections.unmodifiableList(functions); } /** From 8cd72afadf25ca5ec31458707c0f42f78c0fdb4f Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 29 Apr 2025 10:20:39 -0700 Subject: [PATCH 147/277] Fixes --- .../openai/chatcompletion/OpenAIToolCallConfig.java | 5 ++++- .../functionchoice/FunctionChoiceBehavior.java | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIToolCallConfig.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIToolCallConfig.java index 895252d3..4694d2f8 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIToolCallConfig.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIToolCallConfig.java @@ -3,8 +3,10 @@ import com.azure.ai.openai.models.ChatCompletionsToolDefinition; import com.azure.ai.openai.models.ChatCompletionsToolSelection; import com.microsoft.semantickernel.functionchoice.FunctionChoiceBehaviorOptions; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.annotation.Nullable; +import java.util.Collections; import java.util.List; public class OpenAIToolCallConfig { @@ -21,6 +23,7 @@ public class OpenAIToolCallConfig { * @param autoInvoke Indicates whether to automatically invoke the tool. * @param options Additional options for function choice behavior. */ + @SuppressFBWarnings("EI_EXPOSE_REP2") public OpenAIToolCallConfig( List tools, ChatCompletionsToolSelection toolChoice, @@ -38,7 +41,7 @@ public OpenAIToolCallConfig( * @return The list of tools. */ public List getTools() { - return tools; + return Collections.unmodifiableList(tools); } /** diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java index 844e66a5..bfa39785 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java @@ -19,7 +19,7 @@ public abstract class FunctionChoiceBehavior { protected final List> functions; protected final FunctionChoiceBehaviorOptions options; - protected FunctionChoiceBehavior(List> functions, + protected FunctionChoiceBehavior(@Nullable List> functions, @Nullable FunctionChoiceBehaviorOptions options) { this.functions = functions != null ? Collections.unmodifiableList(functions) : null; this.fullFunctionNames = new HashSet<>(); From a222ccb427f5b73f8bb467fc65656daee933281a Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 29 Apr 2025 10:27:16 -0700 Subject: [PATCH 148/277] Fix retry logic --- .../aiservices/openai/chatcompletion/OpenAIChatCompletion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java index 0d0dc304..20be752e 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java @@ -526,7 +526,7 @@ private Mono internalChatMessageContentsAsync( LOGGER.warn("Tool invocation attempt failed: ", e); // If FunctionInvocationError occurred and there are still attempts left, retry, else exit - if (requestIndex > 0) { + if (requestIndex < MAXIMUM_INFLIGHT_AUTO_INVOKES) { ChatMessages currentMessages = messages; if (e instanceof FunctionInvocationError) { currentMessages.assertCommonHistory( From 3c3953818a01d6085ebc2ebfca3ade7088ba56e2 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Wed, 30 Apr 2025 12:25:21 -0700 Subject: [PATCH 149/277] Add suggestions --- .../aiservices/openai/chatcompletion/OpenAIChatCompletion.java | 3 +++ .../aiservices/openai/chatcompletion/OpenAIToolCallConfig.java | 1 + .../semanticfunctions/KernelFunctionFromPrompt.java | 1 + 3 files changed, 5 insertions(+) diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java index 20be752e..b056bfef 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java @@ -969,6 +969,7 @@ private static ChatCompletionsOptions getCompletionsOptions( return options; } + @Nullable private static OpenAIToolCallConfig getToolCallConfig( @Nullable InvocationContext invocationContext, @Nullable List functions, @@ -998,6 +999,7 @@ private static OpenAIToolCallConfig getToolCallConfig( } } + @Nullable private static OpenAIToolCallConfig getFunctionChoiceBehaviorConfig( @Nullable FunctionChoiceBehavior functionChoiceBehavior, @Nullable List functions, @@ -1049,6 +1051,7 @@ private static OpenAIToolCallConfig getFunctionChoiceBehaviorConfig( functionChoiceBehavior.getOptions()); } + @Nullable private static OpenAIToolCallConfig getToolCallBehaviorConfig( @Nullable ToolCallBehavior toolCallBehavior, @Nullable List functions, diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIToolCallConfig.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIToolCallConfig.java index 4694d2f8..df733172 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIToolCallConfig.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIToolCallConfig.java @@ -13,6 +13,7 @@ public class OpenAIToolCallConfig { private final List tools; private final ChatCompletionsToolSelection toolChoice; private final boolean autoInvoke; + @Nullable private final FunctionChoiceBehaviorOptions options; /** diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java index 2ea90974..babc1da6 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java @@ -143,6 +143,7 @@ private Flux> invokeInternalAsync( .getServiceSelector() .trySelectAIService( TextAIService.class, + this, args); AIService client = aiServiceSelection != null ? aiServiceSelection.getService() From 66976a9eed9c88eeae5425f83ea317137648b619 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Wed, 30 Apr 2025 13:31:19 -0700 Subject: [PATCH 150/277] Avoid notify thread when auto-invoke is not active --- .../chatcompletion/ChatCompletionAgent.java | 42 +++++++++++++++---- .../agents/CompletionAgent.java | 28 ++++++------- .../semantickernel/agents/Agent.java | 27 ++++++++++++ .../agents/AgentInvokeOptions.java | 4 ++ .../semantickernel/agents/KernelAgent.java | 19 ++++++++- 5 files changed, 98 insertions(+), 22 deletions(-) diff --git a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java index 7985f5ad..28cb6502 100644 --- a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java +++ b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java @@ -6,10 +6,10 @@ import com.microsoft.semantickernel.agents.AgentThread; import com.microsoft.semantickernel.agents.KernelAgent; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.functionchoice.AutoFunctionChoiceBehavior; import com.microsoft.semantickernel.orchestration.InvocationContext; import com.microsoft.semantickernel.orchestration.InvocationReturnMode; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; -import com.microsoft.semantickernel.orchestration.ToolCallBehavior; import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import com.microsoft.semantickernel.semanticfunctions.PromptTemplate; import com.microsoft.semantickernel.semanticfunctions.PromptTemplateConfig; @@ -76,22 +76,20 @@ public Mono>>> invokeAsync( // Invoke the agent with the chat history return internalInvokeAsync( history, + agentThread, options ) - .flatMapMany(Flux::fromIterable) - // notify on the new thread instance - .concatMap(agentMessage -> this.notifyThreadOfNewMessageAsync(agentThread, agentMessage).thenReturn(agentMessage)) - .collectList() .map(chatMessageContents -> chatMessageContents.stream() - .map(message -> new AgentResponseItem>(message, agentThread)) - .collect(Collectors.toList()) + .map(message -> new AgentResponseItem>(message, agentThread)) + .collect(Collectors.toList()) ); }); } private Mono>> internalInvokeAsync( ChatHistory history, + AgentThread thread, @Nullable AgentInvokeOptions options ) { if (options == null) { @@ -144,6 +142,20 @@ private Mono>> internalInvokeAsync( // Add the chat history to the new chat chat.addAll(history); + // Retrieve the chat message contents asynchronously and notify the thread + if (shouldNotifyFunctionCalls(agentInvocationContext)) { + // Notify all messages including function calls + return chatCompletionService.getChatMessageContentsAsync(chat, kernel, agentInvocationContext) + .flatMapMany(Flux::fromIterable) + .concatMap(message -> notifyThreadOfNewMessageAsync(thread, message).thenReturn(message)) + // Filter out function calls and their results + .filter(message -> message.getContent() != null && message.getAuthorRole() != AuthorRole.TOOL) + .collect(Collectors.toList()); + } + + // Return chat completion messages without notifying the thread + // We shouldn't add the function call content to the thread, since + // we don't know if the user will execute the call. They should add it themselves. return chatCompletionService.getChatMessageContentsAsync(chat, kernel, agentInvocationContext); } ); @@ -153,6 +165,22 @@ private Mono>> internalInvokeAsync( } } + boolean shouldNotifyFunctionCalls(InvocationContext invocationContext) { + if (invocationContext == null) { + return false; + } + + if (invocationContext.getFunctionChoiceBehavior() != null && invocationContext.getFunctionChoiceBehavior() instanceof AutoFunctionChoiceBehavior) { + return ((AutoFunctionChoiceBehavior) invocationContext.getFunctionChoiceBehavior()).isAutoInvoke(); + } + + if (invocationContext.getToolCallBehavior() != null) { + return invocationContext.getToolCallBehavior().isAutoInvokeAllowed(); + } + + return false; + } + @Override public Mono notifyThreadOfNewMessageAsync(AgentThread thread, ChatMessageContent message) { diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionAgent.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionAgent.java index 16941ee4..4c39645c 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionAgent.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionAgent.java @@ -6,14 +6,17 @@ import com.azure.core.credential.KeyCredential; import com.microsoft.semantickernel.Kernel; import com.microsoft.semantickernel.agents.AgentInvokeOptions; +import com.microsoft.semantickernel.agents.AgentThread; import com.microsoft.semantickernel.agents.chatcompletion.ChatCompletionAgent; import com.microsoft.semantickernel.agents.chatcompletion.ChatHistoryAgentThread; import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIChatCompletion; import com.microsoft.semantickernel.contextvariables.ContextVariableTypeConverter; +import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; import com.microsoft.semantickernel.functionchoice.FunctionChoiceBehavior; import com.microsoft.semantickernel.implementation.templateengine.tokenizer.DefaultPromptTemplate; import com.microsoft.semantickernel.orchestration.InvocationContext; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; +import com.microsoft.semantickernel.orchestration.ToolCallBehavior; import com.microsoft.semantickernel.plugin.KernelPluginFactory; import com.microsoft.semantickernel.samples.plugins.github.GitHubModel; import com.microsoft.semantickernel.samples.plugins.github.GitHubPlugin; @@ -105,7 +108,7 @@ public static void main(String[] args) { ) ).build(); - ChatHistoryAgentThread agentThread = new ChatHistoryAgentThread(); + AgentThread agentThread = new ChatHistoryAgentThread(); Scanner scanner = new Scanner(System.in); while (true) { @@ -118,22 +121,19 @@ public static void main(String[] args) { var message = new ChatMessageContent<>(AuthorRole.USER, input); KernelArguments arguments = KernelArguments.builder() - .withVariable("now", System.currentTimeMillis()) - .build(); + .withVariable("now", System.currentTimeMillis()) + .build(); var response = agent.invokeAsync( - List.of(message), - agentThread, - AgentInvokeOptions.builder() - .withKernel(kernel) - .withKernelArguments(arguments) - .build() - ).block(); - - var lastResponse = response.get(response.size() - 1); + message, + agentThread, + AgentInvokeOptions.builder() + .withKernelArguments(arguments) + .build() + ).block().get(0); - System.out.println("> " + lastResponse.getMessage()); - agentThread = (ChatHistoryAgentThread) lastResponse.getThread(); + System.out.println("> " + response.getMessage()); + agentThread = response.getThread(); } } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/Agent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/Agent.java index 3a82550c..6b8c9ad1 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/Agent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/Agent.java @@ -43,6 +43,33 @@ public interface Agent { */ String getDescription(); + /** + * Invokes the agent with the given message. + * + * @param message The message to process + * @return A Mono containing the agent response + */ + Mono>>> invokeAsync(ChatMessageContent message); + + /** + * Invokes the agent with the given message and thread. + * + * @param message The message to process + * @param thread The agent thread to use + * @return A Mono containing the agent response + */ + Mono>>> invokeAsync(ChatMessageContent message, AgentThread thread); + + /** + * Invokes the agent with the given message, thread, and options. + * + * @param message The message to process + * @param thread The agent thread to use + * @param options The options for invoking the agent + * @return A Mono containing the agent response + */ + Mono>>> invokeAsync(ChatMessageContent message, AgentThread thread, AgentInvokeOptions options); + /** * Invoke the agent with the given chat history. * diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java index 3fb4cba1..559a062c 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java @@ -13,9 +13,13 @@ */ public class AgentInvokeOptions { + @Nullable private final KernelArguments kernelArguments; + @Nullable private final Kernel kernel; + @Nullable private final String additionalInstructions; + @Nullable private final InvocationContext invocationContext; /** diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java index 71e81951..a3d87229 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java @@ -11,6 +11,7 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -114,7 +115,6 @@ public PromptTemplate getTemplate() { return template; } - /** * Merges the provided arguments with the current arguments. * Provided arguments will override the current arguments. @@ -167,4 +167,21 @@ protected Mono ensureThreadExistsWithMessagesAsync(Li .then(Mono.just((T) newThread)); }); } + + @Override + public Mono>>> invokeAsync(ChatMessageContent message) { + return invokeAsync(message, null); + } + + @Override + public Mono>>> invokeAsync(ChatMessageContent message, AgentThread thread) { + return invokeAsync(message, thread, AgentInvokeOptions.builder().build()); + } + + @Override + public Mono>>> invokeAsync(ChatMessageContent message, AgentThread thread, AgentInvokeOptions options) { + ArrayList> messages = new ArrayList<>(); + messages.add(message); + return invokeAsync(messages, thread, options); + } } From cbb37c655f34b23a6a92c0718d42739d4c7f01e0 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Wed, 30 Apr 2025 23:51:24 -0700 Subject: [PATCH 151/277] Add @Nullable annotations --- .../chatcompletion/ChatCompletionAgent.java | 2 +- .../semantickernel/agents/Agent.java | 15 +++++++++++---- .../semantickernel/agents/KernelAgent.java | 19 +++++++++++++------ 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java index 28cb6502..776ed6a5 100644 --- a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java +++ b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java @@ -62,7 +62,7 @@ private ChatCompletionAgent( @Override public Mono>>> invokeAsync( List> messages, - AgentThread thread, + @Nullable AgentThread thread, @Nullable AgentInvokeOptions options ) { return ensureThreadExistsWithMessagesAsync(messages, thread, ChatHistoryAgentThread::new) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/Agent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/Agent.java index 6b8c9ad1..2277d522 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/Agent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/Agent.java @@ -17,6 +17,8 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import javax.annotation.Nullable; + /** * Interface for a semantic kernel agent. */ @@ -49,7 +51,7 @@ public interface Agent { * @param message The message to process * @return A Mono containing the agent response */ - Mono>>> invokeAsync(ChatMessageContent message); + Mono>>> invokeAsync(@Nullable ChatMessageContent message); /** * Invokes the agent with the given message and thread. @@ -58,7 +60,8 @@ public interface Agent { * @param thread The agent thread to use * @return A Mono containing the agent response */ - Mono>>> invokeAsync(ChatMessageContent message, AgentThread thread); + Mono>>> invokeAsync(@Nullable ChatMessageContent message, + @Nullable AgentThread thread); /** * Invokes the agent with the given message, thread, and options. @@ -68,7 +71,9 @@ public interface Agent { * @param options The options for invoking the agent * @return A Mono containing the agent response */ - Mono>>> invokeAsync(ChatMessageContent message, AgentThread thread, AgentInvokeOptions options); + Mono>>> invokeAsync(@Nullable ChatMessageContent message, + @Nullable AgentThread thread, + @Nullable AgentInvokeOptions options); /** * Invoke the agent with the given chat history. @@ -78,7 +83,9 @@ public interface Agent { * @param options The options for invoking the agent * @return A Mono containing the agent response */ - Mono>>> invokeAsync(List> messages, AgentThread thread, AgentInvokeOptions options); + Mono>>> invokeAsync(List> messages, + @Nullable AgentThread thread, + @Nullable AgentInvokeOptions options); /** * Notifies the agent of a new message. diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java index a3d87229..4d30569f 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java @@ -11,6 +11,7 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; +import javax.annotation.Nullable; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -169,19 +170,25 @@ protected Mono ensureThreadExistsWithMessagesAsync(Li } @Override - public Mono>>> invokeAsync(ChatMessageContent message) { - return invokeAsync(message, null); + public Mono>>> invokeAsync(@Nullable ChatMessageContent message) { + return invokeAsync(message, null, null); } @Override - public Mono>>> invokeAsync(ChatMessageContent message, AgentThread thread) { - return invokeAsync(message, thread, AgentInvokeOptions.builder().build()); + public Mono>>> invokeAsync(@Nullable ChatMessageContent message, + @Nullable AgentThread thread) { + return invokeAsync(message, thread, null); } @Override - public Mono>>> invokeAsync(ChatMessageContent message, AgentThread thread, AgentInvokeOptions options) { + public Mono>>> invokeAsync( + @Nullable ChatMessageContent message, + @Nullable AgentThread thread, + @Nullable AgentInvokeOptions options) { ArrayList> messages = new ArrayList<>(); - messages.add(message); + if (message != null) { + messages.add(message); + } return invokeAsync(messages, thread, options); } } From b568818bcbd44a617d6046fa8fd229d942b7467c Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 1 May 2025 01:03:14 -0700 Subject: [PATCH 152/277] Add additional constructor for ChatHistoryAgentThread --- .../chatcompletion/ChatHistoryAgentThread.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatHistoryAgentThread.java b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatHistoryAgentThread.java index 1a68f8c4..753759c1 100644 --- a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatHistoryAgentThread.java +++ b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatHistoryAgentThread.java @@ -16,12 +16,25 @@ public class ChatHistoryAgentThread extends BaseAgentThread { private ChatHistory chatHistory; + /** + * Constructor for ChatHistoryAgentThread. + * + */ public ChatHistoryAgentThread() { this(UUID.randomUUID().toString(), new ChatHistory()); } /** - * Constructor for com.microsoft.semantickernel.agents.chatcompletion.ChatHistoryAgentThread. + * Constructor for ChatHistoryAgentThread. + * + * @param chatHistory The chat history. + */ + public ChatHistoryAgentThread(@Nullable ChatHistory chatHistory) { + this(UUID.randomUUID().toString(), chatHistory); + } + + /** + * Constructor for ChatHistoryAgentThread. * * @param id The ID of the thread. * @param chatHistory The chat history. @@ -31,6 +44,8 @@ public ChatHistoryAgentThread(String id, @Nullable ChatHistory chatHistory) { this.chatHistory = chatHistory != null ? chatHistory : new ChatHistory(); } + + /** * Get the chat history. * From 81abcbe681ce7400b494ddf0d2a6ae6d9b2437a6 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 1 May 2025 08:26:13 -0700 Subject: [PATCH 153/277] Fix keys of execution settings --- .../semantickernel/semanticfunctions/KernelArguments.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java index e4fb8319..bd519f61 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java @@ -377,7 +377,12 @@ public Builder withExecutionSettings(PromptExecutionSettings executionSetting * @return {$code this} Builder for fluent coding */ public Builder withExecutionSettings(Map executionSettings) { - return withExecutionSettings(new ArrayList<>(executionSettings.values())); + if (executionSettings == null) { + return this; + } + + this.executionSettings.putAll(executionSettings); + return this; } /** From 394367f1cc99248980074a00e3141eb49f035984 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 1 May 2025 10:48:00 -0700 Subject: [PATCH 154/277] Apply format --- .../chatcompletion/ChatCompletionAgent.java | 125 ++++++------- .../ChatHistoryAgentThread.java | 4 +- .../chatcompletion/OpenAIChatCompletion.java | 86 ++++----- .../openai/chatcompletion/OpenAIFunction.java | 14 +- .../chatcompletion/OpenAIToolCallConfig.java | 9 +- .../openai/chatcompletion/JsonSchemaTest.java | 16 +- .../demos/lights/LightModelTypeConverter.java | 8 +- .../samples/demos/lights/LightsPlugin.java | 2 +- .../samples/plugins/github/GitHubModel.java | 69 ++++--- .../samples/plugins/github/GitHubPlugin.java | 170 +++++++----------- .../agents/CompletionAgent.java | 90 +++++----- .../openapi/OpenAPIHttpRequestPlugin.java | 2 +- .../com/microsoft/semantickernel/Kernel.java | 11 +- .../semantickernel/agents/Agent.java | 23 ++- .../agents/AgentInvokeOptions.java | 12 +- .../agents/AgentResponseItem.java | 1 + .../semantickernel/agents/AgentThread.java | 1 + .../agents/BaseAgentThread.java | 2 + .../semantickernel/agents/KernelAgent.java | 69 +++---- .../AutoFunctionChoiceBehavior.java | 5 +- .../FunctionChoiceBehavior.java | 17 +- .../FunctionChoiceBehaviorOptions.java | 1 + .../NoneFunctionChoiceBehavior.java | 4 +- .../RequiredFunctionChoiceBehavior.java | 4 +- .../orchestration/InvocationContext.java | 3 +- .../semanticfunctions/KernelArguments.java | 63 ++++--- .../KernelFunctionFromPrompt.java | 6 +- .../services/BaseAIServiceSelector.java | 11 +- .../services/OrderedAIServiceSelector.java | 76 ++++---- .../chatcompletion/ChatMessageContent.java | 20 +-- 30 files changed, 460 insertions(+), 464 deletions(-) diff --git a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java index 776ed6a5..140df0f4 100644 --- a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java +++ b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.agents.chatcompletion; import com.microsoft.semantickernel.Kernel; @@ -37,8 +38,7 @@ private ChatCompletionAgent( KernelArguments kernelArguments, InvocationContext context, String instructions, - PromptTemplate template - ) { + PromptTemplate template) { super( id, name, @@ -47,8 +47,7 @@ private ChatCompletionAgent( kernelArguments, context, instructions, - template - ); + template); } /** @@ -61,37 +60,32 @@ private ChatCompletionAgent( */ @Override public Mono>>> invokeAsync( - List> messages, - @Nullable AgentThread thread, - @Nullable AgentInvokeOptions options - ) { + List> messages, + @Nullable AgentThread thread, + @Nullable AgentInvokeOptions options) { return ensureThreadExistsWithMessagesAsync(messages, thread, ChatHistoryAgentThread::new) - .cast(ChatHistoryAgentThread.class) - .flatMap(agentThread -> { - // Extract the chat history from the thread - ChatHistory history = new ChatHistory( - agentThread.getChatHistory().getMessages() - ); - - // Invoke the agent with the chat history - return internalInvokeAsync( - history, - agentThread, - options - ) - .map(chatMessageContents -> - chatMessageContents.stream() - .map(message -> new AgentResponseItem>(message, agentThread)) - .collect(Collectors.toList()) - ); - }); + .cast(ChatHistoryAgentThread.class) + .flatMap(agentThread -> { + // Extract the chat history from the thread + ChatHistory history = new ChatHistory( + agentThread.getChatHistory().getMessages()); + + // Invoke the agent with the chat history + return internalInvokeAsync( + history, + agentThread, + options) + .map(chatMessageContents -> chatMessageContents.stream() + .map(message -> new AgentResponseItem>(message, + agentThread)) + .collect(Collectors.toList())); + }); } private Mono>> internalInvokeAsync( ChatHistory history, AgentThread thread, - @Nullable AgentInvokeOptions options - ) { + @Nullable AgentInvokeOptions options) { if (options == null) { options = new AgentInvokeOptions(); } @@ -99,27 +93,32 @@ private Mono>> internalInvokeAsync( final Kernel kernel = options.getKernel() != null ? options.getKernel() : this.kernel; final KernelArguments arguments = mergeArguments(options.getKernelArguments()); final String additionalInstructions = options.getAdditionalInstructions(); - final InvocationContext invocationContext = options.getInvocationContext() != null ? options.getInvocationContext() : this.invocationContext; + final InvocationContext invocationContext = options.getInvocationContext() != null + ? options.getInvocationContext() + : this.invocationContext; try { - ChatCompletionService chatCompletionService = kernel.getService(ChatCompletionService.class, arguments); + ChatCompletionService chatCompletionService = kernel + .getService(ChatCompletionService.class, arguments); - PromptExecutionSettings executionSettings = invocationContext != null && invocationContext.getPromptExecutionSettings() != null + PromptExecutionSettings executionSettings = invocationContext != null + && invocationContext.getPromptExecutionSettings() != null ? invocationContext.getPromptExecutionSettings() - : kernelArguments.getExecutionSettings().get(chatCompletionService.getServiceId()); + : kernelArguments.getExecutionSettings() + .get(chatCompletionService.getServiceId()); // Build base invocation context InvocationContext.Builder builder = InvocationContext.builder() - .withPromptExecutionSettings(executionSettings) - .withReturnMode(InvocationReturnMode.NEW_MESSAGES_ONLY); + .withPromptExecutionSettings(executionSettings) + .withReturnMode(InvocationReturnMode.NEW_MESSAGES_ONLY); if (invocationContext != null) { builder = builder - .withTelemetry(invocationContext.getTelemetry()) - .withFunctionChoiceBehavior(invocationContext.getFunctionChoiceBehavior()) - .withToolCallBehavior(invocationContext.getToolCallBehavior()) - .withContextVariableConverter(invocationContext.getContextVariableTypes()) - .withKernelHooks(invocationContext.getKernelHooks()); + .withTelemetry(invocationContext.getTelemetry()) + .withFunctionChoiceBehavior(invocationContext.getFunctionChoiceBehavior()) + .withToolCallBehavior(invocationContext.getToolCallBehavior()) + .withContextVariableConverter(invocationContext.getContextVariableTypes()) + .withKernelHooks(invocationContext.getKernelHooks()); } InvocationContext agentInvocationContext = builder.build(); @@ -128,15 +127,13 @@ private Mono>> internalInvokeAsync( instructions -> { // Create a new chat history with the instructions ChatHistory chat = new ChatHistory( - instructions - ); + instructions); // Add agent additional instructions if (additionalInstructions != null) { chat.addMessage(new ChatMessageContent<>( - AuthorRole.SYSTEM, - additionalInstructions - )); + AuthorRole.SYSTEM, + additionalInstructions)); } // Add the chat history to the new chat @@ -145,20 +142,23 @@ private Mono>> internalInvokeAsync( // Retrieve the chat message contents asynchronously and notify the thread if (shouldNotifyFunctionCalls(agentInvocationContext)) { // Notify all messages including function calls - return chatCompletionService.getChatMessageContentsAsync(chat, kernel, agentInvocationContext) + return chatCompletionService + .getChatMessageContentsAsync(chat, kernel, agentInvocationContext) .flatMapMany(Flux::fromIterable) - .concatMap(message -> notifyThreadOfNewMessageAsync(thread, message).thenReturn(message)) + .concatMap(message -> notifyThreadOfNewMessageAsync(thread, message) + .thenReturn(message)) // Filter out function calls and their results - .filter(message -> message.getContent() != null && message.getAuthorRole() != AuthorRole.TOOL) + .filter(message -> message.getContent() != null + && message.getAuthorRole() != AuthorRole.TOOL) .collect(Collectors.toList()); } // Return chat completion messages without notifying the thread // We shouldn't add the function call content to the thread, since // we don't know if the user will execute the call. They should add it themselves. - return chatCompletionService.getChatMessageContentsAsync(chat, kernel, agentInvocationContext); - } - ); + return chatCompletionService.getChatMessageContentsAsync(chat, kernel, + agentInvocationContext); + }); } catch (ServiceNotFoundException e) { return Mono.error(e); @@ -170,8 +170,10 @@ boolean shouldNotifyFunctionCalls(InvocationContext invocationContext) { return false; } - if (invocationContext.getFunctionChoiceBehavior() != null && invocationContext.getFunctionChoiceBehavior() instanceof AutoFunctionChoiceBehavior) { - return ((AutoFunctionChoiceBehavior) invocationContext.getFunctionChoiceBehavior()).isAutoInvoke(); + if (invocationContext.getFunctionChoiceBehavior() != null && invocationContext + .getFunctionChoiceBehavior() instanceof AutoFunctionChoiceBehavior) { + return ((AutoFunctionChoiceBehavior) invocationContext.getFunctionChoiceBehavior()) + .isAutoInvoke(); } if (invocationContext.getToolCallBehavior() != null) { @@ -181,9 +183,9 @@ boolean shouldNotifyFunctionCalls(InvocationContext invocationContext) { return false; } - @Override - public Mono notifyThreadOfNewMessageAsync(AgentThread thread, ChatMessageContent message) { + public Mono notifyThreadOfNewMessageAsync(AgentThread thread, + ChatMessageContent message) { return Mono.defer(() -> { return thread.onNewMessageAsync(message); }); @@ -298,11 +300,10 @@ public ChatCompletionAgent build() { name, description, kernel, - kernelArguments, + kernelArguments, invocationContext, instructions, - template - ); + template); } /** @@ -312,17 +313,17 @@ public ChatCompletionAgent build() { * @param promptTemplateFactory The prompt template factory to use. * @return The ChatCompletionAgent instance. */ - public ChatCompletionAgent build(PromptTemplateConfig promptTemplateConfig, PromptTemplateFactory promptTemplateFactory) { + public ChatCompletionAgent build(PromptTemplateConfig promptTemplateConfig, + PromptTemplateFactory promptTemplateFactory) { return new ChatCompletionAgent( id, name, description, kernel, - kernelArguments, + kernelArguments, invocationContext, promptTemplateConfig.getTemplate(), - promptTemplateFactory.tryCreate(promptTemplateConfig) - ); + promptTemplateFactory.tryCreate(promptTemplateConfig)); } } } diff --git a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatHistoryAgentThread.java b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatHistoryAgentThread.java index 753759c1..6b3f62a9 100644 --- a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatHistoryAgentThread.java +++ b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatHistoryAgentThread.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.agents.chatcompletion; import com.microsoft.semantickernel.agents.AgentThread; @@ -44,8 +45,6 @@ public ChatHistoryAgentThread(String id, @Nullable ChatHistory chatHistory) { this.chatHistory = chatHistory != null ? chatHistory : new ChatHistory(); } - - /** * Get the chat history. * @@ -91,7 +90,6 @@ public List> getMessages() { return chatHistory.getMessages(); } - public static Builder builder() { return new Builder(); } diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java index b056bfef..1db5bd38 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java @@ -210,8 +210,8 @@ public Flux> getStreamingChatMessageContentsAsync( if (invocationContext != null && invocationContext.getFunctionChoiceBehavior() != null && invocationContext.getFunctionChoiceBehavior() instanceof AutoFunctionChoiceBehavior && - ((AutoFunctionChoiceBehavior) invocationContext.getFunctionChoiceBehavior()).isAutoInvoke() - ) { + ((AutoFunctionChoiceBehavior) invocationContext.getFunctionChoiceBehavior()) + .isAutoInvoke()) { throw new SKException( "FunctionChoiceBehavior auto-invoke is not supported for streaming chat message contents"); } @@ -482,7 +482,8 @@ private Mono internalChatMessageContentsAsync( // Just return the result: // If auto-invoking is not enabled // Or if we are auto-invoking, but we somehow end up with other than 1 choice even though only 1 was requested - if (toolCallConfig == null || !toolCallConfig.isAutoInvoke() || responseMessages.size() != 1) { + if (toolCallConfig == null || !toolCallConfig.isAutoInvoke() + || responseMessages.size() != 1) { List> chatMessageContents = getChatMessageContentsAsync( completions); return Mono.just(messages.addChatMessage(chatMessageContents)); @@ -980,11 +981,11 @@ private static OpenAIToolCallConfig getToolCallConfig( return null; } - if (invocationContext.getFunctionChoiceBehavior() == null && invocationContext.getToolCallBehavior() == null) { + if (invocationContext.getFunctionChoiceBehavior() == null + && invocationContext.getToolCallBehavior() == null) { return null; } - if (invocationContext.getFunctionChoiceBehavior() != null) { return getFunctionChoiceBehaviorConfig( invocationContext.getFunctionChoiceBehavior(), @@ -1015,18 +1016,20 @@ private static OpenAIToolCallConfig getFunctionChoiceBehaviorConfig( ChatCompletionsToolSelection toolChoice; boolean autoInvoke; - if (functionChoiceBehavior instanceof RequiredFunctionChoiceBehavior) { + if (functionChoiceBehavior instanceof RequiredFunctionChoiceBehavior) { // After first request a required function must have been called already if (requestIndex >= 1) { return null; } - toolChoice = new ChatCompletionsToolSelection(ChatCompletionsToolSelectionPreset.REQUIRED); + toolChoice = new ChatCompletionsToolSelection( + ChatCompletionsToolSelectionPreset.REQUIRED); autoInvoke = ((RequiredFunctionChoiceBehavior) functionChoiceBehavior).isAutoInvoke(); } else if (functionChoiceBehavior instanceof AutoFunctionChoiceBehavior) { - toolChoice = new ChatCompletionsToolSelection(ChatCompletionsToolSelectionPreset.AUTO); - autoInvoke = ((AutoFunctionChoiceBehavior) functionChoiceBehavior).isAutoInvoke() && requestIndex < MAXIMUM_INFLIGHT_AUTO_INVOKES; - } else if (functionChoiceBehavior instanceof NoneFunctionChoiceBehavior) { + toolChoice = new ChatCompletionsToolSelection(ChatCompletionsToolSelectionPreset.AUTO); + autoInvoke = ((AutoFunctionChoiceBehavior) functionChoiceBehavior).isAutoInvoke() + && requestIndex < MAXIMUM_INFLIGHT_AUTO_INVOKES; + } else if (functionChoiceBehavior instanceof NoneFunctionChoiceBehavior) { toolChoice = new ChatCompletionsToolSelection(ChatCompletionsToolSelectionPreset.NONE); autoInvoke = false; } else { @@ -1036,13 +1039,14 @@ private static OpenAIToolCallConfig getFunctionChoiceBehaviorConfig( // List of functions advertised to the model List toolDefinitions = functions.stream() - .filter(function -> functionChoiceBehavior.isFunctionAllowed(function.getPluginName(), function.getName())) - .map(OpenAIFunction::getFunctionDefinition) - .map(it -> new ChatCompletionsFunctionToolDefinitionFunction(it.getName()) - .setDescription(it.getDescription()) - .setParameters(it.getParameters())) - .map(ChatCompletionsFunctionToolDefinition::new) - .collect(Collectors.toList()); + .filter(function -> functionChoiceBehavior.isFunctionAllowed(function.getPluginName(), + function.getName())) + .map(OpenAIFunction::getFunctionDefinition) + .map(it -> new ChatCompletionsFunctionToolDefinitionFunction(it.getName()) + .setDescription(it.getDescription()) + .setParameters(it.getParameters())) + .map(ChatCompletionsFunctionToolDefinition::new) + .collect(Collectors.toList()); return new OpenAIToolCallConfig( toolDefinitions, @@ -1100,10 +1104,10 @@ private static OpenAIToolCallConfig getToolCallBehaviorConfig( "{\"type\":\"function\",\"function\":{\"name\":\"%s\"}}", toolChoiceName); toolChoice = new ChatCompletionsToolSelection( - ChatCompletionsNamedToolSelection.fromJson( - DefaultJsonReader.fromString( - json, - new JsonOptions()))); + ChatCompletionsNamedToolSelection.fromJson( + DefaultJsonReader.fromString( + json, + new JsonOptions()))); } catch (JsonProcessingException e) { throw SKException.build("Failed to parse tool choice", e); } catch (IOException e) { @@ -1116,21 +1120,21 @@ private static OpenAIToolCallConfig getToolCallBehaviorConfig( ToolCallBehavior.AllowedKernelFunctions enabledKernelFunctions = (ToolCallBehavior.AllowedKernelFunctions) toolCallBehavior; toolDefinitions = functions.stream() - .filter(function -> { - // check if all kernel functions are enabled - if (enabledKernelFunctions.isAllKernelFunctionsAllowed()) { - return true; - } - // otherwise, check for the specific function - return enabledKernelFunctions.isFunctionAllowed(function.getPluginName(), - function.getName()); - }) - .map(OpenAIFunction::getFunctionDefinition) - .map(it -> new ChatCompletionsFunctionToolDefinitionFunction(it.getName()) - .setDescription(it.getDescription()) - .setParameters(it.getParameters())) - .map(ChatCompletionsFunctionToolDefinition::new) - .collect(Collectors.toList()); + .filter(function -> { + // check if all kernel functions are enabled + if (enabledKernelFunctions.isAllKernelFunctionsAllowed()) { + return true; + } + // otherwise, check for the specific function + return enabledKernelFunctions.isFunctionAllowed(function.getPluginName(), + function.getName()); + }) + .map(OpenAIFunction::getFunctionDefinition) + .map(it -> new ChatCompletionsFunctionToolDefinitionFunction(it.getName()) + .setDescription(it.getDescription()) + .setParameters(it.getParameters())) + .map(ChatCompletionsFunctionToolDefinition::new) + .collect(Collectors.toList()); if (toolDefinitions.isEmpty()) { return null; @@ -1138,10 +1142,12 @@ private static OpenAIToolCallConfig getToolCallBehaviorConfig( } return new OpenAIToolCallConfig( - toolDefinitions, - toolChoice, - toolCallBehavior.isAutoInvokeAllowed() && requestIndex < Math.min(MAXIMUM_INFLIGHT_AUTO_INVOKES, toolCallBehavior.getMaximumAutoInvokeAttempts()), - null); + toolDefinitions, + toolChoice, + toolCallBehavior.isAutoInvokeAllowed() + && requestIndex < Math.min(MAXIMUM_INFLIGHT_AUTO_INVOKES, + toolCallBehavior.getMaximumAutoInvokeAttempts()), + null); } private static boolean hasToolCallBeenExecuted(List chatRequestMessages, diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java index e1f2f249..cf126d09 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java @@ -164,7 +164,7 @@ private static String getSchemaForFunctionParameter(@Nullable InputVariable para entries.add("\"type\":\"" + type + "\""); // Add description if present - String description =null; + String description = null; if (parameter != null && parameter.getDescription() != null && !parameter.getDescription() .isEmpty()) { description = parameter.getDescription(); @@ -173,7 +173,7 @@ private static String getSchemaForFunctionParameter(@Nullable InputVariable para entries.add(String.format("\"description\":\"%s\"", description)); } // If custom type, generate schema - if("object".equalsIgnoreCase(type)) { + if ("object".equalsIgnoreCase(type)) { return getObjectSchema(parameter.getType(), description); } @@ -228,17 +228,17 @@ private static String getJavaTypeToOpenAiFunctionType(String javaType) { } } - private static String getObjectSchema(String type, String description){ - String schema= "{ \"type\" : \"object\" }"; + private static String getObjectSchema(String type, String description) { + String schema = "{ \"type\" : \"object\" }"; try { - Class clazz = Class.forName(type); - schema = ResponseSchemaGenerator.jacksonGenerator().generateSchema(clazz); + Class clazz = Class.forName(type); + schema = ResponseSchemaGenerator.jacksonGenerator().generateSchema(clazz); } catch (ClassNotFoundException | SKException ignored) { } Map properties = BinaryData.fromString(schema).toObject(Map.class); - if(StringUtils.isNotBlank(description)) { + if (StringUtils.isNotBlank(description)) { properties.put("description", description); } return BinaryData.fromObject(properties).toString(); diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIToolCallConfig.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIToolCallConfig.java index df733172..454ed3ce 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIToolCallConfig.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIToolCallConfig.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.aiservices.openai.chatcompletion; import com.azure.ai.openai.models.ChatCompletionsToolDefinition; @@ -26,10 +27,10 @@ public class OpenAIToolCallConfig { */ @SuppressFBWarnings("EI_EXPOSE_REP2") public OpenAIToolCallConfig( - List tools, - ChatCompletionsToolSelection toolChoice, - boolean autoInvoke, - @Nullable FunctionChoiceBehaviorOptions options) { + List tools, + ChatCompletionsToolSelection toolChoice, + boolean autoInvoke, + @Nullable FunctionChoiceBehaviorOptions options) { this.tools = tools; this.toolChoice = toolChoice; this.autoInvoke = autoInvoke; diff --git a/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/JsonSchemaTest.java b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/JsonSchemaTest.java index 68c8dae8..33870fba 100644 --- a/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/JsonSchemaTest.java +++ b/aiservices/openai/src/test/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/JsonSchemaTest.java @@ -47,12 +47,12 @@ public void openAIFunctionTest() { testFunction.getMetadata(), plugin.getName()); - String parameters = "{\"type\":\"object\",\"required\":[\"person\",\"input\"],\"properties\":{\"input\":{\"type\":\"string\",\"description\":\"input string\"},\"person\":{\"type\":\"object\",\"properties\":{\"age\":{\"type\":\"integer\",\"description\":\"The age of the person.\"},\"name\":{\"type\":\"string\",\"description\":\"The name of the person.\"},\"title\":{\"type\":\"string\",\"enum\":[\"MS\",\"MRS\",\"MR\"],\"description\":\"The title of the person.\"}},\"required\":[\"age\",\"name\",\"title\"],\"additionalProperties\":false,\"description\":\"input person\"}}}"; - Assertions.assertEquals(parameters, openAIFunction.getFunctionDefinition().getParameters().toString()); + String parameters = "{\"type\":\"object\",\"required\":[\"person\",\"input\"],\"properties\":{\"input\":{\"type\":\"string\",\"description\":\"input string\"},\"person\":{\"type\":\"object\",\"properties\":{\"age\":{\"type\":\"integer\",\"description\":\"The age of the person.\"},\"name\":{\"type\":\"string\",\"description\":\"The name of the person.\"},\"title\":{\"type\":\"string\",\"enum\":[\"MS\",\"MRS\",\"MR\"],\"description\":\"The title of the person.\"}},\"required\":[\"age\",\"name\",\"title\"],\"additionalProperties\":false,\"description\":\"input person\"}}}"; + Assertions.assertEquals(parameters, + openAIFunction.getFunctionDefinition().getParameters().toString()); } - public static class TestPlugin { @DefineKernelFunction @@ -67,19 +67,16 @@ public Mono asyncTestFunction( return Mono.just(1); } - @DefineKernelFunction(returnType = "int", description = "test function description", - name = "asyncPersonFunction", returnDescription = "test return description") + @DefineKernelFunction(returnType = "int", description = "test function description", name = "asyncPersonFunction", returnDescription = "test return description") public Mono asyncPersonFunction( - @KernelFunctionParameter(name = "person",description = "input person", type = Person.class) Person person, + @KernelFunctionParameter(name = "person", description = "input person", type = Person.class) Person person, @KernelFunctionParameter(name = "input", description = "input string") String input) { return Mono.just(1); } } private static enum Title { - MS, - MRS, - MR + MS, MRS, MR } public static class Person { @@ -90,7 +87,6 @@ public static class Person { @JsonPropertyDescription("The title of the person.") private Title title; - public Person(String name, int age) { this.name = name; this.age = age; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightModelTypeConverter.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightModelTypeConverter.java index 250de12e..2752eb62 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightModelTypeConverter.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightModelTypeConverter.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.samples.demos.lights; import com.google.gson.Gson; @@ -10,14 +11,13 @@ public LightModelTypeConverter() { super( LightModel.class, obj -> { - if(obj instanceof String) { - return gson.fromJson((String)obj, LightModel.class); + if (obj instanceof String) { + return gson.fromJson((String) obj, LightModel.class); } else { return gson.fromJson(gson.toJson(obj), LightModel.class); } }, (types, lightModel) -> gson.toJson(lightModel), - json -> gson.fromJson(json, LightModel.class) - ); + json -> gson.fromJson(json, LightModel.class)); } } diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightsPlugin.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightsPlugin.java index fa11addb..398a8d16 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightsPlugin.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/LightsPlugin.java @@ -27,7 +27,7 @@ public List getLights() { @DefineKernelFunction(name = "add_light", description = "Adds a new light") public String addLight( @KernelFunctionParameter(name = "newLight", description = "new Light Details", type = LightModel.class) LightModel light) { - if( light != null) { + if (light != null) { System.out.println("Adding light " + light.getName()); lights.add(light); return "Light added"; diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubModel.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubModel.java index 180ec8ed..0f8065ee 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubModel.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubModel.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.samples.plugins.github; import com.fasterxml.jackson.annotation.JsonCreator; @@ -8,7 +9,7 @@ public abstract class GitHubModel { public final static ObjectMapper objectMapper = new ObjectMapper() - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); @Override public String toString() { @@ -30,12 +31,13 @@ public static class User extends GitHubModel { private String company; @JsonProperty("html_url") private String url; + @JsonCreator public User(@JsonProperty("login") String login, - @JsonProperty("id") long id, - @JsonProperty("name") String name, - @JsonProperty("company") String company, - @JsonProperty("html_url") String url) { + @JsonProperty("id") long id, + @JsonProperty("name") String name, + @JsonProperty("company") String company, + @JsonProperty("html_url") String url) { this.login = login; this.id = id; this.name = name; @@ -46,15 +48,19 @@ public User(@JsonProperty("login") String login, public String getLogin() { return login; } + public long getId() { return id; } + public String getName() { return name; } + public String getCompany() { return company; } + public String getUrl() { return url; } @@ -69,11 +75,12 @@ public static class Repository extends GitHubModel { private String description; @JsonProperty("html_url") private String url; + @JsonCreator public Repository(@JsonProperty("id") long id, - @JsonProperty("full_name") String name, - @JsonProperty("description") String description, - @JsonProperty("html_url") String url) { + @JsonProperty("full_name") String name, + @JsonProperty("description") String description, + @JsonProperty("html_url") String url) { this.id = id; this.name = name; this.description = description; @@ -83,12 +90,15 @@ public Repository(@JsonProperty("id") long id, public long getId() { return id; } + public String getName() { return name; } + public String getDescription() { return description; } + public String getUrl() { return url; } @@ -123,13 +133,13 @@ public static class Issue extends GitHubModel { @JsonCreator public Issue(@JsonProperty("id") long id, - @JsonProperty("number") long number, - @JsonProperty("title") String title, - @JsonProperty("state") String state, - @JsonProperty("html_url") String url, - @JsonProperty("labels") Label[] labels, - @JsonProperty("created_at") String createdAt, - @JsonProperty("closed_at") String closedAt) { + @JsonProperty("number") long number, + @JsonProperty("title") String title, + @JsonProperty("state") String state, + @JsonProperty("html_url") String url, + @JsonProperty("labels") Label[] labels, + @JsonProperty("created_at") String createdAt, + @JsonProperty("closed_at") String closedAt) { this.id = id; this.number = number; this.title = title; @@ -143,24 +153,31 @@ public Issue(@JsonProperty("id") long id, public long getId() { return id; } + public long getNumber() { return number; } + public String getTitle() { return title; } + public String getState() { return state; } + public String getUrl() { return url; } + public Label[] getLabels() { return labels; } + public String getCreatedAt() { return createdAt; } + public String getClosedAt() { return closedAt; } @@ -172,14 +189,14 @@ public static class IssueDetail extends Issue { @JsonCreator public IssueDetail(@JsonProperty("id") long id, - @JsonProperty("number") long number, - @JsonProperty("title") String title, - @JsonProperty("state") String state, - @JsonProperty("html_url") String url, - @JsonProperty("labels") Label[] labels, - @JsonProperty("created_at") String createdAt, - @JsonProperty("closed_at") String closedAt, - @JsonProperty("body") String body) { + @JsonProperty("number") long number, + @JsonProperty("title") String title, + @JsonProperty("state") String state, + @JsonProperty("html_url") String url, + @JsonProperty("labels") Label[] labels, + @JsonProperty("created_at") String createdAt, + @JsonProperty("closed_at") String closedAt, + @JsonProperty("body") String body) { super(id, number, title, state, url, labels, createdAt, closedAt); this.body = body; } @@ -199,8 +216,8 @@ public static class Label extends GitHubModel { @JsonCreator public Label(@JsonProperty("id") long id, - @JsonProperty("name") String name, - @JsonProperty("description") String description) { + @JsonProperty("name") String name, + @JsonProperty("description") String description) { this.id = id; this.name = name; this.description = description; @@ -209,9 +226,11 @@ public Label(@JsonProperty("id") long id, public long getId() { return id; } + public String getName() { return name; } + public String getDescription() { return description; } diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubPlugin.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubPlugin.java index d3c59a15..f0bddee1 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubPlugin.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubPlugin.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.samples.plugins.github; import reactor.core.publisher.Mono; @@ -12,79 +13,47 @@ public class GitHubPlugin { public static final String baseUrl = "https://api.github.com"; private final String token; - public GitHubPlugin(String token) { + public GitHubPlugin(String token) { this.token = token; } - @DefineKernelFunction(name = "get_user_info", description = "Get user information from GitHub", - returnType = "com.microsoft.semantickernel.samples.plugins.github.GitHubModel$User") + @DefineKernelFunction(name = "get_user_info", description = "Get user information from GitHub", returnType = "com.microsoft.semantickernel.samples.plugins.github.GitHubModel$User") public Mono getUserProfileAsync() { HttpClient client = createClient(); return makeRequestAsync(client, "/user") - .map(json -> { - try { - return GitHubModel.objectMapper.readValue(json, GitHubModel.User.class); - } catch (IOException e) { - throw new IllegalStateException("Failed to deserialize GitHubUser", e); - } - }); + .map(json -> { + try { + return GitHubModel.objectMapper.readValue(json, GitHubModel.User.class); + } catch (IOException e) { + throw new IllegalStateException("Failed to deserialize GitHubUser", e); + } + }); } - @DefineKernelFunction(name = "get_repo_info", description = "Get repository information from GitHub", - returnType = "com.microsoft.semantickernel.samples.plugins.github.GitHubModel$Repository") + @DefineKernelFunction(name = "get_repo_info", description = "Get repository information from GitHub", returnType = "com.microsoft.semantickernel.samples.plugins.github.GitHubModel$Repository") public Mono getRepositoryAsync( - @KernelFunctionParameter( - name = "organization", - description = "The name of the repository to retrieve information for" - ) String organization, - @KernelFunctionParameter( - name = "repo_name", - description = "The name of the repository to retrieve information for" - ) String repoName - ) { + @KernelFunctionParameter(name = "organization", description = "The name of the repository to retrieve information for") String organization, + @KernelFunctionParameter(name = "repo_name", description = "The name of the repository to retrieve information for") String repoName) { HttpClient client = createClient(); return makeRequestAsync(client, String.format("/repos/%s/%s", organization, repoName)) - .map(json -> { - try { - return GitHubModel.objectMapper.readValue(json, GitHubModel.Repository.class); - } catch (IOException e) { - throw new IllegalStateException("Failed to deserialize GitHubRepository", e); - } - }); + .map(json -> { + try { + return GitHubModel.objectMapper.readValue(json, GitHubModel.Repository.class); + } catch (IOException e) { + throw new IllegalStateException("Failed to deserialize GitHubRepository", e); + } + }); } - @DefineKernelFunction(name = "get_issues", description = "Get issues from GitHub", - returnType = "java.util.List") + @DefineKernelFunction(name = "get_issues", description = "Get issues from GitHub", returnType = "java.util.List") public Mono> getIssuesAsync( - @KernelFunctionParameter( - name = "organization", - description = "The name of the organization to retrieve issues for" - ) String organization, - @KernelFunctionParameter( - name = "repo_name", - description = "The name of the repository to retrieve issues for" - ) String repoName, - @KernelFunctionParameter( - name = "max_results", - description = "The maximum number of issues to retrieve", - required = false, - defaultValue = "10", - type = int.class - ) int maxResults, - @KernelFunctionParameter( - name = "state", - description = "The state of the issues to retrieve", - required = false, - defaultValue = "open" - ) String state, - @KernelFunctionParameter( - name = "assignee", - description = "The assignee of the issues to retrieve", - required = false - ) String assignee - ) { + @KernelFunctionParameter(name = "organization", description = "The name of the organization to retrieve issues for") String organization, + @KernelFunctionParameter(name = "repo_name", description = "The name of the repository to retrieve issues for") String repoName, + @KernelFunctionParameter(name = "max_results", description = "The maximum number of issues to retrieve", required = false, defaultValue = "10", type = int.class) int maxResults, + @KernelFunctionParameter(name = "state", description = "The state of the issues to retrieve", required = false, defaultValue = "open") String state, + @KernelFunctionParameter(name = "assignee", description = "The assignee of the issues to retrieve", required = false) String assignee) { HttpClient client = createClient(); String query = String.format("/repos/%s/%s/issues", organization, repoName); @@ -93,58 +62,49 @@ public Mono> getIssuesAsync( query = buildQueryString(query, "per_page", String.valueOf(maxResults)); return makeRequestAsync(client, query) - .flatMap(json -> { - try { - GitHubModel.Issue[] issues = GitHubModel.objectMapper.readValue(json, GitHubModel.Issue[].class); - return Mono.just(List.of(issues)); - } catch (IOException e) { - throw new IllegalStateException("Failed to deserialize GitHubIssues", e); - } - }); + .flatMap(json -> { + try { + GitHubModel.Issue[] issues = GitHubModel.objectMapper.readValue(json, + GitHubModel.Issue[].class); + return Mono.just(List.of(issues)); + } catch (IOException e) { + throw new IllegalStateException("Failed to deserialize GitHubIssues", e); + } + }); } - @DefineKernelFunction(name = "get_issue_detail_info", description = "Get detail information of a single issue from GitHub", - returnType = "com.microsoft.semantickernel.samples.plugins.github.GitHubModel$IssueDetail") + @DefineKernelFunction(name = "get_issue_detail_info", description = "Get detail information of a single issue from GitHub", returnType = "com.microsoft.semantickernel.samples.plugins.github.GitHubModel$IssueDetail") public GitHubModel.IssueDetail getIssueDetailAsync( - @KernelFunctionParameter( - name = "organization", - description = "The name of the repository to retrieve information for" - ) String organization, - @KernelFunctionParameter( - name = "repo_name", - description = "The name of the repository to retrieve information for" - ) String repoName, - @KernelFunctionParameter( - name = "issue_number", - description = "The issue number to retrieve information for", - type = int.class - ) int issueNumber - ) { + @KernelFunctionParameter(name = "organization", description = "The name of the repository to retrieve information for") String organization, + @KernelFunctionParameter(name = "repo_name", description = "The name of the repository to retrieve information for") String repoName, + @KernelFunctionParameter(name = "issue_number", description = "The issue number to retrieve information for", type = int.class) int issueNumber) { HttpClient client = createClient(); - return makeRequestAsync(client, String.format("/repos/%s/%s/issues/%d", organization, repoName, issueNumber)) - .map(json -> { - try { - return GitHubModel.objectMapper.readValue(json, GitHubModel.IssueDetail.class); - } catch (IOException e) { - throw new IllegalStateException("Failed to deserialize GitHubIssue", e); - } - }).block(); + return makeRequestAsync(client, + String.format("/repos/%s/%s/issues/%d", organization, repoName, issueNumber)) + .map(json -> { + try { + return GitHubModel.objectMapper.readValue(json, GitHubModel.IssueDetail.class); + } catch (IOException e) { + throw new IllegalStateException("Failed to deserialize GitHubIssue", e); + } + }).block(); } private HttpClient createClient() { return HttpClient.create() - .baseUrl(baseUrl) - .headers(headers -> { - headers.add("User-Agent", "request"); - headers.add("Accept", "application/vnd.github+json"); - headers.add("Authorization", "Bearer " + token); - headers.add("X-GitHub-Api-Version", "2022-11-28"); - }); + .baseUrl(baseUrl) + .headers(headers -> { + headers.add("User-Agent", "request"); + headers.add("Accept", "application/vnd.github+json"); + headers.add("Authorization", "Bearer " + token); + headers.add("X-GitHub-Api-Version", "2022-11-28"); + }); } private static String buildQueryString(String path, String param, String value) { - if (value == null || value.isEmpty() || value.equals(KernelFunctionParameter.NO_DEFAULT_VALUE)) { + if (value == null || value.isEmpty() + || value.equals(KernelFunctionParameter.NO_DEFAULT_VALUE)) { return path; } @@ -153,13 +113,13 @@ private static String buildQueryString(String path, String param, String value) private Mono makeRequestAsync(HttpClient client, String path) { return client - .get() - .uri(path) - .responseSingle((res, content) -> { - if (res.status().code() != 200) { - return Mono.error(new IllegalStateException("Request failed: " + res.status())); - } - return content.asString(); - }); + .get() + .uri(path) + .responseSingle((res, content) -> { + if (res.status().code() != 200) { + return Mono.error(new IllegalStateException("Request failed: " + res.status())); + } + return content.asString(); + }); } } diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionAgent.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionAgent.java index 4c39645c..336406a3 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionAgent.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/agents/CompletionAgent.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.samples.syntaxexamples.agents; import com.azure.ai.openai.OpenAIAsyncClient; @@ -36,9 +37,10 @@ public class CompletionAgent { // Only required if AZURE_CLIENT_KEY is set private static final String CLIENT_ENDPOINT = System.getenv("CLIENT_ENDPOINT"); private static final String MODEL_ID = System.getenv() - .getOrDefault("MODEL_ID", "gpt-4o"); + .getOrDefault("MODEL_ID", "gpt-4o"); private static final String GITHUB_PAT = System.getenv("GITHUB_PAT"); + public static void main(String[] args) { System.out.println("======== ChatCompletion Agent ========"); @@ -46,67 +48,63 @@ public static void main(String[] args) { if (AZURE_CLIENT_KEY != null) { client = new OpenAIClientBuilder() - .credential(new AzureKeyCredential(AZURE_CLIENT_KEY)) - .endpoint(CLIENT_ENDPOINT) - .buildAsyncClient(); + .credential(new AzureKeyCredential(AZURE_CLIENT_KEY)) + .endpoint(CLIENT_ENDPOINT) + .buildAsyncClient(); } else { client = new OpenAIClientBuilder() - .credential(new KeyCredential(CLIENT_KEY)) - .buildAsyncClient(); + .credential(new KeyCredential(CLIENT_KEY)) + .buildAsyncClient(); } System.out.println("------------------------"); ChatCompletionService chatCompletion = OpenAIChatCompletion.builder() - .withModelId(MODEL_ID) - .withOpenAIAsyncClient(client) - .build(); + .withModelId(MODEL_ID) + .withOpenAIAsyncClient(client) + .build(); Kernel kernel = Kernel.builder() - .withAIService(ChatCompletionService.class, chatCompletion) - .withPlugin(KernelPluginFactory.createFromObject(new GitHubPlugin(GITHUB_PAT), - "GitHubPlugin")) - .build(); + .withAIService(ChatCompletionService.class, chatCompletion) + .withPlugin(KernelPluginFactory.createFromObject(new GitHubPlugin(GITHUB_PAT), + "GitHubPlugin")) + .build(); InvocationContext invocationContext = InvocationContext.builder() - .withFunctionChoiceBehavior(FunctionChoiceBehavior.auto(true)) - .withContextVariableConverter(new ContextVariableTypeConverter<>( - GitHubModel.Issue.class, - o -> (GitHubModel.Issue) o, - o -> o.toString(), - s -> null - )) - .build(); + .withFunctionChoiceBehavior(FunctionChoiceBehavior.auto(true)) + .withContextVariableConverter(new ContextVariableTypeConverter<>( + GitHubModel.Issue.class, + o -> (GitHubModel.Issue) o, + o -> o.toString(), + s -> null)) + .build(); ChatCompletionAgent agent = ChatCompletionAgent.builder() - .withKernel(kernel) - .withKernelArguments( - KernelArguments.builder() - .withVariable("repository", "microsoft/semantic-kernel-java") - .withExecutionSettings(PromptExecutionSettings.builder() - .build()) - .build() - ) - .withInvocationContext(invocationContext) - .withTemplate( - DefaultPromptTemplate.build( - PromptTemplateConfig.builder() - .withTemplate( - """ + .withKernel(kernel) + .withKernelArguments( + KernelArguments.builder() + .withVariable("repository", "microsoft/semantic-kernel-java") + .withExecutionSettings(PromptExecutionSettings.builder() + .build()) + .build()) + .withInvocationContext(invocationContext) + .withTemplate( + DefaultPromptTemplate.build( + PromptTemplateConfig.builder() + .withTemplate( + """ You are an agent designed to query and retrieve information from a single GitHub repository in a read-only manner. You are also able to access the profile of the active user. - + Use the current date and time to provide up-to-date details or time-sensitive responses. - + The repository you are querying is a public repository with the following name: {{$repository}} - + The current date and time is: {{$now}}. - """ - ) - .build() - ) - ).build(); + """) + .build())) + .build(); AgentThread agentThread = new ChatHistoryAgentThread(); Scanner scanner = new Scanner(System.in); @@ -128,9 +126,9 @@ public static void main(String[] args) { message, agentThread, AgentInvokeOptions.builder() - .withKernelArguments(arguments) - .build() - ).block().get(0); + .withKernelArguments(arguments) + .build()) + .block().get(0); System.out.println("> " + response.getMessage()); agentThread = response.getThread(); diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/src/main/java/com/microsoft/semantickernel/samples/openapi/OpenAPIHttpRequestPlugin.java b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/src/main/java/com/microsoft/semantickernel/samples/openapi/OpenAPIHttpRequestPlugin.java index 58b3e525..ebdc8449 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/src/main/java/com/microsoft/semantickernel/samples/openapi/OpenAPIHttpRequestPlugin.java +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/src/main/java/com/microsoft/semantickernel/samples/openapi/OpenAPIHttpRequestPlugin.java @@ -142,7 +142,7 @@ private String buildQueryPath(KernelArguments arguments) { } private static String getRenderedParameter( - KernelArguments arguments, String name) { + KernelArguments arguments, String name) { ContextVariable value = arguments.get(name); if (value == null) { diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/Kernel.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/Kernel.java index d8059ef7..5e6cc2fd 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/Kernel.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/Kernel.java @@ -192,7 +192,7 @@ public FunctionInvocation invokePromptAsync(@Nonnull String prompt, */ public FunctionInvocation invokePromptAsync(@Nonnull String prompt, - @Nonnull KernelArguments arguments, @Nonnull InvocationContext invocationContext) { + @Nonnull KernelArguments arguments, @Nonnull InvocationContext invocationContext) { KernelFunction function = KernelFunction.createFromPrompt(prompt).build(); @@ -327,11 +327,12 @@ public T getService(Class clazz) throws ServiceNotFound * @throws ServiceNotFoundException if the service is not found. * @see com.microsoft.semantickernel.services.AIServiceSelector#trySelectAIService(Class, KernelArguments) */ - public T getService(Class clazz, KernelArguments args) throws ServiceNotFoundException { + public T getService(Class clazz, KernelArguments args) + throws ServiceNotFoundException { AIServiceSelection selector = serviceSelector - .trySelectAIService( - clazz, - args); + .trySelectAIService( + clazz, + args); if (selector == null) { throw new ServiceNotFoundException("Unable to find service of type " + clazz.getName()); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/Agent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/Agent.java index 2277d522..f69b2152 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/Agent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/Agent.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.agents; import java.util.HashMap; @@ -51,7 +52,8 @@ public interface Agent { * @param message The message to process * @return A Mono containing the agent response */ - Mono>>> invokeAsync(@Nullable ChatMessageContent message); + Mono>>> invokeAsync( + @Nullable ChatMessageContent message); /** * Invokes the agent with the given message and thread. @@ -60,8 +62,9 @@ public interface Agent { * @param thread The agent thread to use * @return A Mono containing the agent response */ - Mono>>> invokeAsync(@Nullable ChatMessageContent message, - @Nullable AgentThread thread); + Mono>>> invokeAsync( + @Nullable ChatMessageContent message, + @Nullable AgentThread thread); /** * Invokes the agent with the given message, thread, and options. @@ -71,9 +74,10 @@ Mono>>> invokeAsync(@Nullable ChatM * @param options The options for invoking the agent * @return A Mono containing the agent response */ - Mono>>> invokeAsync(@Nullable ChatMessageContent message, - @Nullable AgentThread thread, - @Nullable AgentInvokeOptions options); + Mono>>> invokeAsync( + @Nullable ChatMessageContent message, + @Nullable AgentThread thread, + @Nullable AgentInvokeOptions options); /** * Invoke the agent with the given chat history. @@ -83,9 +87,10 @@ Mono>>> invokeAsync(@Nullable ChatM * @param options The options for invoking the agent * @return A Mono containing the agent response */ - Mono>>> invokeAsync(List> messages, - @Nullable AgentThread thread, - @Nullable AgentInvokeOptions options); + Mono>>> invokeAsync( + List> messages, + @Nullable AgentThread thread, + @Nullable AgentInvokeOptions options); /** * Notifies the agent of a new message. diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java index 559a062c..6b6d57ed 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentInvokeOptions.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.agents; import com.microsoft.semantickernel.Kernel; @@ -38,9 +39,9 @@ public AgentInvokeOptions() { * @param invocationContext The invocation context. */ public AgentInvokeOptions(@Nullable KernelArguments kernelArguments, - @Nullable Kernel kernel, - @Nullable String additionalInstructions, - @Nullable InvocationContext invocationContext) { + @Nullable Kernel kernel, + @Nullable String additionalInstructions, + @Nullable InvocationContext invocationContext) { this.kernelArguments = kernelArguments != null ? kernelArguments.copy() : null; this.kernel = kernel; this.additionalInstructions = additionalInstructions; @@ -84,8 +85,6 @@ public InvocationContext getInvocationContext() { return invocationContext; } - - /** * Builder for AgentInvokeOptions. */ @@ -156,8 +155,7 @@ public AgentInvokeOptions build() { kernelArguments, kernel, additionalInstructions, - invocationContext - ); + invocationContext); } } } \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentResponseItem.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentResponseItem.java index f585bfea..0b455098 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentResponseItem.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentResponseItem.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.agents; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentThread.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentThread.java index d369d999..94538f41 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentThread.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/AgentThread.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.agents; import com.microsoft.semantickernel.services.chatcompletion.ChatMessageContent; diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/BaseAgentThread.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/BaseAgentThread.java index b7c97eea..c66fe9b9 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/BaseAgentThread.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/BaseAgentThread.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.agents; public abstract class BaseAgentThread implements AgentThread { @@ -16,6 +17,7 @@ public BaseAgentThread(String id) { public String getId() { return id; } + @Override public boolean isDeleted() { return isDeleted; diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java index 4d30569f..8403093e 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/agents/KernelAgent.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.agents; import com.microsoft.semantickernel.Kernel; @@ -31,23 +32,24 @@ public abstract class KernelAgent implements Agent { protected final PromptTemplate template; protected KernelAgent( - String id, - String name, - String description, - Kernel kernel, - KernelArguments kernelArguments, - InvocationContext invocationContext, - String instructions, - PromptTemplate template - ) { + String id, + String name, + String description, + Kernel kernel, + KernelArguments kernelArguments, + InvocationContext invocationContext, + String instructions, + PromptTemplate template) { this.id = id != null ? id : UUID.randomUUID().toString(); this.name = name; this.description = description; this.kernel = kernel; this.kernelArguments = kernelArguments != null - ? kernelArguments.copy() : KernelArguments.builder().build(); + ? kernelArguments.copy() + : KernelArguments.builder().build(); this.invocationContext = invocationContext != null - ? invocationContext : InvocationContext.builder().build(); + ? invocationContext + : InvocationContext.builder().build(); this.instructions = instructions; this.template = template; } @@ -127,14 +129,15 @@ protected KernelArguments mergeArguments(KernelArguments arguments) { return kernelArguments; } - Map executionSettings = new HashMap<>(kernelArguments.getExecutionSettings()); + Map executionSettings = new HashMap<>( + kernelArguments.getExecutionSettings()); executionSettings.putAll(arguments.getExecutionSettings()); return KernelArguments.builder() - .withVariables(kernelArguments) - .withVariables(arguments) - .withExecutionSettings(executionSettings) - .build(); + .withVariables(kernelArguments) + .withVariables(arguments) + .withExecutionSettings(executionSettings) + .build(); } /** @@ -145,7 +148,8 @@ protected KernelArguments mergeArguments(KernelArguments arguments) { * @param context The context to use for formatting. * @return A Mono that resolves to the formatted instructions. */ - protected Mono renderInstructionsAsync(Kernel kernel, KernelArguments arguments, InvocationContext context) { + protected Mono renderInstructionsAsync(Kernel kernel, KernelArguments arguments, + InvocationContext context) { if (template != null) { return template.renderAsync(kernel, arguments, context); } else { @@ -153,38 +157,41 @@ protected Mono renderInstructionsAsync(Kernel kernel, KernelArguments ar } } - protected Mono ensureThreadExistsWithMessagesAsync(List> messages, AgentThread thread, Supplier threadSupplier) { + protected Mono ensureThreadExistsWithMessagesAsync( + List> messages, AgentThread thread, Supplier threadSupplier) { return Mono.defer(() -> { // Check if the thread already exists // If it does, we can work with a copy of it AgentThread newThread = thread == null ? threadSupplier.get() : thread.copy(); - return newThread.createAsync() - .thenMany(Flux.fromIterable(messages)) - .concatMap(message -> { - return notifyThreadOfNewMessageAsync(newThread, message) - .then(Mono.just(message)); - }) - .then(Mono.just((T) newThread)); + return newThread.createAsync() + .thenMany(Flux.fromIterable(messages)) + .concatMap(message -> { + return notifyThreadOfNewMessageAsync(newThread, message) + .then(Mono.just(message)); + }) + .then(Mono.just((T) newThread)); }); } @Override - public Mono>>> invokeAsync(@Nullable ChatMessageContent message) { + public Mono>>> invokeAsync( + @Nullable ChatMessageContent message) { return invokeAsync(message, null, null); } @Override - public Mono>>> invokeAsync(@Nullable ChatMessageContent message, - @Nullable AgentThread thread) { + public Mono>>> invokeAsync( + @Nullable ChatMessageContent message, + @Nullable AgentThread thread) { return invokeAsync(message, thread, null); } @Override public Mono>>> invokeAsync( - @Nullable ChatMessageContent message, - @Nullable AgentThread thread, - @Nullable AgentInvokeOptions options) { + @Nullable ChatMessageContent message, + @Nullable AgentThread thread, + @Nullable AgentInvokeOptions options) { ArrayList> messages = new ArrayList<>(); if (message != null) { messages.add(message); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/AutoFunctionChoiceBehavior.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/AutoFunctionChoiceBehavior.java index 3e3878b7..b4993f44 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/AutoFunctionChoiceBehavior.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/AutoFunctionChoiceBehavior.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.functionchoice; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; @@ -22,8 +23,8 @@ public class AutoFunctionChoiceBehavior extends FunctionChoiceBehavior { * @param options Options for the function choice behavior. */ public AutoFunctionChoiceBehavior(boolean autoInvoke, - @Nullable List> functions, - @Nullable FunctionChoiceBehaviorOptions options) { + @Nullable List> functions, + @Nullable FunctionChoiceBehaviorOptions options) { super(functions, options); this.autoInvoke = autoInvoke; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java index bfa39785..cbf64bde 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java @@ -20,14 +20,14 @@ public abstract class FunctionChoiceBehavior { protected final FunctionChoiceBehaviorOptions options; protected FunctionChoiceBehavior(@Nullable List> functions, - @Nullable FunctionChoiceBehaviorOptions options) { + @Nullable FunctionChoiceBehaviorOptions options) { this.functions = functions != null ? Collections.unmodifiableList(functions) : null; this.fullFunctionNames = new HashSet<>(); if (functions != null) { functions.stream().filter(Objects::nonNull).forEach( - f -> this.fullFunctionNames - .add(formFullFunctionName(f.getPluginName(), f.getName()))); + f -> this.fullFunctionNames + .add(formFullFunctionName(f.getPluginName(), f.getName()))); } if (options != null) { @@ -79,8 +79,8 @@ public static FunctionChoiceBehavior auto(boolean autoInvoke) { * @return A new FunctionChoiceBehavior instance with all kernel functions allowed. */ public static FunctionChoiceBehavior auto(boolean autoInvoke, - List> functions, - @Nullable FunctionChoiceBehaviorOptions options) { + List> functions, + @Nullable FunctionChoiceBehaviorOptions options) { return new AutoFunctionChoiceBehavior(autoInvoke, functions, options); } @@ -96,8 +96,8 @@ public static FunctionChoiceBehavior auto(boolean autoInvoke, * @return A new FunctionChoiceBehavior instance with the required function. */ public static FunctionChoiceBehavior required(boolean autoInvoke, - List> functions, - @Nullable FunctionChoiceBehaviorOptions options) { + List> functions, + @Nullable FunctionChoiceBehaviorOptions options) { return new RequiredFunctionChoiceBehavior(autoInvoke, functions, options); } @@ -110,11 +110,10 @@ public static FunctionChoiceBehavior required(boolean autoInvoke, * If empty, no functions are provided to the model, which is equivalent to disabling function calling. */ public static FunctionChoiceBehavior none(List> functions, - @Nullable FunctionChoiceBehaviorOptions options) { + @Nullable FunctionChoiceBehaviorOptions options) { return new NoneFunctionChoiceBehavior(functions, options); } - /** * The separator between the plugin name and the function name. */ diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehaviorOptions.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehaviorOptions.java index bbde7c57..ffb17c78 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehaviorOptions.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehaviorOptions.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.functionchoice; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/NoneFunctionChoiceBehavior.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/NoneFunctionChoiceBehavior.java index 0ee7f1a0..e5bef9f4 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/NoneFunctionChoiceBehavior.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/NoneFunctionChoiceBehavior.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.functionchoice; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; @@ -9,7 +10,8 @@ public class NoneFunctionChoiceBehavior extends FunctionChoiceBehavior { /** * Create a new instance of NoneFunctionChoiceBehavior. */ - public NoneFunctionChoiceBehavior(List> functions, FunctionChoiceBehaviorOptions options) { + public NoneFunctionChoiceBehavior(List> functions, + FunctionChoiceBehaviorOptions options) { super(functions, options); } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/RequiredFunctionChoiceBehavior.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/RequiredFunctionChoiceBehavior.java index 57312b0f..01710cca 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/RequiredFunctionChoiceBehavior.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/RequiredFunctionChoiceBehavior.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.functionchoice; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; @@ -13,7 +14,8 @@ public class RequiredFunctionChoiceBehavior extends AutoFunctionChoiceBehavior { * @param functions A set of functions to advertise to the model. * @param options Options for the function choice behavior. */ - public RequiredFunctionChoiceBehavior(boolean autoInvoke, List> functions, FunctionChoiceBehaviorOptions options) { + public RequiredFunctionChoiceBehavior(boolean autoInvoke, List> functions, + FunctionChoiceBehaviorOptions options) { super(autoInvoke, functions, options); } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/InvocationContext.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/InvocationContext.java index 48c10949..6a8547c4 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/InvocationContext.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/InvocationContext.java @@ -323,7 +323,8 @@ public InvocationContext build() { if (telemetry == null) { telemetry = new SemanticKernelTelemetry(); } - return new InvocationContext(hooks, promptExecutionSettings, toolCallBehavior, functionChoiceBehavior, + return new InvocationContext(hooks, promptExecutionSettings, toolCallBehavior, + functionChoiceBehavior, contextVariableTypes, invocationReturnMode, telemetry); } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java index bd519f61..bf9e6565 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelArguments.java @@ -42,8 +42,8 @@ public class KernelArguments implements Map> { * @param variables The variables to use for the function invocation. */ protected KernelArguments( - @Nullable Map> variables, - @Nullable Map executionSettings) { + @Nullable Map> variables, + @Nullable Map executionSettings) { if (variables == null) { this.variables = new CaseInsensitiveMap<>(); } else { @@ -112,14 +112,14 @@ public ContextVariable getInput() { */ public String prettyPrint() { return variables.entrySet().stream() - .reduce( - "", - (str, entry) -> str - + System.lineSeparator() - + entry.getKey() - + ": " - + entry.getValue().toPromptString(ContextVariableTypes.getGlobalTypes()), - (a, b) -> a + b); + .reduce( + "", + (str, entry) -> str + + System.lineSeparator() + + entry.getKey() + + ": " + + entry.getValue().toPromptString(ContextVariableTypes.getGlobalTypes()), + (a, b) -> a + b); } /** @@ -149,9 +149,9 @@ ContextVariable get(String key, Class clazz) { } throw new SKException( - String.format( - "Variable %s is of type %s, but requested type is %s", - key, value.getType().getClazz(), clazz)); + String.format( + "Variable %s is of type %s, but requested type is %s", + key, value.getType().getClazz(), clazz)); } /** @@ -243,7 +243,6 @@ public static Builder builder() { return new Builder<>(KernelArguments::new); } - /** * Builder for ContextVariables */ @@ -253,7 +252,6 @@ public static class Builder implements SemanticKernel private final Map> variables; private final Map executionSettings; - protected Builder(Function constructor) { this.constructor = constructor; this.variables = new HashMap<>(); @@ -293,10 +291,10 @@ public Builder withInput(Object content) { */ public Builder withInput(T content, ContextVariableTypeConverter typeConverter) { return withInput(new ContextVariable<>( - new ContextVariableType<>( - typeConverter, - typeConverter.getType()), - content)); + new ContextVariableType<>( + typeConverter, + typeConverter.getType()), + content)); } /** @@ -352,12 +350,12 @@ public Builder withVariable(String key, Object value) { * @throws SKException if the value cannot be converted to a ContextVariable */ public Builder withVariable(String key, T value, - ContextVariableTypeConverter typeConverter) { + ContextVariableTypeConverter typeConverter) { return withVariable(key, new ContextVariable<>( - new ContextVariableType<>( - typeConverter, - typeConverter.getType()), - value)); + new ContextVariableType<>( + typeConverter, + typeConverter.getType()), + value)); } /** @@ -376,7 +374,8 @@ public Builder withExecutionSettings(PromptExecutionSettings executionSetting * @param executionSettings Execution settings * @return {$code this} Builder for fluent coding */ - public Builder withExecutionSettings(Map executionSettings) { + public Builder withExecutionSettings( + Map executionSettings) { if (executionSettings == null) { return this; } @@ -402,17 +401,15 @@ public Builder withExecutionSettings(List executionS if (this.executionSettings.containsKey(serviceId)) { if (serviceId.equals(PromptExecutionSettings.DEFAULT_SERVICE_ID)) { throw new SKException( - String.format( - "Multiple prompt execution settings with the default service id '%s' or no service id have been provided. Specify a single default prompt execution settings and provide a unique service id for all other instances.", - PromptExecutionSettings.DEFAULT_SERVICE_ID) - ); + String.format( + "Multiple prompt execution settings with the default service id '%s' or no service id have been provided. Specify a single default prompt execution settings and provide a unique service id for all other instances.", + PromptExecutionSettings.DEFAULT_SERVICE_ID)); } throw new SKException( - String.format( - "Multiple prompt execution settings with the service id '%s' have been provided. Specify a unique service id for all instances.", - serviceId) - ); + String.format( + "Multiple prompt execution settings with the service id '%s' have been provided. Specify a unique service id for all instances.", + serviceId)); } this.executionSettings.put(serviceId, settings); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java index babc1da6..642b5add 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java @@ -135,9 +135,9 @@ private Flux> invokeInternalAsync( .executeHooks(new FunctionInvokingEvent(this, args)); args = KernelArguments.builder() - .withVariables(invokingEvent.getArguments()) - .withExecutionSettings(this.getExecutionSettings()) - .build(); + .withVariables(invokingEvent.getArguments()) + .withExecutionSettings(this.getExecutionSettings()) + .build(); AIServiceSelection aiServiceSelection = kernel .getServiceSelector() diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/BaseAIServiceSelector.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/BaseAIServiceSelector.java index 03ecdd78..b022bf35 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/BaseAIServiceSelector.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/BaseAIServiceSelector.java @@ -38,8 +38,8 @@ public AIServiceSelection trySelectAIService( @Override @Nullable public AIServiceSelection trySelectAIService( - Class serviceType, - @Nullable KernelArguments arguments) { + Class serviceType, + @Nullable KernelArguments arguments) { return trySelectAIService(serviceType, arguments, services); } @@ -64,7 +64,6 @@ protected abstract AIServiceSelection trySelectAIServic @Nullable KernelArguments arguments, Map, AIService> services); - /** * Resolves an {@link AIService} from the {@code services} argument using the specified * {@code function} and {@code arguments} for selection. @@ -79,9 +78,9 @@ protected abstract AIServiceSelection trySelectAIServic */ @Nullable protected AIServiceSelection trySelectAIService( - Class serviceType, - @Nullable KernelArguments arguments, - Map, AIService> services) { + Class serviceType, + @Nullable KernelArguments arguments, + Map, AIService> services) { return trySelectAIService(serviceType, null, arguments, services); } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/OrderedAIServiceSelector.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/OrderedAIServiceSelector.java index 6b04fe2b..30828233 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/OrderedAIServiceSelector.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/OrderedAIServiceSelector.java @@ -66,16 +66,16 @@ public AIServiceSelection trySelectAIService( Map, AIService> services) { if (function == null) { - return selectAIService(serviceType, arguments != null ? arguments.getExecutionSettings() : null); + return selectAIService(serviceType, + arguments != null ? arguments.getExecutionSettings() : null); } return selectAIService(serviceType, function.getExecutionSettings()); } - private AIServiceSelection selectAIService( - Class serviceType, - @Nullable Map executionSettings) { + Class serviceType, + @Nullable Map executionSettings) { if (executionSettings == null || executionSettings.isEmpty()) { AIService service = getAnyService(serviceType); @@ -84,50 +84,50 @@ private AIServiceSelection selectAIService( } } else { AIServiceSelection selection = executionSettings - .entrySet() - .stream() - .map(keyValue -> { - - PromptExecutionSettings settings = keyValue.getValue(); - String serviceId = keyValue.getKey(); - - if (!Verify.isNullOrEmpty(serviceId)) { - AIService service = getService(serviceId); - if (service != null) { - return castServiceSelection( - new AIServiceSelection<>(service, settings)); - } + .entrySet() + .stream() + .map(keyValue -> { + + PromptExecutionSettings settings = keyValue.getValue(); + String serviceId = keyValue.getKey(); + + if (!Verify.isNullOrEmpty(serviceId)) { + AIService service = getService(serviceId); + if (service != null) { + return castServiceSelection( + new AIServiceSelection<>(service, settings)); } + } - return null; - }) - .filter(Objects::nonNull) - .findFirst() - .orElseGet(() -> null); + return null; + }) + .filter(Objects::nonNull) + .findFirst() + .orElseGet(() -> null); if (selection != null) { return castServiceSelection(selection); } selection = executionSettings - .entrySet() - .stream() - .map(keyValue -> { - PromptExecutionSettings settings = keyValue.getValue(); - - if (!Verify.isNullOrEmpty(settings.getModelId())) { - AIService service = getServiceByModelId(settings.getModelId()); - if (service != null) { - return castServiceSelection( - new AIServiceSelection<>(service, settings)); - } + .entrySet() + .stream() + .map(keyValue -> { + PromptExecutionSettings settings = keyValue.getValue(); + + if (!Verify.isNullOrEmpty(settings.getModelId())) { + AIService service = getServiceByModelId(settings.getModelId()); + if (service != null) { + return castServiceSelection( + new AIServiceSelection<>(service, settings)); } + } - return null; - }) - .filter(Objects::nonNull) - .findFirst() - .orElseGet(() -> null); + return null; + }) + .filter(Objects::nonNull) + .findFirst() + .orElseGet(() -> null); if (selection != null) { return castServiceSelection(selection); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java index 0408860e..e06472d1 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java @@ -60,17 +60,17 @@ public ChatMessageContent( * @param content the content */ public ChatMessageContent( - AuthorRole authorRole, - String authorName, - String content) { + AuthorRole authorRole, + String authorName, + String content) { this( - authorRole, - authorName, - content, - null, - null, - null, - null); + authorRole, + authorName, + content, + null, + null, + null, + null); } /** From c535e613f63e1af8855f81bbad6e8f8fd53d727e Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 1 May 2025 11:02:13 -0700 Subject: [PATCH 155/277] Small fix --- .../agents/chatcompletion/ChatCompletionAgent.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java index 140df0f4..b5294fe6 100644 --- a/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java +++ b/agents/semantickernel-agents-core/src/main/java/com/microsoft/semantickernel/agents/chatcompletion/ChatCompletionAgent.java @@ -104,7 +104,7 @@ private Mono>> internalInvokeAsync( PromptExecutionSettings executionSettings = invocationContext != null && invocationContext.getPromptExecutionSettings() != null ? invocationContext.getPromptExecutionSettings() - : kernelArguments.getExecutionSettings() + : arguments.getExecutionSettings() .get(chatCompletionService.getServiceId()); // Build base invocation context From 730af6c571f7a53f0b63f0808149e77fb12e27ef Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 1 May 2025 12:55:50 -0700 Subject: [PATCH 156/277] Add Github plugin to learn resources --- .../samples/plugins/github/GitHubModel.java | 238 ++++++++++++++++++ .../samples/plugins/github/GitHubPlugin.java | 125 +++++++++ 2 files changed, 363 insertions(+) create mode 100644 samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubModel.java create mode 100644 samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubPlugin.java diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubModel.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubModel.java new file mode 100644 index 00000000..0f8065ee --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubModel.java @@ -0,0 +1,238 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.plugins.github; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +public abstract class GitHubModel { + public final static ObjectMapper objectMapper = new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + @Override + public String toString() { + try { + return objectMapper.writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + public static class User extends GitHubModel { + @JsonProperty("login") + private String login; + @JsonProperty("id") + private long id; + @JsonProperty("name") + private String name; + @JsonProperty("company") + private String company; + @JsonProperty("html_url") + private String url; + + @JsonCreator + public User(@JsonProperty("login") String login, + @JsonProperty("id") long id, + @JsonProperty("name") String name, + @JsonProperty("company") String company, + @JsonProperty("html_url") String url) { + this.login = login; + this.id = id; + this.name = name; + this.company = company; + this.url = url; + } + + public String getLogin() { + return login; + } + + public long getId() { + return id; + } + + public String getName() { + return name; + } + + public String getCompany() { + return company; + } + + public String getUrl() { + return url; + } + } + + public static class Repository extends GitHubModel { + @JsonProperty("id") + private long id; + @JsonProperty("full_name") + private String name; + @JsonProperty("description") + private String description; + @JsonProperty("html_url") + private String url; + + @JsonCreator + public Repository(@JsonProperty("id") long id, + @JsonProperty("full_name") String name, + @JsonProperty("description") String description, + @JsonProperty("html_url") String url) { + this.id = id; + this.name = name; + this.description = description; + this.url = url; + } + + public long getId() { + return id; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public String getUrl() { + return url; + } + + @Override + public String toString() { + try { + return objectMapper.writeValueAsString(this); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + } + + public static class Issue extends GitHubModel { + @JsonProperty("id") + private long id; + @JsonProperty("number") + private long number; + @JsonProperty("title") + private String title; + @JsonProperty("state") + private String state; + @JsonProperty("html_url") + private String url; + @JsonProperty("labels") + private Label[] labels; + @JsonProperty("created_at") + private String createdAt; + @JsonProperty("closed_at") + private String closedAt; + + @JsonCreator + public Issue(@JsonProperty("id") long id, + @JsonProperty("number") long number, + @JsonProperty("title") String title, + @JsonProperty("state") String state, + @JsonProperty("html_url") String url, + @JsonProperty("labels") Label[] labels, + @JsonProperty("created_at") String createdAt, + @JsonProperty("closed_at") String closedAt) { + this.id = id; + this.number = number; + this.title = title; + this.state = state; + this.url = url; + this.labels = labels; + this.createdAt = createdAt; + this.closedAt = closedAt; + } + + public long getId() { + return id; + } + + public long getNumber() { + return number; + } + + public String getTitle() { + return title; + } + + public String getState() { + return state; + } + + public String getUrl() { + return url; + } + + public Label[] getLabels() { + return labels; + } + + public String getCreatedAt() { + return createdAt; + } + + public String getClosedAt() { + return closedAt; + } + } + + public static class IssueDetail extends Issue { + @JsonProperty("body") + private String body; + + @JsonCreator + public IssueDetail(@JsonProperty("id") long id, + @JsonProperty("number") long number, + @JsonProperty("title") String title, + @JsonProperty("state") String state, + @JsonProperty("html_url") String url, + @JsonProperty("labels") Label[] labels, + @JsonProperty("created_at") String createdAt, + @JsonProperty("closed_at") String closedAt, + @JsonProperty("body") String body) { + super(id, number, title, state, url, labels, createdAt, closedAt); + this.body = body; + } + + public String getBody() { + return body; + } + } + + public static class Label extends GitHubModel { + @JsonProperty("id") + private long id; + @JsonProperty("name") + private String name; + @JsonProperty("description") + private String description; + + @JsonCreator + public Label(@JsonProperty("id") long id, + @JsonProperty("name") String name, + @JsonProperty("description") String description) { + this.id = id; + this.name = name; + this.description = description; + } + + public long getId() { + return id; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + } +} diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubPlugin.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubPlugin.java new file mode 100644 index 00000000..f0bddee1 --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/plugins/github/GitHubPlugin.java @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.plugins.github; + +import reactor.core.publisher.Mono; +import reactor.netty.http.client.HttpClient; +import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; +import com.microsoft.semantickernel.semanticfunctions.annotations.KernelFunctionParameter; + +import java.io.IOException; +import java.util.List; + +public class GitHubPlugin { + public static final String baseUrl = "https://api.github.com"; + private final String token; + + public GitHubPlugin(String token) { + this.token = token; + } + + @DefineKernelFunction(name = "get_user_info", description = "Get user information from GitHub", returnType = "com.microsoft.semantickernel.samples.plugins.github.GitHubModel$User") + public Mono getUserProfileAsync() { + HttpClient client = createClient(); + + return makeRequestAsync(client, "/user") + .map(json -> { + try { + return GitHubModel.objectMapper.readValue(json, GitHubModel.User.class); + } catch (IOException e) { + throw new IllegalStateException("Failed to deserialize GitHubUser", e); + } + }); + } + + @DefineKernelFunction(name = "get_repo_info", description = "Get repository information from GitHub", returnType = "com.microsoft.semantickernel.samples.plugins.github.GitHubModel$Repository") + public Mono getRepositoryAsync( + @KernelFunctionParameter(name = "organization", description = "The name of the repository to retrieve information for") String organization, + @KernelFunctionParameter(name = "repo_name", description = "The name of the repository to retrieve information for") String repoName) { + HttpClient client = createClient(); + + return makeRequestAsync(client, String.format("/repos/%s/%s", organization, repoName)) + .map(json -> { + try { + return GitHubModel.objectMapper.readValue(json, GitHubModel.Repository.class); + } catch (IOException e) { + throw new IllegalStateException("Failed to deserialize GitHubRepository", e); + } + }); + } + + @DefineKernelFunction(name = "get_issues", description = "Get issues from GitHub", returnType = "java.util.List") + public Mono> getIssuesAsync( + @KernelFunctionParameter(name = "organization", description = "The name of the organization to retrieve issues for") String organization, + @KernelFunctionParameter(name = "repo_name", description = "The name of the repository to retrieve issues for") String repoName, + @KernelFunctionParameter(name = "max_results", description = "The maximum number of issues to retrieve", required = false, defaultValue = "10", type = int.class) int maxResults, + @KernelFunctionParameter(name = "state", description = "The state of the issues to retrieve", required = false, defaultValue = "open") String state, + @KernelFunctionParameter(name = "assignee", description = "The assignee of the issues to retrieve", required = false) String assignee) { + HttpClient client = createClient(); + + String query = String.format("/repos/%s/%s/issues", organization, repoName); + query = buildQueryString(query, "state", state); + query = buildQueryString(query, "assignee", assignee); + query = buildQueryString(query, "per_page", String.valueOf(maxResults)); + + return makeRequestAsync(client, query) + .flatMap(json -> { + try { + GitHubModel.Issue[] issues = GitHubModel.objectMapper.readValue(json, + GitHubModel.Issue[].class); + return Mono.just(List.of(issues)); + } catch (IOException e) { + throw new IllegalStateException("Failed to deserialize GitHubIssues", e); + } + }); + } + + @DefineKernelFunction(name = "get_issue_detail_info", description = "Get detail information of a single issue from GitHub", returnType = "com.microsoft.semantickernel.samples.plugins.github.GitHubModel$IssueDetail") + public GitHubModel.IssueDetail getIssueDetailAsync( + @KernelFunctionParameter(name = "organization", description = "The name of the repository to retrieve information for") String organization, + @KernelFunctionParameter(name = "repo_name", description = "The name of the repository to retrieve information for") String repoName, + @KernelFunctionParameter(name = "issue_number", description = "The issue number to retrieve information for", type = int.class) int issueNumber) { + HttpClient client = createClient(); + + return makeRequestAsync(client, + String.format("/repos/%s/%s/issues/%d", organization, repoName, issueNumber)) + .map(json -> { + try { + return GitHubModel.objectMapper.readValue(json, GitHubModel.IssueDetail.class); + } catch (IOException e) { + throw new IllegalStateException("Failed to deserialize GitHubIssue", e); + } + }).block(); + } + + private HttpClient createClient() { + return HttpClient.create() + .baseUrl(baseUrl) + .headers(headers -> { + headers.add("User-Agent", "request"); + headers.add("Accept", "application/vnd.github+json"); + headers.add("Authorization", "Bearer " + token); + headers.add("X-GitHub-Api-Version", "2022-11-28"); + }); + } + + private static String buildQueryString(String path, String param, String value) { + if (value == null || value.isEmpty() + || value.equals(KernelFunctionParameter.NO_DEFAULT_VALUE)) { + return path; + } + + return path + (path.contains("?") ? "&" : "?") + param + "=" + value; + } + + private Mono makeRequestAsync(HttpClient client, String path) { + return client + .get() + .uri(path) + .responseSingle((res, content) -> { + if (res.status().code() != 200) { + return Mono.error(new IllegalStateException("Request failed: " + res.status())); + } + return content.asString(); + }); + } +} From d3c8fce441658813ff43095236e4983595e3bad1 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Mon, 5 May 2025 09:29:50 -0700 Subject: [PATCH 157/277] Add additional FunctionChoiceBehavior creators --- .../FunctionChoiceBehavior.java | 49 +++++++++++++++++-- .../NoneFunctionChoiceBehavior.java | 5 +- .../RequiredFunctionChoiceBehavior.java | 6 ++- 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java index cbf64bde..f0024952 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java @@ -66,6 +66,22 @@ public static FunctionChoiceBehavior auto(boolean autoInvoke) { return new AutoFunctionChoiceBehavior(autoInvoke, null, null); } + /** + * Gets an instance of the FunctionChoiceBehavior that provides either all the Kernel's plugins functions to the AI model to call or specific functions. + * + * @param autoInvoke Enable or disable auto-invocation. + * If auto-invocation is enabled, the model may request that the Semantic Kernel + * invoke the kernel functions and return the value to the model. + * @param functions Functions to provide to the model. If null, all the Kernel's plugins' functions are provided to the model. + * If empty, no functions are provided to the model, which is equivalent to disabling function calling. + * + * @return A new FunctionChoiceBehavior instance with all kernel functions allowed. + */ + public static FunctionChoiceBehavior auto(boolean autoInvoke, + @Nullable List> functions) { + return new AutoFunctionChoiceBehavior(autoInvoke, functions, null); + } + /** * Gets an instance of the FunctionChoiceBehavior that provides either all the Kernel's plugins functions to the AI model to call or specific functions. * @@ -79,11 +95,25 @@ public static FunctionChoiceBehavior auto(boolean autoInvoke) { * @return A new FunctionChoiceBehavior instance with all kernel functions allowed. */ public static FunctionChoiceBehavior auto(boolean autoInvoke, - List> functions, - @Nullable FunctionChoiceBehaviorOptions options) { + @Nullable List> functions, + @Nullable FunctionChoiceBehaviorOptions options) { return new AutoFunctionChoiceBehavior(autoInvoke, functions, options); } + /** + * Gets an instance of the FunctionChoiceBehavior that provides either all the Kernel's plugins functions to the AI model to call or specific functions. + *

+ * This behavior forces the model to call the provided functions. + * SK connectors will invoke a requested function or multiple requested functions if the model requests multiple ones in one request, + * while handling the first request, and stop advertising the functions for the following requests to prevent the model from repeatedly calling the same function(s). + * + * @return A new FunctionChoiceBehavior instance with the required function. + */ + public static FunctionChoiceBehavior required(boolean autoInvoke, + @Nullable List> functions) { + return new RequiredFunctionChoiceBehavior(autoInvoke, functions, null); + } + /** * Gets an instance of the FunctionChoiceBehavior that provides either all the Kernel's plugins functions to the AI model to call or specific functions. *

@@ -96,11 +126,20 @@ public static FunctionChoiceBehavior auto(boolean autoInvoke, * @return A new FunctionChoiceBehavior instance with the required function. */ public static FunctionChoiceBehavior required(boolean autoInvoke, - List> functions, - @Nullable FunctionChoiceBehaviorOptions options) { + @Nullable List> functions, + @Nullable FunctionChoiceBehaviorOptions options) { return new RequiredFunctionChoiceBehavior(autoInvoke, functions, options); } + /** + * Gets an instance of the FunctionChoiceBehavior that provides either all the Kernel's plugins functions to the AI model to call or specific functions. + *

+ * This behavior is useful if the user should first validate what functions the model will use. + */ + public static FunctionChoiceBehavior none() { + return new NoneFunctionChoiceBehavior(null, null); + } + /** * Gets an instance of the FunctionChoiceBehavior that provides either all the Kernel's plugins functions to the AI model to call or specific functions. *

@@ -109,7 +148,7 @@ public static FunctionChoiceBehavior required(boolean autoInvoke, * @param functions Functions to provide to the model. If null, all the Kernel's plugins' functions are provided to the model. * If empty, no functions are provided to the model, which is equivalent to disabling function calling. */ - public static FunctionChoiceBehavior none(List> functions, + public static FunctionChoiceBehavior none(@Nullable List> functions, @Nullable FunctionChoiceBehaviorOptions options) { return new NoneFunctionChoiceBehavior(functions, options); } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/NoneFunctionChoiceBehavior.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/NoneFunctionChoiceBehavior.java index e5bef9f4..f0d247d3 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/NoneFunctionChoiceBehavior.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/NoneFunctionChoiceBehavior.java @@ -3,6 +3,7 @@ import com.microsoft.semantickernel.semanticfunctions.KernelFunction; +import javax.annotation.Nullable; import java.util.List; public class NoneFunctionChoiceBehavior extends FunctionChoiceBehavior { @@ -10,8 +11,8 @@ public class NoneFunctionChoiceBehavior extends FunctionChoiceBehavior { /** * Create a new instance of NoneFunctionChoiceBehavior. */ - public NoneFunctionChoiceBehavior(List> functions, - FunctionChoiceBehaviorOptions options) { + public NoneFunctionChoiceBehavior(@Nullable List> functions, + @Nullable FunctionChoiceBehaviorOptions options) { super(functions, options); } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/RequiredFunctionChoiceBehavior.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/RequiredFunctionChoiceBehavior.java index 01710cca..ac1ae2bf 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/RequiredFunctionChoiceBehavior.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/RequiredFunctionChoiceBehavior.java @@ -3,6 +3,7 @@ import com.microsoft.semantickernel.semanticfunctions.KernelFunction; +import javax.annotation.Nullable; import java.util.List; public class RequiredFunctionChoiceBehavior extends AutoFunctionChoiceBehavior { @@ -14,8 +15,9 @@ public class RequiredFunctionChoiceBehavior extends AutoFunctionChoiceBehavior { * @param functions A set of functions to advertise to the model. * @param options Options for the function choice behavior. */ - public RequiredFunctionChoiceBehavior(boolean autoInvoke, List> functions, - FunctionChoiceBehaviorOptions options) { + public RequiredFunctionChoiceBehavior(boolean autoInvoke, + @Nullable List> functions, + @Nullable FunctionChoiceBehaviorOptions options) { super(autoInvoke, functions, options); } } From 1bd792a0048733c3174e878ff13db0ced459b5be Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Mon, 5 May 2025 10:35:23 -0700 Subject: [PATCH 158/277] Update CHANGELOG.md --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0630f207..d0a3d203 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# 1.4.4-RC1 + +- Add Agent framework abstractions. +- Add ChatCompletionAgent implementation. +- Add FunctionChoiceBehavior for OpenAI, replacing the older ToolCallBehavior. + # 1.4.3 - Bug fix for execution on Android (https://github.com/microsoft/semantic-kernel-java/pull/284) From 4ea2ad0b9b72d55ae060106287399966a59bb5b3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 May 2025 06:55:12 +0000 Subject: [PATCH 159/277] Bump net.sourceforge.pmd:pmd-core from 6.55.0 to 7.10.0 Bumps [net.sourceforge.pmd:pmd-core](https://github.com/pmd/pmd) from 6.55.0 to 7.10.0. - [Release notes](https://github.com/pmd/pmd/releases) - [Changelog](https://github.com/pmd/pmd/blob/main/docs/render_release_notes.rb) - [Commits](https://github.com/pmd/pmd/compare/pmd_releases/6.55.0...pmd_releases/7.10.0) --- updated-dependencies: - dependency-name: net.sourceforge.pmd:pmd-core dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0d31715d..58fdd10e 100644 --- a/pom.xml +++ b/pom.xml @@ -60,7 +60,7 @@ 5.14.2 0.9.1 - 6.55.0 + 7.10.0 UTF-8 microsoft/semantic-kernel git@github.com:${project.github.repository}.git From 479aef99475d3acfbd4a5c75e305730068b75a76 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 6 May 2025 08:23:46 -0700 Subject: [PATCH 160/277] Set version to 1.4.4-RC-1-SNAPSHOT --- agents/semantickernel-agents-core/pom.xml | 2 +- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- data/semantickernel-data-azureaisearch/pom.xml | 2 +- data/semantickernel-data-jdbc/pom.xml | 2 +- data/semantickernel-data-redis/pom.xml | 2 +- pom.xml | 2 +- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-demos/semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- .../semantickernel-text-splitter-plugin/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-experimental/pom.xml | 2 +- 24 files changed, 24 insertions(+), 24 deletions(-) diff --git a/agents/semantickernel-agents-core/pom.xml b/agents/semantickernel-agents-core/pom.xml index 4eec1b76..520f3e39 100644 --- a/agents/semantickernel-agents-core/pom.xml +++ b/agents/semantickernel-agents-core/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../../pom.xml diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index 97c6957c..081adbd3 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index 68803583..c117eebf 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index 56ad5183..21b1ecf6 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 49624531..c929565e 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index e5dde576..fc05de7e 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../pom.xml diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml index 24cf4174..27f09fcd 100644 --- a/data/semantickernel-data-azureaisearch/pom.xml +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 624779e9..9d60e939 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-redis/pom.xml b/data/semantickernel-data-redis/pom.xml index 69e7c293..9eb7dc4c 100644 --- a/data/semantickernel-data-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../../pom.xml diff --git a/pom.xml b/pom.xml index 0d31715d..288195d6 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT pom https://www.github.com/microsoft/semantic-kernel diff --git a/samples/pom.xml b/samples/pom.xml index d42af3c9..0f6b2e57 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index d94740e8..3ee3d98f 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 3ccc114c..c0aa53bc 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index d7a6ec3b..17604eec 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index eb3c0afe..2c2631d5 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index bea07af9..3f21997d 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index 799d563b..fa4e6a08 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 293da2d3..a7024dbc 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index bb8db139..99e24e00 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index fa9f378d..afc4af3f 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index 8c012a6e..75722db8 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml index a10f1cfc..1e67c8e6 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index 3176c82e..bc0a2c42 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT ../pom.xml diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 1b3aa784..a6771b23 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT semantickernel-experimental From 54512fb5f1cd5c716d3b801a9f1e3ad6b680b27b Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 6 May 2025 08:39:29 -0700 Subject: [PATCH 161/277] Update bom pom --- semantickernel-bom/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 32ca102b..672af9b6 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.4.4-SNAPSHOT + 1.4.4-RC1-SNAPSHOT pom Semantic Kernel Java BOM From f1dee5f2e21e83aac182d93181e78f25b063c5f3 Mon Sep 17 00:00:00 2001 From: GitHub Date: Tue, 6 May 2025 08:49:43 -0700 Subject: [PATCH 162/277] [maven-release-plugin] prepare release java-1.4.4-RC1 --- agents/semantickernel-agents-core/pom.xml | 6 ++---- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- data/semantickernel-data-azureaisearch/pom.xml | 2 +- data/semantickernel-data-jdbc/pom.xml | 2 +- data/semantickernel-data-redis/pom.xml | 2 +- pom.xml | 4 ++-- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- .../semantickernel-text-splitter-plugin/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-bom/pom.xml | 4 ++-- semantickernel-experimental/pom.xml | 2 +- 25 files changed, 28 insertions(+), 30 deletions(-) diff --git a/agents/semantickernel-agents-core/pom.xml b/agents/semantickernel-agents-core/pom.xml index 520f3e39..162868ee 100644 --- a/agents/semantickernel-agents-core/pom.xml +++ b/agents/semantickernel-agents-core/pom.xml @@ -1,12 +1,10 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../../pom.xml diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index 081adbd3..22b492d7 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index c117eebf..6ae91947 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index 21b1ecf6..e826dcc1 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index c929565e..f6c51af7 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index fc05de7e..a6011290 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../pom.xml diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml index 27f09fcd..b2793603 100644 --- a/data/semantickernel-data-azureaisearch/pom.xml +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../../pom.xml diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 9d60e939..ab4dfe5e 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../../pom.xml diff --git a/data/semantickernel-data-redis/pom.xml b/data/semantickernel-data-redis/pom.xml index 9eb7dc4c..871fc931 100644 --- a/data/semantickernel-data-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../../pom.xml diff --git a/pom.xml b/pom.xml index 75d35d71..b8fef17f 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 pom https://www.github.com/microsoft/semantic-kernel @@ -838,6 +838,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - HEAD + java-1.4.4-RC1 diff --git a/samples/pom.xml b/samples/pom.xml index 0f6b2e57..aa9e4605 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index 3ee3d98f..4adace8f 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index c0aa53bc..700f42c3 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index 17604eec..fb1461f5 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index 2c2631d5..7637b820 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index 3f21997d..30d23e79 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index fa4e6a08..ff5520d7 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index a7024dbc..33839471 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index 99e24e00..6f8185b0 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index afc4af3f..085a462f 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index 75722db8..78b75aac 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml index 1e67c8e6..906b62db 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index bc0a2c42..06e1ebc0 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 672af9b6..2fbb7891 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 pom Semantic Kernel Java BOM @@ -296,6 +296,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - HEAD + java-1.4.4-RC1 diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index a6771b23..cb34ede6 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1-SNAPSHOT + 1.4.4-RC1 semantickernel-experimental From 75c980ab43099def327df3843456d7d84c41a58f Mon Sep 17 00:00:00 2001 From: GitHub Date: Tue, 6 May 2025 08:49:46 -0700 Subject: [PATCH 163/277] [maven-release-plugin] prepare for next development iteration --- agents/semantickernel-agents-core/pom.xml | 2 +- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- data/semantickernel-data-azureaisearch/pom.xml | 2 +- data/semantickernel-data-jdbc/pom.xml | 2 +- data/semantickernel-data-redis/pom.xml | 2 +- pom.xml | 4 ++-- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- .../semantickernel-text-splitter-plugin/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-bom/pom.xml | 4 ++-- semantickernel-experimental/pom.xml | 2 +- 25 files changed, 27 insertions(+), 27 deletions(-) diff --git a/agents/semantickernel-agents-core/pom.xml b/agents/semantickernel-agents-core/pom.xml index 162868ee..9d40d75d 100644 --- a/agents/semantickernel-agents-core/pom.xml +++ b/agents/semantickernel-agents-core/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../../pom.xml diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index 22b492d7..a205a60b 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index 6ae91947..aaed6aad 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index e826dcc1..4a93b679 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index f6c51af7..433e14a8 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index a6011290..c347e60a 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../pom.xml diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml index b2793603..74dc84bc 100644 --- a/data/semantickernel-data-azureaisearch/pom.xml +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index ab4dfe5e..5d0883eb 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-redis/pom.xml b/data/semantickernel-data-redis/pom.xml index 871fc931..a8e582eb 100644 --- a/data/semantickernel-data-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../../pom.xml diff --git a/pom.xml b/pom.xml index b8fef17f..8f8ce286 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT pom https://www.github.com/microsoft/semantic-kernel @@ -838,6 +838,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.4.4-RC1 + HEAD diff --git a/samples/pom.xml b/samples/pom.xml index aa9e4605..30246ab2 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index 4adace8f..1018e6c6 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 700f42c3..ffc2adae 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index fb1461f5..f6f8d2a8 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index 7637b820..3ad7a9c2 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index 30d23e79..0029654f 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index ff5520d7..e23396db 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 33839471..68062bf4 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index 6f8185b0..4aaaf412 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index 085a462f..0c88458e 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index 78b75aac..37c49ebc 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml index 906b62db..10fb9a30 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index 06e1ebc0..e10ed097 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 2fbb7891..ad6d7180 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT pom Semantic Kernel Java BOM @@ -296,6 +296,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.4.4-RC1 + HEAD diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index cb34ede6..58f99e03 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC1 + 1.4.4-RC2-SNAPSHOT semantickernel-experimental From 15c185ec9176c25ef453be70733275919e0af0b4 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 13 May 2025 00:37:22 -0700 Subject: [PATCH 164/277] Add FunctionCallContent --- .../chatcompletion/OpenAIChatCompletion.java | 38 +++--- .../OpenAIChatMessageContent.java | 29 ++++- .../OpenAIFunctionToolCall.java | 73 +---------- .../contents/FunctionCallContent.java | 121 ++++++++++++++++++ .../FunctionChoiceBehavior.java | 12 +- .../NoneFunctionChoiceBehavior.java | 2 +- .../RequiredFunctionChoiceBehavior.java | 4 +- .../services/KernelContentImpl.java | 7 + .../chatcompletion/ChatMessageContent.java | 32 ++++- 9 files changed, 216 insertions(+), 102 deletions(-) create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/contents/FunctionCallContent.java diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java index 1db5bd38..b94f96ed 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java @@ -38,6 +38,7 @@ import com.microsoft.semantickernel.aiservices.openai.OpenAiService; import com.microsoft.semantickernel.aiservices.openai.chatcompletion.responseformat.ChatCompletionsJsonSchemaResponseFormat; import com.microsoft.semantickernel.aiservices.openai.implementation.OpenAIRequestSettings; +import com.microsoft.semantickernel.contents.FunctionCallContent; import com.microsoft.semantickernel.contextvariables.ContextVariable; import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; import com.microsoft.semantickernel.exceptions.AIException; @@ -468,7 +469,6 @@ private Mono internalChatMessageContentsAsync( .doOnTerminate(span::close); }) .flatMap(completions -> { - List responseMessages = completions .getChoices() .stream() @@ -488,6 +488,7 @@ private Mono internalChatMessageContentsAsync( completions); return Mono.just(messages.addChatMessage(chatMessageContents)); } + // Or if there are no tool calls to be done ChatResponseMessage response = responseMessages.get(0); List toolCalls = response.getToolCalls(); @@ -633,8 +634,8 @@ private Mono> invokeFunctionTool( ContextVariableTypes contextVariableTypes) { try { - OpenAIFunctionToolCall openAIFunctionToolCall = extractOpenAIFunctionToolCall(toolCall); - String pluginName = openAIFunctionToolCall.getPluginName(); + FunctionCallContent FunctionCallContent = extractFunctionCallContent(toolCall); + String pluginName = FunctionCallContent.getPluginName(); if (pluginName == null || pluginName.isEmpty()) { return Mono.error( new SKException("Plugin name is required for function tool call")); @@ -642,12 +643,12 @@ private Mono> invokeFunctionTool( KernelFunction function = kernel.getFunction( pluginName, - openAIFunctionToolCall.getFunctionName()); + FunctionCallContent.getFunctionName()); PreToolCallEvent hookResult = executeHook(invocationContext, kernel, new PreToolCallEvent( - openAIFunctionToolCall.getFunctionName(), - openAIFunctionToolCall.getArguments(), + FunctionCallContent.getFunctionName(), + FunctionCallContent.getArguments(), function, contextVariableTypes)); @@ -686,7 +687,7 @@ private static T executeHook( } @SuppressWarnings("StringSplitter") - private OpenAIFunctionToolCall extractOpenAIFunctionToolCall( + private FunctionCallContent extractFunctionCallContent( ChatCompletionsFunctionToolCall toolCall) throws JsonProcessingException { @@ -712,10 +713,10 @@ private OpenAIFunctionToolCall extractOpenAIFunctionToolCall( } }); - return new OpenAIFunctionToolCall( - toolCall.getId(), - pluginName, + return new FunctionCallContent( fnName, + pluginName, + toolCall.getId(), arguments); } @@ -744,7 +745,7 @@ private List> getChatMessageContentsAsync( null, null, completionMetadata, - formOpenAiToolCalls(response)); + formFunctionCallContents(response)); } catch (SKCheckedException e) { LOGGER.warn("Failed to form chat message content", e); return null; @@ -784,7 +785,7 @@ private List> toOpenAIChatMessageContent( null); } else if (message instanceof ChatRequestAssistantMessage) { try { - List calls = getToolCalls( + List calls = getFunctionCallContents( ((ChatRequestAssistantMessage) message).getToolCalls()); return new OpenAIChatMessageContent<>( AuthorRole.ASSISTANT, @@ -823,7 +824,7 @@ private List> toOpenAIChatMessageContent( } @Nullable - private List getToolCalls( + private List getFunctionCallContents( @Nullable List toolCalls) throws SKCheckedException { if (toolCalls == null || toolCalls.isEmpty()) { return null; @@ -835,7 +836,7 @@ private List getToolCalls( .map(call -> { if (call instanceof ChatCompletionsFunctionToolCall) { try { - return extractOpenAIFunctionToolCall( + return extractFunctionCallContent( (ChatCompletionsFunctionToolCall) call); } catch (JsonProcessingException e) { throw SKException.build("Failed to parse tool arguments", e); @@ -852,7 +853,7 @@ private List getToolCalls( } @Nullable - private List formOpenAiToolCalls( + private List formFunctionCallContents( ChatResponseMessage response) throws SKCheckedException { if (response.getToolCalls() == null || response.getToolCalls().isEmpty()) { return null; @@ -864,7 +865,7 @@ private List formOpenAiToolCalls( .map(call -> { if (call instanceof ChatCompletionsFunctionToolCall) { try { - return extractOpenAIFunctionToolCall( + return extractFunctionCallContent( (ChatCompletionsFunctionToolCall) call); } catch (JsonProcessingException e) { throw SKException.build("Failed to parse tool arguments", e); @@ -1251,10 +1252,7 @@ private static ChatRequestAssistantMessage formAssistantMessage( // TODO: handle tools other than function calls ChatRequestAssistantMessage asstMessage = new ChatRequestAssistantMessage(content); - List toolCalls = null; - if (message instanceof OpenAIChatMessageContent) { - toolCalls = ((OpenAIChatMessageContent) message).getToolCall(); - } + List toolCalls = FunctionCallContent.getFunctionCalls(message); if (toolCalls != null) { asstMessage.setToolCalls( diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatMessageContent.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatMessageContent.java index 89f45014..ed1e2832 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatMessageContent.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatMessageContent.java @@ -1,12 +1,15 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.aiservices.openai.chatcompletion; +import com.microsoft.semantickernel.contents.FunctionCallContent; import com.microsoft.semantickernel.orchestration.FunctionResultMetadata; +import com.microsoft.semantickernel.services.KernelContent; import com.microsoft.semantickernel.services.chatcompletion.AuthorRole; import com.microsoft.semantickernel.services.chatcompletion.ChatMessageContent; import java.nio.charset.Charset; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import javax.annotation.Nullable; /** @@ -16,6 +19,7 @@ */ public class OpenAIChatMessageContent extends ChatMessageContent { + @Deprecated @Nullable private final List toolCall; @@ -28,7 +32,7 @@ public class OpenAIChatMessageContent extends ChatMessageContent { * @param innerContent The inner content. * @param encoding The encoding. * @param metadata The metadata. - * @param toolCall The tool call. + * @param functionCalls The tool call. */ public OpenAIChatMessageContent( AuthorRole authorRole, @@ -37,13 +41,25 @@ public OpenAIChatMessageContent( @Nullable T innerContent, @Nullable Charset encoding, @Nullable FunctionResultMetadata metadata, - @Nullable List toolCall) { - super(authorRole, content, modelId, innerContent, encoding, metadata); + @Nullable List functionCalls) { + super(authorRole, content, (List>) functionCalls, modelId, + innerContent, encoding, metadata); - if (toolCall == null) { + if (functionCalls == null) { this.toolCall = null; } else { - this.toolCall = Collections.unmodifiableList(toolCall); + // Keep OpenAIFunctionToolCall list for legacy + this.toolCall = Collections.unmodifiableList(functionCalls.stream().map(t -> { + if (t instanceof OpenAIFunctionToolCall) { + return (OpenAIFunctionToolCall) t; + } else { + return new OpenAIFunctionToolCall( + t.getId(), + t.getPluginName(), + t.getFunctionName(), + t.getArguments()); + } + }).collect(Collectors.toList())); } } @@ -51,7 +67,10 @@ public OpenAIChatMessageContent( * Gets any tool calls requested. * * @return The tool call. + * + * @deprecated Use {@link FunctionCallContent#getFunctionCalls(ChatMessageContent)} instead. */ + @Deprecated @Nullable public List getToolCall() { return toolCall; diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunctionToolCall.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunctionToolCall.java index b7999cf0..c1def337 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunctionToolCall.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunctionToolCall.java @@ -1,29 +1,17 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.aiservices.openai.chatcompletion; +import com.microsoft.semantickernel.contents.FunctionCallContent; import com.microsoft.semantickernel.semanticfunctions.KernelArguments; import javax.annotation.Nullable; /** * Represents a call to a function in the OpenAI tool. + * + * @deprecated Use {@link FunctionCallContent} instead. */ -public class OpenAIFunctionToolCall { - - ///

Gets the ID of the tool call. - @Nullable - private final String id; - - /// Gets the name of the plugin with which this function is associated, if any. - - @Nullable - private final String pluginName; - - /// Gets the name of the function. - private final String functionName; - - /// Gets a name/value collection of the arguments to the function, if any. - @Nullable - private final KernelArguments arguments; +@Deprecated +public class OpenAIFunctionToolCall extends FunctionCallContent { /** * Creates a new instance of the {@link OpenAIFunctionToolCall} class. @@ -38,55 +26,6 @@ public OpenAIFunctionToolCall( @Nullable String pluginName, String functionName, @Nullable KernelArguments arguments) { - this.id = id; - this.pluginName = pluginName; - this.functionName = functionName; - if (arguments == null) { - this.arguments = null; - } else { - this.arguments = arguments.copy(); - } - } - - /** - * Gets the ID of the tool call. - * - * @return The ID of the tool call. - */ - @Nullable - public String getId() { - return id; - } - - /** - * Gets the name of the plugin with which this function is associated, if any. - * - * @return The name of the plugin with which this function is associated, if any. - */ - @Nullable - public String getPluginName() { - return pluginName; - } - - /** - * Gets the name of the function. - * - * @return The name of the function. - */ - public String getFunctionName() { - return functionName; - } - - /** - * Gets a name/value collection of the arguments to the function, if any. - * - * @return A name/value collection of the arguments to the function, if any. - */ - @Nullable - public KernelArguments getArguments() { - if (arguments == null) { - return null; - } - return arguments.copy(); + super(functionName, pluginName, id, arguments); } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contents/FunctionCallContent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contents/FunctionCallContent.java new file mode 100644 index 00000000..8c973ab0 --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contents/FunctionCallContent.java @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.contents; + +import com.microsoft.semantickernel.orchestration.FunctionResultMetadata; +import com.microsoft.semantickernel.semanticfunctions.KernelArguments; +import com.microsoft.semantickernel.services.KernelContentImpl; +import com.microsoft.semantickernel.services.chatcompletion.ChatMessageContent; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Represents the content of a function call. + *

+ * This class is used to represent a function call in the context of a chat message. + */ +public class FunctionCallContent extends KernelContentImpl { + + @Nullable + private final String id; + @Nullable + private final String pluginName; + private final String functionName; + @Nullable + private final KernelArguments arguments; + + /** + * Creates a new instance of the {@link FunctionCallContent} class. + * + * @param functionName The name of the function. + * @param pluginName The name of the plugin with which this function is associated, if any. + * @param id The ID of the tool call. + * @param arguments A name/value collection of the arguments to the function, if any. + */ + public FunctionCallContent( + String functionName, + @Nullable String pluginName, + @Nullable String id, + @Nullable KernelArguments arguments) { + this.functionName = functionName; + this.pluginName = pluginName; + this.id = id; + if (arguments == null) { + this.arguments = null; + } else { + this.arguments = arguments.copy(); + } + } + + /** + * Gets the ID of the tool call. + * + * @return The ID of the tool call. + */ + @Nullable + public String getId() { + return id; + } + + /** + * Gets the name of the plugin with which this function is associated, if any. + * + * @return The name of the plugin with which this function is associated, if any. + */ + @Nullable + public String getPluginName() { + return pluginName; + } + + /** + * Gets the name of the function. + * + * @return The name of the function. + */ + public String getFunctionName() { + return functionName; + } + + /** + * Gets a name/value collection of the arguments to the function, if any. + * + * @return A name/value collection of the arguments to the function, if any. + */ + @Nullable + public KernelArguments getArguments() { + if (arguments == null) { + return null; + } + return arguments.copy(); + } + + /** + * Gets list of function calls from the message content. + * + * @param messageContent The message content. + * @return The function calls. + */ + public static List getFunctionCalls(ChatMessageContent messageContent) { + if (messageContent.getItems() == null) { + return null; + } + + return messageContent.getItems().stream().filter( + item -> item instanceof FunctionCallContent) + .map(item -> (FunctionCallContent) item) + .collect(Collectors.toList()); + } + + /** + * Gets the content returned by the AI service. + * + * @return The content. + */ + @Nullable + @Override + public String getContent() { + return null; + } +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java index f0024952..d74a77a7 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/FunctionChoiceBehavior.java @@ -78,7 +78,7 @@ public static FunctionChoiceBehavior auto(boolean autoInvoke) { * @return A new FunctionChoiceBehavior instance with all kernel functions allowed. */ public static FunctionChoiceBehavior auto(boolean autoInvoke, - @Nullable List> functions) { + @Nullable List> functions) { return new AutoFunctionChoiceBehavior(autoInvoke, functions, null); } @@ -95,8 +95,8 @@ public static FunctionChoiceBehavior auto(boolean autoInvoke, * @return A new FunctionChoiceBehavior instance with all kernel functions allowed. */ public static FunctionChoiceBehavior auto(boolean autoInvoke, - @Nullable List> functions, - @Nullable FunctionChoiceBehaviorOptions options) { + @Nullable List> functions, + @Nullable FunctionChoiceBehaviorOptions options) { return new AutoFunctionChoiceBehavior(autoInvoke, functions, options); } @@ -110,7 +110,7 @@ public static FunctionChoiceBehavior auto(boolean autoInvoke, * @return A new FunctionChoiceBehavior instance with the required function. */ public static FunctionChoiceBehavior required(boolean autoInvoke, - @Nullable List> functions) { + @Nullable List> functions) { return new RequiredFunctionChoiceBehavior(autoInvoke, functions, null); } @@ -126,8 +126,8 @@ public static FunctionChoiceBehavior required(boolean autoInvoke, * @return A new FunctionChoiceBehavior instance with the required function. */ public static FunctionChoiceBehavior required(boolean autoInvoke, - @Nullable List> functions, - @Nullable FunctionChoiceBehaviorOptions options) { + @Nullable List> functions, + @Nullable FunctionChoiceBehaviorOptions options) { return new RequiredFunctionChoiceBehavior(autoInvoke, functions, options); } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/NoneFunctionChoiceBehavior.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/NoneFunctionChoiceBehavior.java index f0d247d3..1842ba5f 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/NoneFunctionChoiceBehavior.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/NoneFunctionChoiceBehavior.java @@ -12,7 +12,7 @@ public class NoneFunctionChoiceBehavior extends FunctionChoiceBehavior { * Create a new instance of NoneFunctionChoiceBehavior. */ public NoneFunctionChoiceBehavior(@Nullable List> functions, - @Nullable FunctionChoiceBehaviorOptions options) { + @Nullable FunctionChoiceBehaviorOptions options) { super(functions, options); } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/RequiredFunctionChoiceBehavior.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/RequiredFunctionChoiceBehavior.java index ac1ae2bf..8bfee535 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/RequiredFunctionChoiceBehavior.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/functionchoice/RequiredFunctionChoiceBehavior.java @@ -16,8 +16,8 @@ public class RequiredFunctionChoiceBehavior extends AutoFunctionChoiceBehavior { * @param options Options for the function choice behavior. */ public RequiredFunctionChoiceBehavior(boolean autoInvoke, - @Nullable List> functions, - @Nullable FunctionChoiceBehaviorOptions options) { + @Nullable List> functions, + @Nullable FunctionChoiceBehaviorOptions options) { super(autoInvoke, functions, options); } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/KernelContentImpl.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/KernelContentImpl.java index 48bc6d77..e83e909d 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/KernelContentImpl.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/KernelContentImpl.java @@ -47,6 +47,13 @@ public KernelContentImpl( this.metadata = metadata; } + /** + * Initializes a new instance of the {@link KernelContentImpl} class. + */ + public KernelContentImpl() { + this(null, null, null); + } + /** * Gets the inner content representation. * diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java index e06472d1..2d4ce491 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatMessageContent.java @@ -135,7 +135,7 @@ public ChatMessageContent( */ public ChatMessageContent( AuthorRole authorRole, - @Nullable List> items, + @Nullable List> items, String modelId, T innerContent, Charset encoding, @@ -153,6 +153,36 @@ public ChatMessageContent( this.contentType = contentType; } + /** + * Creates a new instance of the {@link ChatMessageContent} class. + * + * @param authorRole the author role that generated the content + * @param items the items + * @param modelId the model id + * @param innerContent the inner content + * @param encoding the encoding + * @param metadata the metadata + */ + public ChatMessageContent( + AuthorRole authorRole, + @Nullable String content, + @Nullable List> items, + String modelId, + T innerContent, + Charset encoding, + FunctionResultMetadata metadata) { + super(innerContent, modelId, metadata); + this.content = content; + this.authorRole = authorRole; + this.encoding = encoding != null ? encoding : StandardCharsets.UTF_8; + if (items == null) { + this.items = null; + } else { + this.items = new ArrayList<>(items); + } + this.contentType = ChatMessageContentType.TEXT; + } + /** * Gets the author role that generated the content * From 43cb50381ee9884ddb0100702e8a425ea29efd36 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Tue, 13 May 2025 08:06:19 -0700 Subject: [PATCH 165/277] Fix variable name --- .../openai/chatcompletion/OpenAIChatCompletion.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java index b94f96ed..8256bb00 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java @@ -634,8 +634,8 @@ private Mono> invokeFunctionTool( ContextVariableTypes contextVariableTypes) { try { - FunctionCallContent FunctionCallContent = extractFunctionCallContent(toolCall); - String pluginName = FunctionCallContent.getPluginName(); + FunctionCallContent functionCallContent = extractFunctionCallContent(toolCall); + String pluginName = functionCallContent.getPluginName(); if (pluginName == null || pluginName.isEmpty()) { return Mono.error( new SKException("Plugin name is required for function tool call")); @@ -643,12 +643,12 @@ private Mono> invokeFunctionTool( KernelFunction function = kernel.getFunction( pluginName, - FunctionCallContent.getFunctionName()); + functionCallContent.getFunctionName()); PreToolCallEvent hookResult = executeHook(invocationContext, kernel, new PreToolCallEvent( - FunctionCallContent.getFunctionName(), - FunctionCallContent.getArguments(), + functionCallContent.getFunctionName(), + functionCallContent.getArguments(), function, contextVariableTypes)); From 7d476fcaf0fe0505de5534e4f48ef667303bb061 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Thu, 22 May 2025 12:09:40 +0200 Subject: [PATCH 166/277] Split out the data into its own module --- data/semantickernel-data-hsqldb/pom.xml | 64 +++++++++++++++++++ .../HSQLDBVectorStoreQueryProvider.java | 0 data/semantickernel-data-jdbc/pom.xml | 2 +- .../jdbc/JDBCVectorStoreQueryProvider.java | 13 +++- .../jdbc/JDBCVectorStoreRecordCollection.java | 25 +------- .../jdbc/SQLVectorStoreQueryProvider.java | 12 ++++ data/semantickernel-data-mysql/pom.xml | 64 +++++++++++++++++++ .../mysql/MySQLVectorStoreQueryProvider.java | 0 data/semantickernel-data-postgres/pom.xml | 64 +++++++++++++++++++ .../PostgreSQLVectorDistanceFunction.java | 0 .../postgres/PostgreSQLVectorIndexKind.java | 0 .../PostgreSQLVectorStoreQueryProvider.java | 11 +++- .../PostgreSQLVectorStoreRecordMapper.java | 0 data/semantickernel-data-sqlite/pom.xml | 63 ++++++++++++++++++ .../SQLiteVectorStoreQueryProvider.java | 0 pom.xml | 34 ++++++++++ semantickernel-api-builders/pom.xml | 30 +++++++++ .../builders/SemanticKernelBuilder.java | 0 semantickernel-api-data/pom.xml | 60 +++++++++++++++++ .../data/VectorStoreTextSearch.java | 0 .../data/VectorStoreTextSearchOptions.java | 0 .../data/VolatileVectorStore.java | 0 ...ileVectorStoreCollectionSearchMapping.java | 0 .../VolatileVectorStoreRecordCollection.java | 0 ...ileVectorStoreRecordCollectionOptions.java | 0 .../filter/AnyTagEqualToFilterClause.java | 0 .../data/filter/EqualToFilterClause.java | 0 .../data/filter/FilterClause.java | 0 .../data/filter/FilterMapping.java | 0 .../DefaultTextSearchResultMapper.java | 0 .../DefaultTextSearchStringMapper.java | 0 .../data/textsearch/KernelSearchResults.java | 0 .../data/textsearch/TextSearch.java | 0 .../data/textsearch/TextSearchFilter.java | 0 .../data/textsearch/TextSearchOptions.java | 0 .../data/textsearch/TextSearchResult.java | 0 .../data/textsearch/TextSearchResultLink.java | 0 .../textsearch/TextSearchResultMapper.java | 0 .../data/textsearch/TextSearchResultName.java | 0 .../textsearch/TextSearchResultValue.java | 0 .../textsearch/TextSearchStringMapper.java | 0 .../data/vectorsearch/VectorOperations.java | 0 .../data/vectorsearch/VectorSearchFilter.java | 0 .../data/vectorsearch/VectorSearchResult.java | 0 .../vectorsearch/VectorSearchResults.java | 0 .../vectorsearch/VectorizableTextSearch.java | 0 .../data/vectorsearch/VectorizedSearch.java | 0 .../data/vectorstorage/VectorStore.java | 0 .../VectorStoreRecordCollection.java | 0 .../VectorStoreRecordCollectionOptions.java | 0 .../VectorStoreRecordMapper.java | 0 .../annotations/VectorStoreRecordData.java | 0 .../annotations/VectorStoreRecordKey.java | 0 .../annotations/VectorStoreRecordVector.java | 0 .../definition/DistanceFunction.java | 0 .../vectorstorage/definition/IndexKind.java | 0 .../VectorStoreRecordDataField.java | 0 .../VectorStoreRecordDefinition.java | 0 .../definition/VectorStoreRecordField.java | 0 .../definition/VectorStoreRecordKeyField.java | 0 .../VectorStoreRecordVectorField.java | 0 .../options/DeleteRecordOptions.java | 0 .../options/GetRecordOptions.java | 0 .../options/UpsertRecordOptions.java | 0 .../options/VectorSearchOptions.java | 0 semantickernel-api-exceptions/pom.xml | 43 +++++++++++++ .../exceptions/AIException.java | 0 .../exceptions/ConfigurationException.java | 0 .../exceptions/SKCheckedException.java | 0 .../exceptions/SKException.java | 0 semantickernel-api-localization/pom.xml | 31 +++++++++ .../localization/SemanticKernelResources.java | 0 .../localization/ResourceBundle.properties | 0 .../pom.xml | 49 ++++++++++++++ .../semantickernel/services/AIService.java | 0 .../services/textembedding/Embedding.java | 0 .../EmbeddingGenerationService.java | 0 .../TextEmbeddingGenerationService.java | 0 semantickernel-api/pom.xml | 20 ++++++ 79 files changed, 560 insertions(+), 25 deletions(-) create mode 100644 data/semantickernel-data-hsqldb/pom.xml rename data/{semantickernel-data-jdbc => semantickernel-data-hsqldb}/src/main/java/com/microsoft/semantickernel/data/jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java (100%) create mode 100644 data/semantickernel-data-mysql/pom.xml rename data/{semantickernel-data-jdbc => semantickernel-data-mysql}/src/main/java/com/microsoft/semantickernel/data/jdbc/mysql/MySQLVectorStoreQueryProvider.java (100%) create mode 100644 data/semantickernel-data-postgres/pom.xml rename data/{semantickernel-data-jdbc => semantickernel-data-postgres}/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorDistanceFunction.java (100%) rename data/{semantickernel-data-jdbc => semantickernel-data-postgres}/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorIndexKind.java (100%) rename data/{semantickernel-data-jdbc => semantickernel-data-postgres}/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java (97%) rename data/{semantickernel-data-jdbc => semantickernel-data-postgres}/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreRecordMapper.java (100%) create mode 100644 data/semantickernel-data-sqlite/pom.xml rename data/{semantickernel-data-jdbc => semantickernel-data-sqlite}/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java (100%) create mode 100644 semantickernel-api-builders/pom.xml rename {semantickernel-api => semantickernel-api-builders}/src/main/java/com/microsoft/semantickernel/builders/SemanticKernelBuilder.java (100%) create mode 100644 semantickernel-api-data/pom.xml rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearch.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearchOptions.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java (100%) rename {semantickernel-api => semantickernel-api-data}/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java (100%) create mode 100644 semantickernel-api-exceptions/pom.xml rename {semantickernel-api => semantickernel-api-exceptions}/src/main/java/com/microsoft/semantickernel/exceptions/AIException.java (100%) rename {semantickernel-api => semantickernel-api-exceptions}/src/main/java/com/microsoft/semantickernel/exceptions/ConfigurationException.java (100%) rename {semantickernel-api => semantickernel-api-exceptions}/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java (100%) rename {semantickernel-api => semantickernel-api-exceptions}/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java (100%) create mode 100644 semantickernel-api-localization/pom.xml rename {semantickernel-api => semantickernel-api-localization}/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java (100%) rename {semantickernel-api => semantickernel-api-localization}/src/main/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties (100%) create mode 100644 semantickernel-api-textembedding-services/pom.xml rename {semantickernel-api => semantickernel-api-textembedding-services}/src/main/java/com/microsoft/semantickernel/services/AIService.java (100%) rename {semantickernel-api => semantickernel-api-textembedding-services}/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java (100%) rename {semantickernel-api => semantickernel-api-textembedding-services}/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java (100%) rename {semantickernel-api => semantickernel-api-textembedding-services}/src/main/java/com/microsoft/semantickernel/services/textembedding/TextEmbeddingGenerationService.java (100%) diff --git a/data/semantickernel-data-hsqldb/pom.xml b/data/semantickernel-data-hsqldb/pom.xml new file mode 100644 index 00000000..72421ae0 --- /dev/null +++ b/data/semantickernel-data-hsqldb/pom.xml @@ -0,0 +1,64 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-data-hsqldb + Semantic Kernel HLSQLDB connector + Provides a HLSQLDB connector for the Semantic Kernel + + + + com.microsoft.semantic-kernel + semantickernel-api + + + com.microsoft.semantic-kernel + semantickernel-data-jdbc + + + + org.slf4j + slf4j-api + + + com.fasterxml.jackson.core + jackson-databind + compile + + + com.fasterxml.jackson.core + jackson-core + compile + + + com.github.jknack + handlebars + + + com.google.code.findbugs + jsr305 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + compile + + + com.github.spotbugs + spotbugs-annotations + + + org.apache.commons + commons-text + + + \ No newline at end of file diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java b/data/semantickernel-data-hsqldb/src/main/java/com/microsoft/semantickernel/data/jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java similarity index 100% rename from data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java rename to data/semantickernel-data-hsqldb/src/main/java/com/microsoft/semantickernel/data/jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 5d0883eb..2e9cd67e 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -15,7 +15,7 @@ com.microsoft.semantic-kernel - semantickernel-api + semantickernel-api-data diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java index ac21c61f..afc8d82f 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -696,10 +696,19 @@ public String getEqualToFilter(EqualToFilterClause filterClause) { @Override public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { String fieldName = JDBCVectorStoreQueryProvider - .validateSQLidentifier(filterClause.getFieldName()); + .validateSQLidentifier(filterClause.getFieldName()); return String.format("%s LIKE ?", fieldName); } + + @Override + public VectorStoreRecordMapper getVectorStoreRecordMapper(Class recordClass, + VectorStoreRecordDefinition recordDefinition) { + return JDBCVectorStoreRecordMapper.builder() + .withRecordClass(recordClass) + .withVectorStoreRecordDefinition(recordDefinition) + .build(); + } /** * The builder for {@link JDBCVectorStoreQueryProvider}. @@ -755,4 +764,6 @@ public JDBCVectorStoreQueryProvider build() { prefixForCollectionTables); } } + + } diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java index 1c1cea5d..2d75af14 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -2,9 +2,6 @@ package com.microsoft.semantickernel.data.jdbc; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; -import com.microsoft.semantickernel.data.jdbc.mysql.MySQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; @@ -73,25 +70,9 @@ public JDBCVectorStoreRecordCollection( // If mapper is not provided, set a default one if (options.getVectorStoreRecordMapper() == null) { - // Default mapper for PostgreSQL - if (this.queryProvider instanceof PostgreSQLVectorStoreQueryProvider) { - vectorStoreRecordMapper = PostgreSQLVectorStoreRecordMapper.builder() - .withRecordClass(options.getRecordClass()) - .withVectorStoreRecordDefinition(recordDefinition) - .build(); - // Default mapper for MySQL - } else if (this.queryProvider instanceof MySQLVectorStoreQueryProvider) { - vectorStoreRecordMapper = JDBCVectorStoreRecordMapper.builder() - .withRecordClass(options.getRecordClass()) - .withVectorStoreRecordDefinition(recordDefinition) - .build(); - // Default mapper for other databases - } else { - vectorStoreRecordMapper = JDBCVectorStoreRecordMapper.builder() - .withRecordClass(options.getRecordClass()) - .withVectorStoreRecordDefinition(recordDefinition) - .build(); - } + vectorStoreRecordMapper = options.getQueryProvider() + .getVectorStoreRecordMapper(options.getRecordClass(), + recordDefinition); } else { vectorStoreRecordMapper = options.getVectorStoreRecordMapper(); } diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java index 18806c5d..cadfdaf2 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java @@ -151,6 +151,18 @@ VectorSearchResults search(String collectionName, VectorStoreRecordDefinition recordDefinition, VectorStoreRecordMapper mapper); + /** + * Gets the record mapper for the given record class and definition. + * + * @param the record type + * @param recordClass the record class + * @param recordDefinition the record definition + * @return the record mapper that maps JDBC result sets to the given record. + */ + VectorStoreRecordMapper getVectorStoreRecordMapper( + final Class recordClass, + final VectorStoreRecordDefinition recordDefinition); + /** * The builder for the JDBC vector store query provider. */ diff --git a/data/semantickernel-data-mysql/pom.xml b/data/semantickernel-data-mysql/pom.xml new file mode 100644 index 00000000..d2d12f13 --- /dev/null +++ b/data/semantickernel-data-mysql/pom.xml @@ -0,0 +1,64 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-data-mysql + Semantic Kernel MySQL connector + Provides a MySQL connector for the Semantic Kernel + + + + com.microsoft.semantic-kernel + semantickernel-api + + + com.microsoft.semantic-kernel + semantickernel-data-jdbc + + + + org.slf4j + slf4j-api + + + com.fasterxml.jackson.core + jackson-databind + compile + + + com.fasterxml.jackson.core + jackson-core + compile + + + com.github.jknack + handlebars + + + com.google.code.findbugs + jsr305 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + compile + + + com.github.spotbugs + spotbugs-annotations + + + org.apache.commons + commons-text + + + \ No newline at end of file diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/mysql/MySQLVectorStoreQueryProvider.java b/data/semantickernel-data-mysql/src/main/java/com/microsoft/semantickernel/data/jdbc/mysql/MySQLVectorStoreQueryProvider.java similarity index 100% rename from data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/mysql/MySQLVectorStoreQueryProvider.java rename to data/semantickernel-data-mysql/src/main/java/com/microsoft/semantickernel/data/jdbc/mysql/MySQLVectorStoreQueryProvider.java diff --git a/data/semantickernel-data-postgres/pom.xml b/data/semantickernel-data-postgres/pom.xml new file mode 100644 index 00000000..e29e758e --- /dev/null +++ b/data/semantickernel-data-postgres/pom.xml @@ -0,0 +1,64 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-data-postgres + Semantic Kernel PostreSQL connector + Provides a PostreSQL connector for the Semantic Kernel + + + + com.microsoft.semantic-kernel + semantickernel-api + + + com.microsoft.semantic-kernel + semantickernel-data-jdbc + + + + org.slf4j + slf4j-api + + + com.fasterxml.jackson.core + jackson-databind + compile + + + com.fasterxml.jackson.core + jackson-core + compile + + + com.github.jknack + handlebars + + + com.google.code.findbugs + jsr305 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + compile + + + com.github.spotbugs + spotbugs-annotations + + + org.apache.commons + commons-text + + + \ No newline at end of file diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorDistanceFunction.java b/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorDistanceFunction.java similarity index 100% rename from data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorDistanceFunction.java rename to data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorDistanceFunction.java diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorIndexKind.java b/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorIndexKind.java similarity index 100% rename from data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorIndexKind.java rename to data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorIndexKind.java diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java b/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java similarity index 97% rename from data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java rename to data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java index 4734f484..bd8dbba7 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -442,10 +442,19 @@ public List getFilterParameters(VectorSearchFilter filter) { @Override public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { String fieldName = JDBCVectorStoreQueryProvider - .validateSQLidentifier(filterClause.getFieldName()); + .validateSQLidentifier(filterClause.getFieldName()); return String.format("%s @> ?::jsonb", fieldName); } + + @Override + public VectorStoreRecordMapper getVectorStoreRecordMapper(Class recordClass, + VectorStoreRecordDefinition recordDefinition) { + return PostgreSQLVectorStoreRecordMapper.builder() + .withRecordClass(recordClass) + .withVectorStoreRecordDefinition(recordDefinition) + .build(); + } /** * A builder for the PostgreSQLVectorStoreQueryProvider class. diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreRecordMapper.java b/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreRecordMapper.java similarity index 100% rename from data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreRecordMapper.java rename to data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreRecordMapper.java diff --git a/data/semantickernel-data-sqlite/pom.xml b/data/semantickernel-data-sqlite/pom.xml new file mode 100644 index 00000000..69e3bc6f --- /dev/null +++ b/data/semantickernel-data-sqlite/pom.xml @@ -0,0 +1,63 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + + + com.microsoft.semantic-kernel + semantickernel-data-sqlite + Semantic Kernel SQLite JDBC driver connector + Provides a SQLite connector for the Semantic Kernel + + + + com.microsoft.semantic-kernel + semantickernel-api + + + com.microsoft.semantic-kernel + semantickernel-data-jdbc + + + + org.slf4j + slf4j-api + + + com.fasterxml.jackson.core + jackson-databind + compile + + + com.fasterxml.jackson.core + jackson-core + compile + + + com.github.jknack + handlebars + + + com.google.code.findbugs + jsr305 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + compile + + + com.github.spotbugs + spotbugs-annotations + + + org.apache.commons + commons-text + + + \ No newline at end of file diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java b/data/semantickernel-data-sqlite/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java similarity index 100% rename from data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java rename to data/semantickernel-data-sqlite/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java diff --git a/pom.xml b/pom.xml index 8f8ce286..354e87de 100644 --- a/pom.xml +++ b/pom.xml @@ -77,7 +77,16 @@ data/semantickernel-data-azureaisearch data/semantickernel-data-jdbc data/semantickernel-data-redis + data/semantickernel-data-mysql + data/semantickernel-data-hsqldb + data/semantickernel-data-postgres + data/semantickernel-data-sqlite agents/semantickernel-agents-core + semantickernel-api-data + semantickernel-api-exceptions + semantickernel-api-builders + semantickernel-api-textembedding-services + semantickernel-api-localization @@ -135,6 +144,31 @@ semantickernel-connectors-ai-openai ${project.version} + + com.microsoft.semantic-kernel + semantickernel-api-builders + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-api-data + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-api-localization + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-api-textembedding-services + ${project.version} + com.microsoft.semantic-kernel.extensions semantickernel-sequentialplanner-extension diff --git a/semantickernel-api-builders/pom.xml b/semantickernel-api-builders/pom.xml new file mode 100644 index 00000000..f48a2297 --- /dev/null +++ b/semantickernel-api-builders/pom.xml @@ -0,0 +1,30 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + + + com.microsoft.semantic-kernel + semantickernel-api-builders + Semantic Kernel Builders API + Defines the public interface for the Semantic Kernel Builders + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + 1 + + + + + + \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/builders/SemanticKernelBuilder.java b/semantickernel-api-builders/src/main/java/com/microsoft/semantickernel/builders/SemanticKernelBuilder.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/builders/SemanticKernelBuilder.java rename to semantickernel-api-builders/src/main/java/com/microsoft/semantickernel/builders/SemanticKernelBuilder.java diff --git a/semantickernel-api-data/pom.xml b/semantickernel-api-data/pom.xml new file mode 100644 index 00000000..895e70e1 --- /dev/null +++ b/semantickernel-api-data/pom.xml @@ -0,0 +1,60 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-api-data + Semantic Kernel Data API + Defines the public interface for the Semantic Kernel Data + + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + com.microsoft.semantic-kernel + semantickernel-api-textembedding-services + + + com.fasterxml.jackson.core + jackson-databind + compile + + + com.fasterxml.jackson.core + jackson-core + compile + + + io.projectreactor + reactor-core + 3.4.38 + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + 1 + + + + + \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearch.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearch.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearch.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearch.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearchOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearchOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearchOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VectorStoreTextSearchOptions.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreCollectionSearchMapping.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionOptions.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/AnyTagEqualToFilterClause.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/EqualToFilterClause.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/FilterClause.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/filter/FilterMapping.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchResultMapper.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/DefaultTextSearchStringMapper.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/KernelSearchResults.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearch.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchFilter.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchOptions.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResult.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultLink.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultMapper.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultName.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchResultValue.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/textsearch/TextSearchStringMapper.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorOperations.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchFilter.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResult.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorSearchResults.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizableTextSearch.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorsearch/VectorizedSearch.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStore.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollection.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordCollectionOptions.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/VectorStoreRecordMapper.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordData.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordKey.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/annotations/VectorStoreRecordVector.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/DistanceFunction.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/IndexKind.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordKeyField.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/DeleteRecordOptions.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/GetRecordOptions.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/UpsertRecordOptions.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java rename to semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/options/VectorSearchOptions.java diff --git a/semantickernel-api-exceptions/pom.xml b/semantickernel-api-exceptions/pom.xml new file mode 100644 index 00000000..71d1794b --- /dev/null +++ b/semantickernel-api-exceptions/pom.xml @@ -0,0 +1,43 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + Semantic Kernel Exceptions API + Defines the public interface for the Semantic Kernel Exceptions + + + + com.microsoft.semantic-kernel + semantickernel-api-localization + + + com.google.code.findbugs + jsr305 + provided + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + 1 + + + + + + \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/AIException.java b/semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/AIException.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/AIException.java rename to semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/AIException.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/ConfigurationException.java b/semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/ConfigurationException.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/ConfigurationException.java rename to semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/ConfigurationException.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java b/semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java rename to semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/SKCheckedException.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java b/semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java rename to semantickernel-api-exceptions/src/main/java/com/microsoft/semantickernel/exceptions/SKException.java diff --git a/semantickernel-api-localization/pom.xml b/semantickernel-api-localization/pom.xml new file mode 100644 index 00000000..84be7960 --- /dev/null +++ b/semantickernel-api-localization/pom.xml @@ -0,0 +1,31 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-api-localization + Semantic Kernel Localization API + Defines the public interface for the Semantic Kernel Localization + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + 1 + + + + + + \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java b/semantickernel-api-localization/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java rename to semantickernel-api-localization/src/main/java/com/microsoft/semantickernel/localization/SemanticKernelResources.java diff --git a/semantickernel-api/src/main/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties b/semantickernel-api-localization/src/main/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties similarity index 100% rename from semantickernel-api/src/main/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties rename to semantickernel-api-localization/src/main/resources/com/microsoft/semantickernel/localization/ResourceBundle.properties diff --git a/semantickernel-api-textembedding-services/pom.xml b/semantickernel-api-textembedding-services/pom.xml new file mode 100644 index 00000000..dbc5fa6a --- /dev/null +++ b/semantickernel-api-textembedding-services/pom.xml @@ -0,0 +1,49 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-api-textembedding-services + Semantic Kernel Services API + Defines the public interface for the Semantic Kernel Services + + + + io.projectreactor + reactor-core + 3.4.38 + + + com.google.code.findbugs + jsr305 + provided + + + com.github.spotbugs + spotbugs-annotations + ${spotbugs.version} + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + 1 + + + + + + \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIService.java b/semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/AIService.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/services/AIService.java rename to semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/AIService.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java b/semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java rename to semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/textembedding/Embedding.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java b/semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java rename to semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/textembedding/EmbeddingGenerationService.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/TextEmbeddingGenerationService.java b/semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/textembedding/TextEmbeddingGenerationService.java similarity index 100% rename from semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textembedding/TextEmbeddingGenerationService.java rename to semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/textembedding/TextEmbeddingGenerationService.java diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index e10ed097..d34e5a17 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -15,6 +15,26 @@ Semantic Kernel API Defines the public interface for the Semantic Kernel + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + com.microsoft.semantic-kernel + semantickernel-api-localization + + + com.microsoft.semantic-kernel + semantickernel-api-textembedding-services + io.opentelemetry.instrumentation opentelemetry-reactor-3.1 From bf0c167d80296f225aade77a87e89cbff568cfab Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Thu, 22 May 2025 13:18:38 +0200 Subject: [PATCH 167/277] Removed dependencies --- data/semantickernel-data-hsqldb/pom.xml | 22 -------------- data/semantickernel-data-jdbc/pom.xml | 36 +++-------------------- data/semantickernel-data-mysql/pom.xml | 22 -------------- data/semantickernel-data-postgres/pom.xml | 23 ++------------- data/semantickernel-data-sqlite/pom.xml | 23 ++------------- 5 files changed, 10 insertions(+), 116 deletions(-) diff --git a/data/semantickernel-data-hsqldb/pom.xml b/data/semantickernel-data-hsqldb/pom.xml index 72421ae0..f755f0f1 100644 --- a/data/semantickernel-data-hsqldb/pom.xml +++ b/data/semantickernel-data-hsqldb/pom.xml @@ -24,11 +24,6 @@ com.microsoft.semantic-kernel semantickernel-data-jdbc - - - org.slf4j - slf4j-api - com.fasterxml.jackson.core jackson-databind @@ -39,26 +34,9 @@ jackson-core compile - - com.github.jknack - handlebars - - - com.google.code.findbugs - jsr305 - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - compile - com.github.spotbugs spotbugs-annotations - - org.apache.commons - commons-text - \ No newline at end of file diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 2e9cd67e..7cd77f3f 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -13,15 +13,14 @@ Provides a JDBC connector for the Semantic Kernel - - com.microsoft.semantic-kernel - semantickernel-api-data - - org.slf4j slf4j-api + + com.microsoft.semantic-kernel + semantickernel-api-data + com.fasterxml.jackson.core jackson-databind @@ -32,36 +31,9 @@ jackson-core compile - - com.github.jknack - handlebars - - - com.google.code.findbugs - jsr305 - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - compile - com.github.spotbugs spotbugs-annotations - - org.apache.commons - commons-text - - - org.postgresql - postgresql - 42.7.4 - - - org.xerial - sqlite-jdbc - 3.47.0.0 - \ No newline at end of file diff --git a/data/semantickernel-data-mysql/pom.xml b/data/semantickernel-data-mysql/pom.xml index d2d12f13..aefd1cd9 100644 --- a/data/semantickernel-data-mysql/pom.xml +++ b/data/semantickernel-data-mysql/pom.xml @@ -24,11 +24,6 @@ com.microsoft.semantic-kernel semantickernel-data-jdbc - - - org.slf4j - slf4j-api - com.fasterxml.jackson.core jackson-databind @@ -39,26 +34,9 @@ jackson-core compile - - com.github.jknack - handlebars - - - com.google.code.findbugs - jsr305 - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - compile - com.github.spotbugs spotbugs-annotations - - org.apache.commons - commons-text - \ No newline at end of file diff --git a/data/semantickernel-data-postgres/pom.xml b/data/semantickernel-data-postgres/pom.xml index e29e758e..4382122b 100644 --- a/data/semantickernel-data-postgres/pom.xml +++ b/data/semantickernel-data-postgres/pom.xml @@ -24,11 +24,6 @@ com.microsoft.semantic-kernel semantickernel-data-jdbc - - - org.slf4j - slf4j-api - com.fasterxml.jackson.core jackson-databind @@ -39,26 +34,14 @@ jackson-core compile - - com.github.jknack - handlebars - - - com.google.code.findbugs - jsr305 - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - compile - com.github.spotbugs spotbugs-annotations - org.apache.commons - commons-text + org.postgresql + postgresql + 42.7.4 \ No newline at end of file diff --git a/data/semantickernel-data-sqlite/pom.xml b/data/semantickernel-data-sqlite/pom.xml index 69e3bc6f..ff4da8b0 100644 --- a/data/semantickernel-data-sqlite/pom.xml +++ b/data/semantickernel-data-sqlite/pom.xml @@ -23,11 +23,6 @@ com.microsoft.semantic-kernel semantickernel-data-jdbc - - - org.slf4j - slf4j-api - com.fasterxml.jackson.core jackson-databind @@ -38,26 +33,14 @@ jackson-core compile - - com.github.jknack - handlebars - - - com.google.code.findbugs - jsr305 - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - compile - com.github.spotbugs spotbugs-annotations - org.apache.commons - commons-text + org.xerial + sqlite-jdbc + 3.47.0.0 \ No newline at end of file From 376d8d215fd5b07a9ed0e7ed0b3d2d08e872c4d6 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 2 Jun 2025 23:31:46 +0200 Subject: [PATCH 168/277] Added oracle store --- data/semantickernel-data-oracle/src/test/resources/initialize.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 data/semantickernel-data-oracle/src/test/resources/initialize.sql diff --git a/data/semantickernel-data-oracle/src/test/resources/initialize.sql b/data/semantickernel-data-oracle/src/test/resources/initialize.sql new file mode 100644 index 00000000..e69de29b From 458ace8694d3462b7448dc0757c16cdb4da1b6f3 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 2 Jun 2025 23:33:40 +0200 Subject: [PATCH 169/277] Added Oracle store --- .../jdbc/JDBCVectorStoreQueryProvider.java | 2 +- data/semantickernel-data-oracle/pom.xml | 123 ++++ .../OracleVectorStoreQueryProvider.java | 570 ++++++++++++++++++ .../oracle/OracleVectorStoreRecordMapper.java | 204 +++++++ .../data/jdbc/oracle/Hotel.java | 125 ++++ ...OracleVectorStoreRecordCollectionTest.java | 322 ++++++++++ .../src/test/resources/initialize.sql | 10 + pom.xml | 2 + .../VectorStoreRecordDataField.java | 13 + .../VectorStoreRecordDefinition.java | 3 +- .../definition/VectorStoreRecordField.java | 35 ++ 11 files changed, 1407 insertions(+), 2 deletions(-) create mode 100644 data/semantickernel-data-oracle/pom.xml create mode 100644 data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java create mode 100644 data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java create mode 100644 data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java create mode 100644 data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java index afc8d82f..9381ef06 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -220,7 +220,7 @@ public Map, String> getSupportedVectorTypes() { @Override public void prepareVectorStore() { String createCollectionsTable = formatQuery( - "CREATE TABLE IF NOT EXISTS %s (collectionId VARCHAR(255) PRIMARY KEY);", + "CREATE TABLE IF NOT EXISTS %s (collectionId VARCHAR(255) PRIMARY KEY)", validateSQLidentifier(collectionsTable)); try (Connection connection = dataSource.getConnection(); diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml new file mode 100644 index 00000000..cda12f4f --- /dev/null +++ b/data/semantickernel-data-oracle/pom.xml @@ -0,0 +1,123 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../../pom.xml + + + semantickernel-data-oracle + Semantic Kernel Oracle connector + Provides a Oracle connector for the Semantic Kernel + + + 1.20.4 + + + + + + org.testcontainers + testcontainers-bom + ${testcontainers.version} + pom + import + + + + + + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-data-jdbc + + + org.slf4j + slf4j-api + + + com.fasterxml.jackson.core + jackson-databind + compile + + + com.fasterxml.jackson.core + jackson-core + compile + + + com.github.jknack + handlebars + + + com.google.code.findbugs + jsr305 + provided + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + compile + + + com.github.spotbugs + spotbugs-annotations + + + org.apache.commons + commons-text + + + org.postgresql + postgresql + 42.7.4 + + + org.xerial + sqlite-jdbc + 3.47.0.0 + + + com.oracle.database.jdbc + ojdbc11 + 23.7.0.25.01 + + + com.oracle.database.jdbc + ojdbc-provider-jackson-oson + 1.0.4 + + + org.junit.jupiter + junit-jupiter + test + + + org.junit.jupiter + junit-jupiter-api + test + + + + org.testcontainers + testcontainers + test + + + org.testcontainers + junit-jupiter + test + + + org.testcontainers + oracle-free + test + + + \ No newline at end of file diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java new file mode 100644 index 00000000..b74226fe --- /dev/null +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -0,0 +1,570 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.microsoft.semantickernel.data.jdbc.*; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import com.microsoft.semantickernel.exceptions.SKException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import oracle.jdbc.OracleResultSet; +import oracle.jdbc.OracleStatement; +import oracle.jdbc.OracleType; +import oracle.jdbc.OracleTypes; + +import javax.annotation.Nonnull; +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import java.sql.SQLException; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +public class OracleVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider { + + // This could be removed if super.collectionTable made protected + private final String collectionsTable; + + // This could be common to all query providers + private final ObjectMapper objectMapper; + + private static final Object dbCreationLock = new Object(); + + Logger logger = Logger.getLogger(OracleVectorStoreQueryProvider.class.getName()); + + public enum StringTypeMapping { + /** + * Maps String to CLOB + */ + USE_CLOB, + /** + * Maps String to VARCHAR2(4000) + */ + USE_VARCHAR + } + + private OracleVectorStoreQueryProvider( + @Nonnull DataSource dataSource, + @Nonnull String collectionsTable, + @Nonnull String prefixForCollectionTables, + int defaultVarcharSize, + @Nonnull StringTypeMapping stringTypeMapping, + ObjectMapper objectMapper) { + super( + dataSource, + collectionsTable, + prefixForCollectionTables, + buildSupportedKeyTypes(), + buildSupportedDataTypes(stringTypeMapping, defaultVarcharSize), + buildSupportedVectorTypes(defaultVarcharSize)); + this.collectionsTable = collectionsTable; + this.objectMapper = objectMapper; + } + + private static HashMap, String> buildSupportedKeyTypes() { + HashMap, String> supportedKeyTypes = new HashMap<>(); + supportedKeyTypes.put(String.class, "VARCHAR(255)"); + return supportedKeyTypes; + } + private static Map, String> buildSupportedVectorTypes(int defaultVarCharLength) { + HashMap, String> supportedVectorTypes = new HashMap<>(); + supportedVectorTypes.put(String.class, "VECTOR(%s)"); + supportedVectorTypes.put(List.class, "VECTOR(%s)"); + supportedVectorTypes.put(Collection.class, "VECTOR(%s)"); + return supportedVectorTypes; + } + + private static Map, String> buildSupportedDataTypes(StringTypeMapping stringTypeMapping, int defaultVarCharLength) { + HashMap, String> supportedDataTypes = new HashMap<>(); + if (stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR)) { + supportedDataTypes.put(String.class, "VARCHAR(" + defaultVarCharLength + ")"); + } else { + supportedDataTypes.put(String.class, "CLOB"); + } + supportedDataTypes.put(Integer.class, "INTEGER"); + supportedDataTypes.put(int.class, "INTEGER"); + supportedDataTypes.put(Long.class, "LONG"); + supportedDataTypes.put(long.class, "LONG"); + supportedDataTypes.put(Float.class, "REAL"); + supportedDataTypes.put(float.class, "REAL"); + supportedDataTypes.put(Double.class, "DOUBLE PRECISION"); + supportedDataTypes.put(double.class, "DOUBLE PRECISION"); + supportedDataTypes.put(Boolean.class, "BOOLEAN"); + supportedDataTypes.put(boolean.class, "BOOLEAN"); + supportedDataTypes.put(OffsetDateTime.class, "TIMESTAMPTZ"); + supportedDataTypes.put(List.class, "JSON"); + return supportedDataTypes; + } + + private String createIndexForVectorField(String collectionName, VectorStoreRecordVectorField vectorField) { + switch (vectorField.getIndexKind()) { + case IVFFLAT: + return "CREATE VECTOR INDEX IF NOT EXISTS " + + getIndexName(vectorField.getEffectiveStorageName()) + + " ON " + + getCollectionTableName(collectionName) + "( " + vectorField.getEffectiveStorageName() + " ) " + + " ORGANIZATION NEIGHBOR PARTITIONS " + + " WITH DISTANCE COSINE " + + "PARAMETERS ( TYPE IVF )"; + case HNSW: + return "CREATE VECTOR INDEX IF NOT EXISTS " + getIndexName(vectorField.getEffectiveStorageName()) + + " ON " + + getCollectionTableName(collectionName) + "( " + vectorField.getEffectiveStorageName() + " ) " + + "ORGANIZATION INMEMORY GRAPH " + + "WITH DISTANCE COSINE " + + "PARAMETERS (TYPE HNSW)"; + case UNDEFINED: + return null; + default: + logger.warning("Unsupported index kind: " + vectorField.getIndexKind()); + return null; + } + } + + private String getIndexName(String effectiveStorageName) { + return effectiveStorageName + "_VECTOR_INDEX"; + } + + + protected String getVectorColumnNamesAndTypes(List fields, + Map, String> types) { + List columns = fields.stream() + .map(field -> validateSQLidentifier(field.getEffectiveStorageName()) + " " + + String.format(types.get(field.getFieldType()), field.getDimensions() > 0 ? field.getDimensions() + ", FLOAT32" : "FLOAT32")) + .collect(Collectors.toList()); + + return String.join(", ", columns); + } + + @Override + protected String getInsertCollectionQuery(String collectionsTable) { + return formatQuery( + "MERGE INTO %s existing "+ + "USING (SELECT ? AS collectionId FROM DUAL) new ON (existing.collectionId = new.collectionId) " + + "WHEN NOT MATCHED THEN INSERT (existing.collectionId) VALUES (new.collectionId)", + collectionsTable); + } + + @Override + public void createCollection(String collectionName, + VectorStoreRecordDefinition recordDefinition) { + + synchronized (dbCreationLock) { + + List vectorFields = recordDefinition.getVectorFields(); + String createStorageTable = formatQuery("CREATE TABLE IF NOT EXISTS %s (" + + "%s VARCHAR(255) PRIMARY KEY, " + + "%s, " + + "%s)", + getCollectionTableName(collectionName), + getKeyColumnName(recordDefinition.getKeyField()), + getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), + getSupportedDataTypes()), + getVectorColumnNamesAndTypes(new ArrayList<>(vectorFields), + getSupportedVectorTypes())); + + String insertCollectionQuery = this.getInsertCollectionQuery(collectionsTable); + + try (Connection connection = dataSource.getConnection()) { + connection.createStatement().execute(formatQuery("DROP TABLE IF EXISTS %s", getCollectionTableName(collectionName))); + connection.setAutoCommit(false); + try (Statement statement = connection.createStatement()) { + // Create table + System.out.println(createStorageTable); + statement.addBatch(createStorageTable); + + // Index filterable columns + for (VectorStoreRecordDataField dataField : recordDefinition.getDataFields()) { + if (dataField.isFilterable()) { + String dataFieldIndex = createIndexForDataField(collectionName, dataField); + System.out.println(dataFieldIndex); + statement.addBatch(dataFieldIndex); + } + } + + // Create indexed for vectorFields + for (VectorStoreRecordVectorField vectorField : vectorFields) { + String createVectorIndex = createIndexForVectorField(collectionName, + vectorField); + + if (createVectorIndex != null) { + System.out.println(createVectorIndex); + statement.addBatch(createVectorIndex); + } + } + statement.executeBatch(); + + try (PreparedStatement insert = connection.prepareStatement( + insertCollectionQuery)) { + System.out.println(insertCollectionQuery); + insert.setString(1, collectionName); + insert.execute(); + } + + connection.commit(); + } catch (SQLException e) { + connection.rollback(); + throw new SKException("Failed to create collection", e); + } + } catch (SQLException e) { + throw new SKException("Failed to create collection", e); + } + } + } + + private String createIndexForDataField(String collectionName, VectorStoreRecordDataField dataField) { + if (supportedDataTypes.get(dataField.getFieldType()) == "JSON") { + String dataFieldIndex = "CREATE MULTIVALUE INDEX %s ON %s t (t.%s.%s)"; + return formatQuery(dataFieldIndex, + getCollectionTableName(collectionName) + "_" + dataField.getEffectiveStorageName(), + getCollectionTableName(collectionName), + dataField.getEffectiveStorageName(), + getFunctionForType(supportedDataTypes.get(dataField.getFieldSubType()))); + } else { + String dataFieldIndex = "CREATE INDEX %s ON %s (%s ASC)"; + return formatQuery(dataFieldIndex, + getCollectionTableName(collectionName) + "_" + dataField.getEffectiveStorageName(), + getCollectionTableName(collectionName), + dataField.getEffectiveStorageName() + ); + } + + } + + private String getFunctionForType(String jdbcType) { + switch (jdbcType) { + case "BOOLEAN": + return "boolean()"; + case "INTEGER": + case "LONG": + case "REAL": + case "DOUBLE PRECISION": + return "numberOnly()"; + case "TIMESTAMPTZ": + return "timestamp()"; + default: + return "string()"; + } + } + + @Override + public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { + + String upsertQuery = formatQuery("MERGE INTO %s existing "+ + "USING (SELECT %s FROM DUAL) new ON (existing.%s = new.%s) " + + "WHEN MATCHED THEN UPDATE SET %s " + + "WHEN NOT MATCHED THEN INSERT (%s) VALUES (%s)", + getCollectionTableName(collectionName), + getNamedWildcard(recordDefinition.getAllFields()), + getKeyColumnName(recordDefinition.getKeyField()), + getKeyColumnName(recordDefinition.getKeyField()), + getUpdateFieldList(recordDefinition.getKeyField(), recordDefinition.getAllFields(), "existing", "new"), + getInsertFieldList(recordDefinition.getKeyField(), recordDefinition.getAllFields(), "existing"), + getInsertFieldList(recordDefinition.getKeyField(), recordDefinition.getAllFields(), "new")); + + System.out.println(upsertQuery); + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(upsertQuery)) { + for (Object record : records) { + setUpsertStatementValues(statement, record, recordDefinition.getAllFields()); + statement.addBatch(); + } + + statement.executeBatch(); + } catch (SQLException e) { + throw new SKException("Failed to upsert records", e); + } + } + + private void setUpsertStatementValues(PreparedStatement statement, Object record, + List fields) { + JsonNode jsonNode = objectMapper.valueToTree(record); + + for (int i = 0; i < fields.size(); ++i) { + VectorStoreRecordField field = fields.get(i); + try { + JsonNode valueNode = jsonNode.get(field.getEffectiveStorageName()); + + if (field instanceof VectorStoreRecordVectorField) { + // Convert the vector field to a string + if (!field.getFieldType().equals(String.class)) { + double[] values = valueNode == null ? null : StreamSupport.stream(((ArrayNode)valueNode).spliterator(), false).mapToDouble(d -> d.asDouble()).toArray(); + statement.setObject(i + 1, values, OracleType.VECTOR_FLOAT64); + System.out.println("Set values: " + values); + continue; + } + } else if (field instanceof VectorStoreRecordDataField) { + // Convert List field to a string + if (field.getFieldType().equals(List.class)) { + statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); + System.out.println("Set values: " + objectMapper.writeValueAsString(valueNode)); + continue; + } + } + + statement.setObject(i + 1, + objectMapper.convertValue(valueNode, field.getFieldType())); + System.out.println("Set values: " + objectMapper.convertValue(valueNode, field.getFieldType())); + } catch (SQLException | JsonProcessingException e) { + throw new RuntimeException(e); + } + } + } + private String getInsertFieldList(VectorStoreRecordKeyField key, List fields, String alias) { + return fields.stream().map(f -> alias + "." + f.getEffectiveStorageName()) + .collect(Collectors.joining(", ")); + } + + private String getUpdateFieldList(VectorStoreRecordKeyField key, List fields, String oldAlias, String newAlias) { + return fields.stream().filter(f -> f != key).map(f -> oldAlias + "." + f.getEffectiveStorageName() + " = " + + newAlias + "." + f.getEffectiveStorageName()) + .collect(Collectors.joining(", ")); + + } + + + private String getNamedWildcard(List fields) { + return fields.stream().map(f -> "? " + f.getEffectiveStorageName()) + .collect(Collectors.joining(", ")); + } + + @Override + public VectorSearchResults search(String collectionName, List vector, + VectorSearchOptions options, VectorStoreRecordDefinition recordDefinition, + VectorStoreRecordMapper mapper) { + + VectorStoreRecordVectorField firstVectorField = recordDefinition.getVectorFields() + .get(0); + VectorStoreRecordVectorField vectorField = options.getVectorFieldName() == null + ? firstVectorField + : (VectorStoreRecordVectorField) recordDefinition + .getField(options.getVectorFieldName()); + DistanceFunction distanceFunction = vectorField.getDistanceFunction(); + + List fields; + if (options.isIncludeVectors()) { + fields = recordDefinition.getAllFields(); + } else { + fields = recordDefinition.getNonVectorFields(); + } + + String filter = getFilter(options.getVectorSearchFilter(), recordDefinition); + List parameters = getFilterParameters(options.getVectorSearchFilter()); + + String selectQuery = "SELECT " + + formatQuery("VECTOR_DISTANCE(%s, ?, %s) distance, ", vectorField.getEffectiveStorageName(), toOracleDistanceFunction(distanceFunction)) + + getQueryColumnsFromFields(fields) + + " FROM " + getCollectionTableName(collectionName) + + (filter != null && !filter.isEmpty() ? " WHERE " + filter : "") + + " ORDER BY distance" + + (options.getSkip() > 0 ? " OFFSET " + options.getSkip() + " ROWS" : "") + + (options.getTop() > 0 ? " FETCH " + (options.getSkip() > 0 ? "NEXT " : "FIRST ") + options.getTop() + " ROWS ONLY" : ""); + + System.out.println(selectQuery); + List> records = new ArrayList<>(); + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(selectQuery)) { + // set parameters from filters + int parameterIndex = 1; + + statement.setString(parameterIndex++, + objectMapper.writeValueAsString(vector)); + System.out.println("Set vector parameter to: " + objectMapper.writeValueAsString(vector)); + for (Object parameter : parameters) { + statement.setObject(parameterIndex++, parameter); + System.out.println("Set parameter " + parameterIndex + " to: " + parameter); + } + + // Calls to defineColumnType reduce the number of network requests. When Oracle JDBC knows that it is + // fetching VECTOR, CLOB, and/or JSON columns, the first request it sends to the database can include a LOB + // prefetch size (VECTOR and JSON are value-based-lobs). If defineColumnType is not called, then JDBC needs + // to send an additional request with the LOB prefetch size, after the first request has the database + // respond with the column data types. To request all data, the prefetch size is Integer.MAX_VALUE. + OracleStatement oracleStatement = statement.unwrap(OracleStatement.class); + int columnIndex = 1; + defineDataColumnType(columnIndex++, oracleStatement, Double.class); + for (VectorStoreRecordField field : fields) { + if (field instanceof VectorStoreRecordDataField) + defineDataColumnType(columnIndex++, oracleStatement, field.getFieldType()); + else + oracleStatement.defineColumnType(columnIndex++, OracleTypes.VECTOR_FLOAT32, Integer.MAX_VALUE); + } + oracleStatement.setLobPrefetchSize(Integer.MAX_VALUE); // Workaround for Oracle JDBC bug 37030121 + + // get result set + try (ResultSet rs = statement.executeQuery()) { + GetRecordOptions getRecordOptions = new GetRecordOptions(options.isIncludeVectors()); + while (rs.next()) { + // Cosine distance function. 1 - cosine similarity. + double score = Math.abs(rs.getDouble("distance")); + if (distanceFunction == DistanceFunction.COSINE_SIMILARITY) { + score = 1d - score; + } + records.add(new VectorSearchResult<>(mapper.mapStorageModelToRecord(rs, getRecordOptions), score)); + } + } + } catch (SQLException | JsonProcessingException e) { + logger.info(e.getMessage()); + throw new SKException("Search failed", e); + } + + + + return new VectorSearchResults<>(records); + } + + private void defineDataColumnType(int columnIndex, OracleStatement statement, Class fieldType) throws SQLException { + // swich between supported classes and define the column type on the statement + switch (supportedDataTypes.get(fieldType)) { + case "CLOB": + statement.defineColumnType(columnIndex, OracleTypes.CLOB, Integer.MAX_VALUE); + break; + case "INTEGER": + statement.defineColumnType(columnIndex, OracleTypes.INTEGER); + break; + case "LONG": + statement.defineColumnType(columnIndex, OracleTypes.BIGINT); + break; + case "REAL": + statement.defineColumnType(columnIndex, OracleTypes.REAL); + break; + case "DOUBLE PRECISION": + statement.defineColumnType(columnIndex, OracleTypes.BINARY_DOUBLE); + break; + case "BOOLEAN": + statement.defineColumnType(columnIndex, OracleTypes.BOOLEAN); + break; + case "TIMESTAMPTZ": + statement.defineColumnType(columnIndex, OracleTypes.TIMESTAMPTZ); + break; + case "JSON": + statement.defineColumnType(columnIndex, OracleTypes.JSON, Integer.MAX_VALUE); + break; + default: + statement.defineColumnType(columnIndex, OracleTypes.VARCHAR); + } + } + + + private String toOracleDistanceFunction(DistanceFunction distanceFunction) { + switch (distanceFunction) { + case DOT_PRODUCT: + return "DOT"; + case COSINE_SIMILARITY: + case COSINE_DISTANCE: + return "COSINE"; + case EUCLIDEAN_DISTANCE: + return "EUCLIDEAN"; + default: + return "COSINE"; + } + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public VectorStoreRecordMapper getVectorStoreRecordMapper( + Class recordClass, + VectorStoreRecordDefinition vectorStoreRecordDefinition) { + return OracleVectorStoreRecordMapper.builder() + .withRecordClass(recordClass) + .withVectorStoreRecordDefinition(vectorStoreRecordDefinition) + .withSupportedDataTypesMapping(getSupportedDataTypes()) + .build(); + } + + public static class Builder + extends JDBCVectorStoreQueryProvider.Builder { + + private DataSource dataSource; + private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; + private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; + private ObjectMapper objectMapper = new ObjectMapper(); + private StringTypeMapping stringTypeMapping = StringTypeMapping.USE_VARCHAR; + private int defaultVarcharSize = 4000; + + + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withDataSource(DataSource dataSource) { + this.dataSource = dataSource; + return this; + } + + /** + * Sets the collections table name. + * @param collectionsTable the collections table name + * @return the builder + */ + public Builder withCollectionsTable(String collectionsTable) { + this.collectionsTable = validateSQLidentifier(collectionsTable); + return this; + } + + /** + * Sets the prefix for collection tables. + * @param prefixForCollectionTables the prefix for collection tables + * @return the builder + */ + public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { + this.prefixForCollectionTables = validateSQLidentifier(prefixForCollectionTables); + return this; + } + + public Builder withObjectMapper( + ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + return this; + } + + /** + * Sets the desired String type mapping. + * @param stringTypeMapping the desired String type mapping. The default value is + * {@link StringTypeMapping#USE_VARCHAR} + * @return the builder + */ + public Builder withStringTypeMapping (StringTypeMapping stringTypeMapping) { + this.stringTypeMapping = stringTypeMapping; + return this; + } + + /** + * Sets the default size of the VARHCHAR2 fields. + * @param defaultVarcharSize the default size of the VARHCHAR2 fields. By default, the size + * is 4000. + * @return then builder + */ + public Builder withDefaultVarcharSize (int defaultVarcharSize) { + this.defaultVarcharSize = defaultVarcharSize; + return this; + } + + @Override + public OracleVectorStoreQueryProvider build() { + return new OracleVectorStoreQueryProvider(dataSource, collectionsTable, + prefixForCollectionTables, defaultVarcharSize, stringTypeMapping, objectMapper); + } + } +} \ No newline at end of file diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java new file mode 100644 index 00000000..1ba9fe7a --- /dev/null +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -0,0 +1,204 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.exceptions.SKException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import oracle.jdbc.OracleResultSet; +import oracle.jdbc.provider.oson.OsonModule; +import oracle.sql.json.OracleJsonArray; +import oracle.sql.json.OracleJsonObject; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; + +/** + * Maps a Oracle result set to a record. + * + * @param the record type + */ +public class OracleVectorStoreRecordMapper + extends VectorStoreRecordMapper { + + /** + * Constructs a new instance of the VectorStoreRecordMapper. + * + * @param storageModelToRecordMapper the function to convert a storage model to a record + */ + protected OracleVectorStoreRecordMapper( + BiFunction storageModelToRecordMapper) { + super(null, storageModelToRecordMapper); + } + + /** + * Creates a new builder. + * + * @param the record type + * @return the builder + */ + public static Builder builder() { + return new Builder<>(); + } + + /** + * Operation not supported. + */ + @Override + public ResultSet mapRecordToStorageModel(Record record) { + throw new UnsupportedOperationException("Not implemented"); + } + + /** + * Builder for {@link OracleVectorStoreRecordMapper}. + * + * @param the record type + */ + public static class Builder + implements SemanticKernelBuilder> { + private Class recordClass; + private VectorStoreRecordDefinition vectorStoreRecordDefinition; + private Map, String> supportedDataTypesMapping; + private ObjectMapper objectMapper = new ObjectMapper(); + + /** + * Sets the record class. + * + * @param recordClass the record class + * @return the builder + */ + public Builder withRecordClass(Class recordClass) { + this.recordClass = recordClass; + return this; + } + + /** + * Sets the vector store record definition. + * + * @param vectorStoreRecordDefinition the vector store record definition + * @return the builder + */ + public Builder withVectorStoreRecordDefinition( + VectorStoreRecordDefinition vectorStoreRecordDefinition) { + this.vectorStoreRecordDefinition = vectorStoreRecordDefinition; + return this; + } + + /** + * Sets the object mapper. + * + * @param objectMapper the object mapper + * @return the builder + */ + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withObjectMapper(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + return this; + } + + /** + * Sets the Map of supported data types and their database representation + * + * @param supportedDataTypesMapping the Map of supported data types and their + * database representation + * @return the builder + */ + public Builder withSupportedDataTypesMapping( + Map, String> supportedDataTypesMapping) { + this.supportedDataTypesMapping = supportedDataTypesMapping; + return this; + } + + /** + * Builds the {@link OracleVectorStoreRecordMapper}. + * + * @return the {@link OracleVectorStoreRecordMapper} + */ + public OracleVectorStoreRecordMapper build() { + if (recordClass == null) { + throw new SKException("recordClass is required"); + } + if (vectorStoreRecordDefinition == null) { + throw new SKException("vectorStoreRecordDefinition is required"); + } + + return new OracleVectorStoreRecordMapper<>( + (resultSet, options) -> { + try { + objectMapper.registerModule(new OsonModule()); + // Create an ObjectNode to hold the values + ObjectNode objectNode = objectMapper.createObjectNode(); + + // Read non vector fields + for (VectorStoreRecordField field : vectorStoreRecordDefinition.getNonVectorFields()) { + Class fieldType = field.getFieldType(); + + Object value; + switch (supportedDataTypesMapping.get(fieldType)) { + case "CLOB": + value = resultSet.getString(field.getEffectiveStorageName()); + break; + case "INTEGER": + value = resultSet.getInt(field.getEffectiveStorageName()); + break; + case "LONG": + value = resultSet.getInt(field.getEffectiveStorageName()); + break; + case "REAL": + value = resultSet.getFloat(field.getEffectiveStorageName()); + break; + case "DOUBLE PRECISION": + value = resultSet.getDouble(field.getEffectiveStorageName()); + break; + case "BOOLEAN": + value = resultSet.getBoolean(field.getEffectiveStorageName()); + break; + case "TIMESTAMPTZ": + value = ((OracleResultSet)resultSet).getTIMESTAMPTZ(field.getEffectiveStorageName()) + .offsetDateTimeValue(); + break; + case "JSON": + value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); + break; + default: + value = resultSet.getString(field.getEffectiveStorageName()); + } + JsonNode genericNode = objectMapper.valueToTree(value); + objectNode.set(field.getEffectiveStorageName(), genericNode); + } + if (options != null && options.isIncludeVectors()) { + for (VectorStoreRecordVectorField field : vectorStoreRecordDefinition.getVectorFields()) { + Object value = resultSet.getObject(field.getEffectiveStorageName(), float[].class); + JsonNode genericNode = objectMapper.valueToTree(value); + objectNode.set(field.getEffectiveStorageName(), genericNode); + } + } else { + for (VectorStoreRecordVectorField field : vectorStoreRecordDefinition.getVectorFields()) { + JsonNode genericNode = objectMapper.valueToTree(null); + objectNode.set(field.getEffectiveStorageName(), genericNode); + } + } + + // Deserialize the object node to the record class + return objectMapper.convertValue(objectNode, recordClass); + } catch (SQLException e) { + throw new SKException( + "Failure to serialize object, by default the JDBC connector uses Jackson, ensure your model object can be serialized by Jackson, i.e the class is visible, has getters, constructor, annotations etc.", + e); + } + }); + } + } +} diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java new file mode 100644 index 00000000..197cb8c3 --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; + +import java.util.List; + +import static com.fasterxml.jackson.annotation.JsonCreator.Mode.DELEGATING; +import static com.fasterxml.jackson.annotation.JsonCreator.Mode.PROPERTIES; + +public class Hotel { + @VectorStoreRecordKey + private final String id; + + @VectorStoreRecordData(isFilterable = true) + private final String name; + + @VectorStoreRecordData + private final int code; + + @VectorStoreRecordData + private final double price; + + @VectorStoreRecordData(isFilterable = true) + private final List tags; + + @JsonProperty("summary") + @VectorStoreRecordData( isFilterable = true, isFullTextSearchable = true ) + private final String description; + + @JsonProperty("summaryEmbedding1") + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.EUCLIDEAN_DISTANCE, indexKind = IndexKind.IVFFLAT) + private final List euclidean; + + @JsonProperty("summaryEmbedding2") + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.HNSW) + private final List cosineDistance; + + @JsonProperty("summaryEmbedding3") + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_SIMILARITY, indexKind = IndexKind.IVFFLAT) + private final List cosineSimilarity; + + @JsonProperty("summaryEmbedding4") + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.DOT_PRODUCT, indexKind = IndexKind.IVFFLAT) + private final List dotProduct; + @VectorStoreRecordData + private double rating; + + @JsonCreator(mode = DELEGATING) + public Hotel() { + this(null, null, 0, 0d, null, null, null, null, null, null, 0.0); + } + + @JsonCreator(mode = PROPERTIES) + protected Hotel( + @JsonProperty("id") String id, + @JsonProperty("name") String name, + @JsonProperty("code") int code, + @JsonProperty("price") double price, + @JsonProperty("tags") List tags, + @JsonProperty("summary") String description, + @JsonProperty("summaryEmbedding1") List euclidean, + @JsonProperty("summaryEmbedding2") List cosineDistance, + @JsonProperty("summaryEmbedding3") List cosineSimilarity, + @JsonProperty("summaryEmbedding4") List dotProduct, + @JsonProperty("rating") double rating) { + this.id = id; + this.name = name; + this.code = code; + this.price = price; + this.tags = tags; + this.description = description; + this.euclidean = euclidean; + this.cosineDistance = euclidean; + this.cosineSimilarity = euclidean; + this.dotProduct = euclidean; + this.rating = rating; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public int getCode() { + return code; + } + + public double getPrice() { return price; } + + public List getTags() { return tags; } + + public String getDescription() { + return description; + } + + public List getEuclidean() { + return euclidean; + } + + public List getCosineDistance() { + return cosineDistance; + } + + public List getDotProduct() { + return dotProduct; + } + + public double getRating() { + return rating; + } + + public void setRating(double rating) { + this.rating = rating; + } +} diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java new file mode 100644 index 00000000..546e251f --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -0,0 +1,322 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollection; +import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import oracle.jdbc.OracleConnection; +import oracle.jdbc.datasource.impl.OracleDataSource; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import java.sql.SQLException; +import java.time.Duration; +import java.util.Arrays; +import java.util.List; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.testcontainers.oracle.OracleContainer; +import org.testcontainers.utility.MountableFile; + +public class OracleVectorStoreRecordCollectionTest { + private static VectorStoreRecordCollection recordCollection; + + private static final String ORACLE_IMAGE_NAME = "gvenzl/oracle-free:23.7-slim-faststart"; + private static final OracleDataSource DATA_SOURCE; + private static final OracleDataSource SYSDBA_DATA_SOURCE; + + + static { + + try { + DATA_SOURCE = new oracle.jdbc.datasource.impl.OracleDataSource(); + SYSDBA_DATA_SOURCE = new oracle.jdbc.datasource.impl.OracleDataSource(); + String urlFromEnv = System.getenv("ORACLE_JDBC_URL"); + + if (urlFromEnv == null) { + // The Ryuk component is relied upon to stop this container. + OracleContainer oracleContainer = new OracleContainer(ORACLE_IMAGE_NAME) + .withCopyFileToContainer(MountableFile.forClasspathResource("/initialize.sql"), + "/container-entrypoint-initdb.d/initialize.sql") + .withStartupTimeout(Duration.ofSeconds(600)) + .withConnectTimeoutSeconds(600) + .withDatabaseName("pdb1") + .withUsername("testuser") + .withPassword("testpwd"); + oracleContainer.start(); + + initDataSource( + DATA_SOURCE, + oracleContainer.getJdbcUrl(), + oracleContainer.getUsername(), + oracleContainer.getPassword()); + initDataSource(SYSDBA_DATA_SOURCE, oracleContainer.getJdbcUrl(), "sys", oracleContainer.getPassword()); + } else { + initDataSource( + DATA_SOURCE, + urlFromEnv, + System.getenv("ORACLE_JDBC_USER"), + System.getenv("ORACLE_JDBC_PASSWORD")); + initDataSource( + SYSDBA_DATA_SOURCE, + urlFromEnv, + System.getenv("ORACLE_JDBC_USER"), + System.getenv("ORACLE_JDBC_PASSWORD")); + } + SYSDBA_DATA_SOURCE.setConnectionProperty(OracleConnection.CONNECTION_PROPERTY_INTERNAL_LOGON, "SYSDBA"); + + } catch (SQLException sqlException) { + throw new AssertionError(sqlException); + } + } + + static void initDataSource(OracleDataSource dataSource, String url, String username, String password) { + dataSource.setURL(url); + dataSource.setUser(username); + dataSource.setPassword(password); + } + + @BeforeAll + public static void setup() throws Exception { + + // Build a query provider + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + // Build a vector store + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + // Get a collection from the vector store + recordCollection = + vectorStore.getCollection("skhotels", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); + + recordCollection.createCollectionIfNotExistsAsync().block(); + } + + @BeforeEach + public void clearCollection() { + recordCollection.deleteCollectionAsync().block(); + recordCollection.createCollectionAsync().block(); + } + + private static List getHotels() { + return Arrays.asList( + new Hotel("id_1", "Hotel 1", 1, 1.49d, null, "Hotel 1 description", + Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), null, null, null, + 4.0), + new Hotel("id_2", "Hotel 2", 2, 1.44d, null, "Hotel 2 description with free-text search", + Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), null, null, null, + 4.0), + new Hotel("id_3", "Hotel 3", 3, 1.53d, null, "Hotel 3 description", + Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), null, null, null, + 5.0), + new Hotel("id_4", "Hotel 4", 4, 1.35d, null, "Hotel 4 description", + Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), null, null, null, + 4.0), + new Hotel("id_5", "Hotel 5", 5, 1.89d, null,"Hotel 5 description", + Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), null, null, null, + 4.0)); + } + + /** + * Search embeddings similar to the third hotel embeddings. + * In order of similarity: + * 1. Hotel 3 + * 2. Hotel 1 + * 3. Hotel 4 + */ + private static final List SEARCH_EMBEDDINGS = Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, + -0.8f, 1.1f, -2.2f, 8.2f); + + @Test + public void createAndDeleteCollectionAsync() { + assertEquals(true, recordCollection.collectionExistsAsync().block()); + + recordCollection.deleteCollectionAsync().block(); + assertEquals(false, recordCollection.collectionExistsAsync().block()); + + recordCollection.createCollectionAsync().block(); + assertEquals(true, recordCollection.collectionExistsAsync().block()); + } + + @Test + public void upsertRecordAsync() { + List hotels = getHotels(); + for (Hotel hotel : hotels) { + recordCollection.upsertAsync(hotel, null).block(); + } + + for (Hotel hotel : hotels) { + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block(); + assertNotNull(retrievedHotel); + assertEquals(hotel.getId(), retrievedHotel.getId()); + assertEquals(hotel.getName(), retrievedHotel.getName()); + assertEquals(hotel.getDescription(), retrievedHotel.getDescription()); + } + } + + @Test + public void upsertBatchAsync() { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + for (Hotel hotel : hotels) { + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block(); + assertNotNull(retrievedHotel); + assertEquals(hotel.getId(), retrievedHotel.getId()); + assertEquals(hotel.getName(), retrievedHotel.getName()); + assertEquals(hotel.getDescription(), retrievedHotel.getDescription()); + } + } + + @Test + public void getBatchAsync() { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + List keys = hotels.stream().map(Hotel::getId).collect(Collectors.toList()); + List retrievedHotels = recordCollection.getBatchAsync(keys, null).block(); + + assertNotNull(retrievedHotels); + assertEquals(keys.size(), retrievedHotels.size()); + for (Hotel hotel : retrievedHotels) { + assertTrue(keys.contains(hotel.getId())); + } + } + + @Test + public void deleteRecordAsync() { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + for (Hotel hotel : hotels) { + recordCollection.deleteAsync(hotel.getId(), null).block(); + assertNull(recordCollection.getAsync(hotel.getId(), null).block()); + } + } + + @Test + public void deleteBatchAsync() { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + List keys = hotels.stream().map(Hotel::getId).collect(Collectors.toList()); + recordCollection.deleteBatchAsync(keys, null).block(); + + for (String key : keys) { + assertNull(recordCollection.getAsync(key, null).block()); + } + } + + @ParameterizedTest + @MethodSource("parametersExactSearch") + public void exactSearch(DistanceFunction distanceFunction, List expectedDistance) { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions options = VectorSearchOptions.builder() + .withVectorFieldName(distanceFunction.getValue()) + .withTop(3) + .build(); + + // Embeddings similar to the third hotel + List> results = recordCollection + .searchAsync(SEARCH_EMBEDDINGS, options).block().getResults(); + assertNotNull(results); + assertEquals(3, results.size()); + // The third hotel should be the most similar + System.out.println(results.get(0).getScore()); + System.out.println(results.get(1).getScore()); + System.out.println(results.get(2).getScore()); + assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); + assertEquals(expectedDistance.get(0).doubleValue(), results.get(0).getScore(), 0.0001d); + assertEquals(hotels.get(0).getId(), results.get(1).getRecord().getId()); + assertEquals(expectedDistance.get(1).doubleValue(), results.get(1).getScore(), 0.0001d); + assertEquals(hotels.get(3).getId(), results.get(2).getRecord().getId()); + assertEquals(expectedDistance.get(2).doubleValue(), results.get(2).getScore(), 0.0001d); + + options = VectorSearchOptions.builder() + .withVectorFieldName(distanceFunction.getValue()) + .withSkip(1) + .withTop(-100) + .build(); + + // Skip the first result + results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, options).block().getResults(); + assertNotNull(results); + assertEquals(1, results.size()); + // The first hotel should be the most similar + assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); + assertEquals(results.get(0).getScore(), expectedDistance.get(1), 0.001d); + } + + @ParameterizedTest + @MethodSource("distanceFunctionAndDistance") + public void searchWithFilter(DistanceFunction distanceFunction, double expectedDistance) { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions options = VectorSearchOptions.builder() + .withVectorFieldName(distanceFunction.getValue()) + .withTop(3) + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("rating", 4.0).build()) + .build(); + + // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned + List> results = recordCollection + .searchAsync(SEARCH_EMBEDDINGS, options).block().getResults(); + assertNotNull(results); + assertEquals(3, results.size()); + // The first hotel should be the most similar + assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); + assertEquals(results.get(0).getScore(), expectedDistance, 0.0001d); + } + + private static Stream distanceFunctionAndDistance() { + return Stream.of( + Arguments.of (DistanceFunction.COSINE_DISTANCE, 0.8548d), + Arguments.of (DistanceFunction.COSINE_SIMILARITY, 0.1451d), + Arguments.of (DistanceFunction.DOT_PRODUCT, 30.3399d), + Arguments.of (DistanceFunction.EUCLIDEAN_DISTANCE, 18.9081d), + Arguments.of (DistanceFunction.UNDEFINED, 18.9081d) + ); + } + + private static Stream parametersExactSearch() { + return Stream.of( + Arguments.of (DistanceFunction.COSINE_SIMILARITY, Arrays.asList(0.9999d, 0.1451d, 0.0178d)), + Arguments.of (DistanceFunction.COSINE_DISTANCE, Arrays.asList(1.6422E-5d, 0.8548d, 0.9821d)), + Arguments.of (DistanceFunction.DOT_PRODUCT, Arrays.asList(202.3399d, 30.3399d, 3.6199d)), + Arguments.of (DistanceFunction.EUCLIDEAN_DISTANCE, Arrays.asList(0.1000d, 18.9081d, 19.9669d)), + Arguments.of (DistanceFunction.UNDEFINED, Arrays.asList(0.1000d, 18.9081d, 19.9669d)) + ); + } +} diff --git a/data/semantickernel-data-oracle/src/test/resources/initialize.sql b/data/semantickernel-data-oracle/src/test/resources/initialize.sql index e69de29b..4b065473 100644 --- a/data/semantickernel-data-oracle/src/test/resources/initialize.sql +++ b/data/semantickernel-data-oracle/src/test/resources/initialize.sql @@ -0,0 +1,10 @@ +-- Exit on any errors +WHENEVER SQLERROR EXIT SQL.SQLCODEAdd commentMore actions + +-- Configure the size of the Vector Pool to 1 GiB. +ALTER SYSTEM SET vector_memory_size=1G SCOPE=SPFILE; + +SHUTDOWN ABORT; +STARTUP; + +exit; \ No newline at end of file diff --git a/pom.xml b/pom.xml index 354e87de..b52f7484 100644 --- a/pom.xml +++ b/pom.xml @@ -81,6 +81,7 @@ data/semantickernel-data-hsqldb data/semantickernel-data-postgres data/semantickernel-data-sqlite + data/semantickernel-data-oracle agents/semantickernel-agents-core semantickernel-api-data semantickernel-api-exceptions @@ -249,6 +250,7 @@ ${maven.compiler.release} ${maven.compiler.release} 8 + -Xlint:unchecked diff --git a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java index 9e5aea11..713cae29 100644 --- a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java +++ b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDataField.java @@ -39,6 +39,18 @@ public VectorStoreRecordDataField( this.isFullTextSearchable = isFullTextSearchable; } + public VectorStoreRecordDataField( + @Nonnull String name, + @Nullable String storageName, + @Nonnull Class fieldType, + @Nonnull Class fieldSubType, + boolean isFilterable, + boolean isFullTextSearchable) { + super(name, storageName, fieldType, fieldSubType); + this.isFilterable = isFilterable; + this.isFullTextSearchable = isFullTextSearchable; + } + /** * Gets a value indicating whether the field is filterable. * @@ -105,6 +117,7 @@ public VectorStoreRecordDataField build() { name, storageName, fieldType, + fieldSubType, isFilterable, isFullTextSearchable); } diff --git a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java index 54b2bf2b..8f8d6212 100644 --- a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java +++ b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java @@ -7,6 +7,7 @@ import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import com.microsoft.semantickernel.exceptions.SKException; import java.lang.reflect.Field; +import java.lang.reflect.ParameterizedType; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -193,7 +194,7 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) dataFields.add(VectorStoreRecordDataField.builder() .withName(field.getName()) .withStorageName(storageName) - .withFieldType(field.getType()) + .withFieldType(field.getType(), List.class.equals(field.getType()) ? (Class)((ParameterizedType) field.getGenericType()).getActualTypeArguments()[0] : null) .isFilterable(dataAttribute.isFilterable()) .build()); } diff --git a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java index 0ba377af..f777bd5c 100644 --- a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java +++ b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordField.java @@ -14,6 +14,7 @@ public class VectorStoreRecordField { @Nullable private final String storageName; private final Class fieldType; + private final Class fieldSubType; /** * Creates a new instance of the VectorStoreRecordField class. @@ -29,6 +30,27 @@ public VectorStoreRecordField( this.name = name; this.storageName = storageName; this.fieldType = fieldType; + this.fieldSubType = null; + } + + /** + * Creates a new instance of the VectorStoreRecordField class. + * + * @param name the name of the field + * @param storageName the storage name of the field + * @param fieldType the field type + * @param fieldSubType if the field type is a list, the type of + * the list elements, otherwise null + */ + public VectorStoreRecordField( + @Nonnull String name, + @Nullable String storageName, + @Nonnull Class fieldType, + @Nonnull Class fieldSubType) { + this.name = name; + this.storageName = storageName; + this.fieldType = fieldType; + this.fieldSubType = fieldSubType; } /** @@ -68,6 +90,10 @@ public Class getFieldType() { return fieldType; } + public Class getFieldSubType() { + return fieldSubType; + } + /** * A builder for the VectorStoreRecordField class. * @param the type of the field @@ -83,6 +109,9 @@ public abstract static class Builder> @Nullable protected Class fieldType; + @Nullable + protected Class fieldSubType; + /** * Sets the name of the field. * @@ -116,6 +145,12 @@ public U withFieldType(Class fieldType) { return (U) this; } + public U withFieldType(Class fieldType, Class fieldSubType) { + this.fieldType = fieldType; + this.fieldSubType = fieldSubType; + return (U) this; + } + /** * Builds the field. * From 8bef1ffad35925c95059e0ba7ea24439847e220f Mon Sep 17 00:00:00 2001 From: psilberk Date: Mon, 2 Jun 2025 16:14:02 -0700 Subject: [PATCH 170/277] Added reference to parent's pom in sqlite --- data/semantickernel-data-sqlite/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/data/semantickernel-data-sqlite/pom.xml b/data/semantickernel-data-sqlite/pom.xml index ff4da8b0..c8a747a9 100644 --- a/data/semantickernel-data-sqlite/pom.xml +++ b/data/semantickernel-data-sqlite/pom.xml @@ -7,6 +7,7 @@ com.microsoft.semantic-kernel semantickernel-parent 1.4.4-RC2-SNAPSHOT + ../../pom.xml com.microsoft.semantic-kernel From a514b289a7b663b6a87074db5ba9dfc7abbfd10f Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Tue, 3 Jun 2025 17:01:00 +0200 Subject: [PATCH 171/277] Any tag filter --- .../OracleVectorStoreQueryProvider.java | 47 +++++++++++++++++-- ...OracleVectorStoreRecordCollectionTest.java | 34 ++++++++++++-- 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index b74226fe..b217b0e4 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -4,7 +4,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; +import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; +import com.microsoft.semantickernel.data.filter.EqualToFilterClause; import com.microsoft.semantickernel.data.jdbc.*; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; @@ -34,6 +37,7 @@ import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -387,13 +391,12 @@ public VectorSearchResults search(String collectionName, List getFilterParameters(VectorSearchFilter filter) { + if (filter == null + || filter.getFilterClauses().isEmpty()) { + return Collections.emptyList(); + } + + return filter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof EqualToFilterClause) { + EqualToFilterClause equalToFilterClause = (EqualToFilterClause) filterClause; + return equalToFilterClause.getValue(); + } else if (filterClause instanceof AnyTagEqualToFilterClause) { + AnyTagEqualToFilterClause anyTagEqualToFilterClause = (AnyTagEqualToFilterClause) filterClause; + return anyTagEqualToFilterClause.getValue(); + } else { + throw new SKException("Unsupported filter clause type '" + + filterClause.getClass().getSimpleName() + "'."); + } + }).collect(Collectors.toList()); + } + + @Override + public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { + String fieldName = JDBCVectorStoreQueryProvider + .validateSQLidentifier(filterClause.getFieldName()); + + return String.format("JSON_EXISTS(%s, '$[*]?(@ == $v_%s)' PASSING ? AS \"v_%s\")", + fieldName, fieldName, fieldName); + } + + public static Builder builder() { return new Builder(); } diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 546e251f..72ed3524 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -127,19 +127,19 @@ public void clearCollection() { private static List getHotels() { return Arrays.asList( - new Hotel("id_1", "Hotel 1", 1, 1.49d, null, "Hotel 1 description", + new Hotel("id_1", "Hotel 1", 1, 1.49d, Arrays.asList("one", "two"), "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), null, null, null, 4.0), - new Hotel("id_2", "Hotel 2", 2, 1.44d, null, "Hotel 2 description with free-text search", + new Hotel("id_2", "Hotel 2", 2, 1.44d, Arrays.asList("three", "four"), "Hotel 2 description with free-text search", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), null, null, null, 4.0), - new Hotel("id_3", "Hotel 3", 3, 1.53d, null, "Hotel 3 description", + new Hotel("id_3", "Hotel 3", 3, 1.53d, Arrays.asList("five", "six"), "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), null, null, null, 5.0), - new Hotel("id_4", "Hotel 4", 4, 1.35d, null, "Hotel 4 description", + new Hotel("id_4", "Hotel 4", 4, 1.35d, Arrays.asList("seven", "eight"), "Hotel 4 description", Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), null, null, null, 4.0), - new Hotel("id_5", "Hotel 5", 5, 1.89d, null,"Hotel 5 description", + new Hotel("id_5", "Hotel 5", 5, 1.89d, Arrays.asList("nine", "ten"),"Hotel 5 description", Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), null, null, null, 4.0)); } @@ -300,6 +300,30 @@ public void searchWithFilter(DistanceFunction distanceFunction, double expectedD assertEquals(results.get(0).getScore(), expectedDistance, 0.0001d); } + + @Test + public void searchWithTagFilter() { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions options = VectorSearchOptions.builder() +// .withVectorFieldName("") + .withTop(3) + .withVectorSearchFilter( + VectorSearchFilter.builder() + .anyTagEqualTo("tags", "three") + .build()) + .build(); + + // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned + List> results = recordCollection + .searchAsync(SEARCH_EMBEDDINGS, options).block().getResults(); + assertNotNull(results); + assertEquals(1, results.size()); + // The first hotel should be the most similar + assertEquals(hotels.get(1).getId(), results.get(0).getRecord().getId()); + } + private static Stream distanceFunctionAndDistance() { return Stream.of( Arguments.of (DistanceFunction.COSINE_DISTANCE, 0.8548d), From d8b6895931dac5ae418622ecb4222f14fac4fc62 Mon Sep 17 00:00:00 2001 From: psilberk Date: Thu, 5 Jun 2025 15:29:10 -0700 Subject: [PATCH 172/277] Clean pom and 1st Sample --- data/semantickernel-data-oracle/pom.xml | 40 ----------------- .../src/test/resources/initialize.sql | 4 +- .../data/vectorstores/oracle/Main.java | 43 +++++++++++++++++++ 3 files changed, 46 insertions(+), 41 deletions(-) create mode 100644 samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml index cda12f4f..d6218548 100644 --- a/data/semantickernel-data-oracle/pom.xml +++ b/data/semantickernel-data-oracle/pom.xml @@ -29,18 +29,10 @@ - - com.microsoft.semantic-kernel - semantickernel-api-data - com.microsoft.semantic-kernel semantickernel-data-jdbc - - org.slf4j - slf4j-api - com.fasterxml.jackson.core jackson-databind @@ -51,38 +43,6 @@ jackson-core compile - - com.github.jknack - handlebars - - - com.google.code.findbugs - jsr305 - provided - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - compile - - - com.github.spotbugs - spotbugs-annotations - - - org.apache.commons - commons-text - - - org.postgresql - postgresql - 42.7.4 - - - org.xerial - sqlite-jdbc - 3.47.0.0 - com.oracle.database.jdbc ojdbc11 diff --git a/data/semantickernel-data-oracle/src/test/resources/initialize.sql b/data/semantickernel-data-oracle/src/test/resources/initialize.sql index 4b065473..8756f121 100644 --- a/data/semantickernel-data-oracle/src/test/resources/initialize.sql +++ b/data/semantickernel-data-oracle/src/test/resources/initialize.sql @@ -4,7 +4,9 @@ WHENEVER SQLERROR EXIT SQL.SQLCODEAdd commentMore actions -- Configure the size of the Vector Pool to 1 GiB. ALTER SYSTEM SET vector_memory_size=1G SCOPE=SPFILE; +sqlplus / as sysdba + SHUTDOWN ABORT; STARTUP; -exit; \ No newline at end of file +exit \ No newline at end of file diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java new file mode 100644 index 00000000..950a62f3 --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.documentationexamples.data.vectorstores.oracle; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollection; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.samples.documentationexamples.data.index.Hotel; + +public class Main { + public static void main(String[] args) { + // Configure the data source + OracleDataSource dataSource = new OracleDataSource(); + dataSource.setURL("jdbc:oracle:thin:@localhost:1521/FREEPDB1"); + dataSource.setUser("scott"); + dataSource.setPassword("tiger"); + + // Build a query provider + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build(); + + // Build a vector store + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(dataSource) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + VectorStoreRecordCollection collection = vectorStore.getCollection( + "skhotels", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); + + // Create the collection if it doesn't exist yet. + collection.createCollectionIfNotExistsAsync().block(); + + collection.upsertBatchAsync(getHotels(), null).block(); + } +} From df07b40a9d09f91acc0e0633e0f3493670d03687 Mon Sep 17 00:00:00 2001 From: psilberk Date: Thu, 5 Jun 2025 15:41:19 -0700 Subject: [PATCH 173/277] Sample wip #2 --- .../data/vectorstores/oracle/Main.java | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java index 2c172bd4..fe37be8a 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java @@ -39,5 +39,40 @@ public static void main(String[] args) { collection.createCollectionIfNotExistsAsync().block(); collection.upsertBatchAsync(getHotels(), null).block(); + + // Retrieve the upserted record. + var retrievedHotel = collection.getAsync("1", null).block(); + + // Generate a vector for your search text, using your chosen embedding generation implementation. + // Just showing a placeholder method here for brevity. + // var searchVector = generateEmbeddingsAsync( + // "I'm looking for a hotel where customer happiness is the priority.").block(); + + // Do the search. + // var searchResult = collection.searchAsync(searchVector, VectorSearchOptions.builder() + // .withTop(1).build()).block(); + + // Hotel record = searchResult.getResults().get(0).getRecord(); + // System.out.printf("Found hotel description: %s\n", record.getDescription()); + + } + + private static List getHotels() { + return Arrays.asList( + new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", + Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), null, null, null, + 4.0), + new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", + Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), null, null, null, + 4.0), + new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", + Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), null, null, null, + 5.0), + new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", + Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), null, null, null, + 4.0), + new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", + Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), null, null, null, + 4.0)); } } \ No newline at end of file From 805f5fa8d55a8236972715d1f87eb457c1e89c1f Mon Sep 17 00:00:00 2001 From: Kaiyuan Li Date: Sat, 7 Jun 2025 00:52:21 -0700 Subject: [PATCH 174/277] Add test for IndexKind.UNDEFINED and IndexKind.HNSW --- ...OracleVectorStoreRecordCollectionTest.java | 161 +++++++++++++++++- 1 file changed, 160 insertions(+), 1 deletion(-) diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 72ed3524..4ec1599d 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -9,17 +9,28 @@ import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import oracle.jdbc.OracleConnection; import oracle.jdbc.datasource.impl.OracleDataSource; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.time.Duration; import java.util.Arrays; import java.util.List; @@ -42,7 +53,6 @@ public class OracleVectorStoreRecordCollectionTest { private static final OracleDataSource DATA_SOURCE; private static final OracleDataSource SYSDBA_DATA_SOURCE; - static { try { @@ -324,6 +334,155 @@ public void searchWithTagFilter() { assertEquals(hotels.get(1).getId(), results.get(0).getRecord().getId()); } + @Nested + class HNSWIndexTests { + @Test + void testHNSWIndexIsCreatedSuccessfully() throws Exception { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + VectorStoreRecordDataField dummyField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(String.class) + .isFilterable(false) + .build(); + + VectorStoreRecordVectorField hnswVector= VectorStoreRecordVectorField.builder() + .withName("hnsw") + .withStorageName("hnsw") + .withFieldType(List.class) + .withDimensions(8) + .withDistanceFunction(DistanceFunction.COSINE_SIMILARITY) + .withIndexKind(IndexKind.HNSW) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dummyField, hnswVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "skhotels_hnsw"; + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Object.class) + .withRecordDefinition(definition) + .build()); + + // create collection + collection.createCollectionAsync().block(); + + String expectedIndexName = hnswVector.getEffectiveStorageName().toUpperCase() + "_VECTOR_INDEX"; + + // check if index exist + try (Connection conn = DATA_SOURCE.getConnection(); + PreparedStatement stmt = conn.prepareStatement( + "SELECT COUNT(*) FROM USER_INDEXES WHERE INDEX_NAME=?")) { + stmt.setString(1, expectedIndexName); + ResultSet rs = stmt.executeQuery(); + rs.next(); + int count = rs.getInt(1); + + assertEquals(1, count, "hnsw vector index should have been created"); + } finally { + // clean up + try (Connection conn = DATA_SOURCE.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE " + "SKCOLLECTION_" + collectionName); + } + } + } + } + + @Nested + class UndefinedIndexTests { + @Test + void testNoIndexIsCreatedForUndefined() throws Exception { + // create key field + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + // create vector field, set IndexKind to UNDEFINED + VectorStoreRecordVectorField undefinedVector= VectorStoreRecordVectorField.builder() + .withName("undef") + .withStorageName("undef") + .withFieldType(List.class) + .withDimensions(8) + .withDistanceFunction(DistanceFunction.COSINE_SIMILARITY) + .withIndexKind(IndexKind.UNDEFINED) + .build(); + + VectorStoreRecordDataField dummyField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(String.class) + .isFilterable(false) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dummyField, undefinedVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "skhotels_undefined"; + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Object.class) + .withRecordDefinition(definition) + .build()); + + // create collection + collection.createCollectionAsync().block(); + + // check if index exist + String expectedIndexName = undefinedVector.getEffectiveStorageName().toUpperCase() + "_VETCOR_INDEX"; + try (Connection conn = DATA_SOURCE.getConnection(); + PreparedStatement stmt = conn.prepareStatement( + "SELECT COUNT(*) FROM USER_INDEXES WHERE INDEX_NAME = ?")) { + stmt.setString(1, expectedIndexName); + ResultSet rs = stmt.executeQuery(); + rs.next(); + int count = rs.getInt(1); + + assertEquals(0,count,"Vector index should not be created for IndexKind.UNDEFINED"); + } finally { + // clean up + try (Connection conn = DATA_SOURCE.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE " + "SKCOLLECTION_" + collectionName); + } + } + } + } + private static Stream distanceFunctionAndDistance() { return Stream.of( Arguments.of (DistanceFunction.COSINE_DISTANCE, 0.8548d), From 1ca94390fe2078fbb0bdecfef2dc647e5bb1fc5a Mon Sep 17 00:00:00 2001 From: psilberk Date: Mon, 9 Jun 2025 15:22:43 -0700 Subject: [PATCH 175/277] Remove old implementation --- .../OracleVectorStoreQueryProvider.java | 144 ------------------ 1 file changed, 144 deletions(-) delete mode 100644 data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java deleted file mode 100644 index 10af511c..00000000 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ /dev/null @@ -1,144 +0,0 @@ -package com.microsoft.semantickernel.data.jdbc.oracle; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; -import com.microsoft.semantickernel.exceptions.SKException; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; - -import javax.annotation.Nonnull; -import javax.sql.DataSource; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.util.List; - -public class OracleVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider { - - // This could be removed if super.collectionTable made protected - private final String collectionsTable; - - // This could be common to all query providers - private final ObjectMapper objectMapper; - - private OracleVectorStoreQueryProvider(@Nonnull DataSource dataSource, @Nonnull String collectionsTable, @Nonnull String prefixForCollectionTables, - ObjectMapper objectMapper) { - super(dataSource, collectionsTable, prefixForCollectionTables); - this.collectionsTable = collectionsTable; - this.objectMapper = objectMapper; - } - - @Override - public void prepareVectorStore() { - String createCollectionsTable = formatQuery( - "CREATE TABLE IF NOT EXISTS %s (collectionId VARCHAR(255) PRIMARY KEY)", - validateSQLidentifier(collectionsTable)); - - try (Connection connection = dataSource.getConnection(); - PreparedStatement createTable = connection.prepareStatement(createCollectionsTable)) { - createTable.execute(); - } catch (SQLException e) { - throw new SKException("Failed to prepare vector store", e); - } - } - - @Override - public void createCollection(String collectionName, - VectorStoreRecordDefinition recordDefinition) { - // TODO Override implementation. Eg: mapping TEXT to VARCHAR - super.createCollection(collectionName, recordDefinition); - } - - @Override - public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { - - // TODO look for public void createCollection(String collectionName, VectorStoreRecordDefinition recordDefinition) { - - // TODO Make this a MERGE query - -// String upsertStatemente = formatQuery(""" -// MERGE INTO %s EXIST_REC USING (SELECT ? AS ID) NEW_REC ON (EXIST_REC.%s = NEW_REC.ID) -// WHEN MATACHED THEN UPDATE SET EXISTING REC -// """, -// getCollectionTableName(collectionName), -// recordDefinition.getKeyField().getName(), -// getQueryColumnsFromFields(fields), -// getWildcardString(fields.size()), -// onDuplicateKeyUpdate);super.upsertRecords(collectionName, records, recordDefinition, options); - - String query = formatQuery("INSERT INTO %s (%s, %s, %s) values (?, ?, ?)", - getCollectionTableName(collectionName), - recordDefinition.getAllFields().get(0).getStorageName(), - recordDefinition.getAllFields().get(1).getStorageName(), - recordDefinition.getAllFields().get(2).getStorageName()); - - try (Connection connection = dataSource.getConnection(); - PreparedStatement statement = connection.prepareStatement(query)) { - for (Object record : records) { - JsonNode jsonNode = objectMapper.valueToTree(record); - for (int i = 0; i < 3; i++) { - statement.setObject(i + 1, jsonNode - .get(recordDefinition.getAllFields().get(i).getStorageName()).asText()); - } - statement.addBatch(); - } - statement.executeBatch(); - } catch (SQLException e) { - throw new SKException("Failed to upsert records", e); - } - } - - public static Builder builder() { - return new Builder(); - } - - public static class Builder - extends JDBCVectorStoreQueryProvider.Builder { - - private DataSource dataSource; - private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; - private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; - private ObjectMapper objectMapper = new ObjectMapper(); - - @SuppressFBWarnings("EI_EXPOSE_REP2") - public Builder withDataSource(DataSource dataSource) { - this.dataSource = dataSource; - return this; - } - - /** - * Sets the collections table name. - * @param collectionsTable the collections table name - * @return the builder - */ - public Builder withCollectionsTable(String collectionsTable) { - this.collectionsTable = validateSQLidentifier(collectionsTable); - return this; - } - - /** - * Sets the prefix for collection tables. - * @param prefixForCollectionTables the prefix for collection tables - * @return the builder - */ - public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { - this.prefixForCollectionTables = validateSQLidentifier(prefixForCollectionTables); - return this; - } - - public Builder withObjectMapper( - ObjectMapper objectMapper) { - this.objectMapper = objectMapper; - return this; - } - - @Override - public OracleVectorStoreQueryProvider build() { - return new OracleVectorStoreQueryProvider(dataSource, collectionsTable, - prefixForCollectionTables, objectMapper); - } - } -} \ No newline at end of file From c3bfafbd6109782c7c452c82345f82559d6d38ea Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Tue, 10 Jun 2025 17:55:16 +0200 Subject: [PATCH 176/277] Added constants for type mapping and matched ODP type mapping when possible. --- .../jdbc/oracle/OracleDataTypesMapping.java | 18 ++++++ .../OracleVectorStoreQueryProvider.java | 64 +++++++++++++------ .../oracle/OracleVectorStoreRecordMapper.java | 32 +++++++--- 3 files changed, 84 insertions(+), 30 deletions(-) create mode 100644 data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java new file mode 100644 index 00000000..f441a440 --- /dev/null +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java @@ -0,0 +1,18 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +public class OracleDataTypesMapping { + public static final String STRING_VARCHAR = "NVARCHAR2(%s)"; + public static final String STRING_CLOB = "CLOB"; + public static final String BOOLEAN = "BOOLEAN"; + public static final String BYTE = "NUMBER(3)"; + public static final String BYTE_ARRAY = "RAW(2000)"; + public static final String SHORT = "NUMBER(5)"; + public static final String INTEGER = "NUMBER(10)"; + public static final String LONG = "NUMBER(19)"; + public static final String FLOAT = "BINARY_FLOAT"; + public static final String DOUBLE = "BINARY_DOUBLE"; + public static final String DECIMAL = "NUMBER(18,2)"; + public static final String OFFSET_DATE_TIME = "TIMESTAMP(7) WITH TIME ZONE"; + public static final String UUID = "RAW(16)"; + public static final String JSON = "JSON"; +} diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index b217b0e4..12466a6f 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -29,6 +29,7 @@ import javax.annotation.Nonnull; import javax.sql.DataSource; +import java.math.BigDecimal; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -41,6 +42,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -81,17 +83,25 @@ private OracleVectorStoreQueryProvider( prefixForCollectionTables, buildSupportedKeyTypes(), buildSupportedDataTypes(stringTypeMapping, defaultVarcharSize), - buildSupportedVectorTypes(defaultVarcharSize)); + buildSupportedVectorTypes()); this.collectionsTable = collectionsTable; this.objectMapper = objectMapper; } private static HashMap, String> buildSupportedKeyTypes() { HashMap, String> supportedKeyTypes = new HashMap<>(); - supportedKeyTypes.put(String.class, "VARCHAR(255)"); + supportedKeyTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, 255)); + supportedKeyTypes.put(short.class, OracleDataTypesMapping.SHORT); + supportedKeyTypes.put(Short.class, OracleDataTypesMapping.SHORT); + supportedKeyTypes.put(int.class, OracleDataTypesMapping.INTEGER); + supportedKeyTypes.put(Integer.class, OracleDataTypesMapping.INTEGER); + supportedKeyTypes.put(long.class, OracleDataTypesMapping.LONG); + supportedKeyTypes.put(Long.class, OracleDataTypesMapping.LONG); + supportedKeyTypes.put(UUID.class, OracleDataTypesMapping.UUID); + return supportedKeyTypes; } - private static Map, String> buildSupportedVectorTypes(int defaultVarCharLength) { + private static Map, String> buildSupportedVectorTypes() { HashMap, String> supportedVectorTypes = new HashMap<>(); supportedVectorTypes.put(String.class, "VECTOR(%s)"); supportedVectorTypes.put(List.class, "VECTOR(%s)"); @@ -102,22 +112,29 @@ private static Map, String> buildSupportedVectorTypes(int defaultVarCha private static Map, String> buildSupportedDataTypes(StringTypeMapping stringTypeMapping, int defaultVarCharLength) { HashMap, String> supportedDataTypes = new HashMap<>(); if (stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR)) { - supportedDataTypes.put(String.class, "VARCHAR(" + defaultVarCharLength + ")"); + supportedDataTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, defaultVarCharLength)); } else { - supportedDataTypes.put(String.class, "CLOB"); + supportedDataTypes.put(String.class, OracleDataTypesMapping.STRING_CLOB); } - supportedDataTypes.put(Integer.class, "INTEGER"); - supportedDataTypes.put(int.class, "INTEGER"); - supportedDataTypes.put(Long.class, "LONG"); - supportedDataTypes.put(long.class, "LONG"); - supportedDataTypes.put(Float.class, "REAL"); - supportedDataTypes.put(float.class, "REAL"); - supportedDataTypes.put(Double.class, "DOUBLE PRECISION"); - supportedDataTypes.put(double.class, "DOUBLE PRECISION"); - supportedDataTypes.put(Boolean.class, "BOOLEAN"); - supportedDataTypes.put(boolean.class, "BOOLEAN"); - supportedDataTypes.put(OffsetDateTime.class, "TIMESTAMPTZ"); - supportedDataTypes.put(List.class, "JSON"); + supportedDataTypes.put(byte.class, OracleDataTypesMapping.BYTE); + supportedDataTypes.put(Byte.class, OracleDataTypesMapping.BYTE); + supportedDataTypes.put(short.class, OracleDataTypesMapping.SHORT); + supportedDataTypes.put(Short.class, OracleDataTypesMapping.SHORT); + supportedDataTypes.put(int.class, OracleDataTypesMapping.INTEGER); + supportedDataTypes.put(Integer.class, OracleDataTypesMapping.INTEGER); + supportedDataTypes.put(long.class, OracleDataTypesMapping.LONG); + supportedDataTypes.put(Long.class, OracleDataTypesMapping.LONG); + supportedDataTypes.put(Float.class, OracleDataTypesMapping.FLOAT); + supportedDataTypes.put(float.class, OracleDataTypesMapping.FLOAT); + supportedDataTypes.put(Double.class, OracleDataTypesMapping.DOUBLE); + supportedDataTypes.put(double.class, OracleDataTypesMapping.DOUBLE); + supportedDataTypes.put(BigDecimal.class, OracleDataTypesMapping.DECIMAL); + supportedDataTypes.put(Boolean.class, OracleDataTypesMapping.BOOLEAN); + supportedDataTypes.put(boolean.class, OracleDataTypesMapping.BOOLEAN); + supportedDataTypes.put(OffsetDateTime.class, OracleDataTypesMapping.OFFSET_DATE_TIME); + supportedDataTypes.put(UUID.class, OracleDataTypesMapping.UUID); + supportedDataTypes.put(byte[].class, OracleDataTypesMapping.BYTE_ARRAY); + supportedDataTypes.put(List.class, OracleDataTypesMapping.JSON); return supportedDataTypes; } @@ -178,11 +195,11 @@ public void createCollection(String collectionName, List vectorFields = recordDefinition.getVectorFields(); String createStorageTable = formatQuery("CREATE TABLE IF NOT EXISTS %s (" - + "%s VARCHAR(255) PRIMARY KEY, " + + "%s PRIMARY KEY, " + "%s, " + "%s)", getCollectionTableName(collectionName), - getKeyColumnName(recordDefinition.getKeyField()), + getKeyColumnNameAndType(recordDefinition.getKeyField(), getSupportedDataTypes()), getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), getSupportedDataTypes()), getVectorColumnNamesAndTypes(new ArrayList<>(vectorFields), @@ -237,6 +254,11 @@ public void createCollection(String collectionName, } } + private String getKeyColumnNameAndType(VectorStoreRecordKeyField field, Map, String> types) { + return validateSQLidentifier(field.getEffectiveStorageName()) + " " + + types.get(field.getFieldType()); + } + private String createIndexForDataField(String collectionName, VectorStoreRecordDataField dataField) { if (supportedDataTypes.get(dataField.getFieldType()) == "JSON") { String dataFieldIndex = "CREATE MULTIVALUE INDEX %s ON %s t (t.%s.%s)"; @@ -545,7 +567,7 @@ public static class Builder private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; private ObjectMapper objectMapper = new ObjectMapper(); private StringTypeMapping stringTypeMapping = StringTypeMapping.USE_VARCHAR; - private int defaultVarcharSize = 4000; + private int defaultVarcharSize = 2000; @SuppressFBWarnings("EI_EXPOSE_REP2") @@ -594,7 +616,7 @@ public Builder withStringTypeMapping (StringTypeMapping stringTypeMapping) { /** * Sets the default size of the VARHCHAR2 fields. * @param defaultVarcharSize the default size of the VARHCHAR2 fields. By default, the size - * is 4000. + * is 2000. * @return then builder */ public Builder withDefaultVarcharSize (int defaultVarcharSize) { diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index 1ba9fe7a..eaa8e7dc 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -147,29 +147,43 @@ public OracleVectorStoreRecordMapper build() { Object value; switch (supportedDataTypesMapping.get(fieldType)) { - case "CLOB": + case OracleDataTypesMapping.STRING_CLOB: value = resultSet.getString(field.getEffectiveStorageName()); break; - case "INTEGER": - value = resultSet.getInt(field.getEffectiveStorageName()); + case OracleDataTypesMapping.BYTE: + value = resultSet.getByte(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.SHORT: + value = resultSet.getShort(field.getEffectiveStorageName()); break; - case "LONG": + case OracleDataTypesMapping.INTEGER: value = resultSet.getInt(field.getEffectiveStorageName()); break; - case "REAL": + case OracleDataTypesMapping.LONG: + value = resultSet.getLong(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.FLOAT: value = resultSet.getFloat(field.getEffectiveStorageName()); break; - case "DOUBLE PRECISION": + case OracleDataTypesMapping.DOUBLE: value = resultSet.getDouble(field.getEffectiveStorageName()); break; - case "BOOLEAN": + case OracleDataTypesMapping.DECIMAL: + value = resultSet.getBigDecimal(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.BOOLEAN: value = resultSet.getBoolean(field.getEffectiveStorageName()); break; - case "TIMESTAMPTZ": + case OracleDataTypesMapping.OFFSET_DATE_TIME: value = ((OracleResultSet)resultSet).getTIMESTAMPTZ(field.getEffectiveStorageName()) .offsetDateTimeValue(); break; - case "JSON": + case OracleDataTypesMapping.BYTE_ARRAY: + value = resultSet.getBytes(field.getEffectiveStorageName()); + break; + // fallthrough + case OracleDataTypesMapping.UUID: + case OracleDataTypesMapping.JSON: value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); break; default: From bc0f64060c91f6f15928a2361879c5225dac7a4f Mon Sep 17 00:00:00 2001 From: psilberk Date: Tue, 10 Jun 2025 23:07:44 -0700 Subject: [PATCH 177/277] Adding Book class and pom fixes --- data/semantickernel-data-postgres/pom.xml | 1 - .../semantickernel-learn-resources/pom.xml | 11 +++- .../data/vectorstores/oracle/Book.java | 45 ++++++++++++++++ .../data/vectorstores/oracle/Main.java | 51 ++++++++----------- 4 files changed, 75 insertions(+), 33 deletions(-) create mode 100644 samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java diff --git a/data/semantickernel-data-postgres/pom.xml b/data/semantickernel-data-postgres/pom.xml index 4382122b..73591658 100644 --- a/data/semantickernel-data-postgres/pom.xml +++ b/data/semantickernel-data-postgres/pom.xml @@ -10,7 +10,6 @@ ../../pom.xml - com.microsoft.semantic-kernel semantickernel-data-postgres Semantic Kernel PostreSQL connector Provides a PostreSQL connector for the Semantic Kernel diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 68062bf4..b8a1ea0b 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -41,7 +41,16 @@ com.microsoft.semantic-kernel semantickernel-data-redis - + + com.microsoft.semantic-kernel + semantickernel-data-oracle + 1.4.4-RC2-SNAPSHOT + + + com.microsoft.semantic-kernel + semantickernel-data-postgres + 1.4.4-RC2-SNAPSHOT + org.apache.logging.log4j log4j-api diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java new file mode 100644 index 00000000..4812c778 --- /dev/null +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.documentationexamples.data.vectorstores.oracle; + +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; +import java.util.List; + +public class Book { + + @VectorStoreRecordKey + private final String isbn; + + public Book(String isbn, String title, String author, int pages, + List tags, String summary, List summaryEmbedding) { + this.isbn = isbn; + this.title = title; + this.author = author; + this.pages = pages; + this.tags = tags; + this.summary = summary; + this.summaryEmbedding = summaryEmbedding; + } + + @VectorStoreRecordData(isFilterable = true) + private final String title; + + @VectorStoreRecordData(isFilterable = true) + private final String author; + + @VectorStoreRecordData + private final int pages; + + @VectorStoreRecordData(isFilterable = true) + private final List tags; + + @VectorStoreRecordData( isFilterable = true, isFullTextSearchable = true ) + private final String summary; + + @VectorStoreRecordVector(dimensions = 4, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.HNSW) + private final List summaryEmbedding; + +} \ No newline at end of file diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java index fe37be8a..d71a0f7e 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java @@ -3,13 +3,17 @@ import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; -import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollection; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.data.jdbc.postgres.PostgreSQLVectorStoreQueryProvider; -import com.microsoft.semantickernel.samples.documentationexamples.data.index.Hotel; +import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; +import oracle.jdbc.datasource.impl.OracleDataSource; public class Main { - public static void main(String[] args) { + public static void main(String[] args) throws SQLException { + // Configure the data source OracleDataSource dataSource = new OracleDataSource(); dataSource.setURL("jdbc:oracle:thin:@localhost:1521/FREEPDB1"); @@ -29,50 +33,35 @@ public static void main(String[] args) { .build()) .build(); - VectorStoreRecordCollection collection = vectorStore.getCollection( - "skhotels", - JDBCVectorStoreRecordCollectionOptions.builder() - .withRecordClass(Hotel.class) + VectorStoreRecordCollection collection = vectorStore.getCollection( + "books", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Book.class) .build()); // Create the collection if it doesn't exist yet. collection.createCollectionIfNotExistsAsync().block(); - collection.upsertBatchAsync(getHotels(), null).block(); + collection.upsertBatchAsync(books, null).block(); // Retrieve the upserted record. - var retrievedHotel = collection.getAsync("1", null).block(); + //var retrievedBook = collection.getAsync("1", null).block(); // Generate a vector for your search text, using your chosen embedding generation implementation. // Just showing a placeholder method here for brevity. // var searchVector = generateEmbeddingsAsync( - // "I'm looking for a hotel where customer happiness is the priority.").block(); + // "I'm looking for a Book where customer happiness is the priority.").block(); // Do the search. // var searchResult = collection.searchAsync(searchVector, VectorSearchOptions.builder() // .withTop(1).build()).block(); - // Hotel record = searchResult.getResults().get(0).getRecord(); - // System.out.printf("Found hotel description: %s\n", record.getDescription()); + // Book record = searchResult.getResults().get(0).getRecord(); + // System.out.printf("Found Book description: %s\n", record.getDescription()); } - private static List getHotels() { - return Arrays.asList( - new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", - Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), null, null, null, - 4.0), - new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", - Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), null, null, null, - 4.0), - new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", - Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), null, null, null, - 5.0), - new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", - Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), null, null, null, - 4.0), - new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", - Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), null, null, null, - 4.0)); - } + static List books = Arrays.asList( + new Book("1", "one", "sking", 0, null, "sum", null)); + } \ No newline at end of file From 3cbee729a3e55a302f5d7cc00df54bd1e6ab5256 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Wed, 11 Jun 2025 11:45:30 +0200 Subject: [PATCH 178/277] Missing getter methods and wrong null check --- .../OracleVectorStoreQueryProvider.java | 2 +- .../data/vectorstores/oracle/Book.java | 34 +++++++++++++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index b74226fe..71da3199 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -309,7 +309,7 @@ private void setUpsertStatementValues(PreparedStatement statement, Object record if (field instanceof VectorStoreRecordVectorField) { // Convert the vector field to a string if (!field.getFieldType().equals(String.class)) { - double[] values = valueNode == null ? null : StreamSupport.stream(((ArrayNode)valueNode).spliterator(), false).mapToDouble(d -> d.asDouble()).toArray(); + double[] values = valueNode.isNull() ? null : StreamSupport.stream(((ArrayNode)valueNode).spliterator(), false).mapToDouble(d -> d.asDouble()).toArray(); statement.setObject(i + 1, values, OracleType.VECTOR_FLOAT64); System.out.println("Set values: " + values); continue; diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java index 4812c778..aeaff6e8 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java @@ -10,9 +10,6 @@ public class Book { - @VectorStoreRecordKey - private final String isbn; - public Book(String isbn, String title, String author, int pages, List tags, String summary, List summaryEmbedding) { this.isbn = isbn; @@ -24,6 +21,10 @@ public Book(String isbn, String title, String author, int pages, this.summaryEmbedding = summaryEmbedding; } + + @VectorStoreRecordKey + private final String isbn; + @VectorStoreRecordData(isFilterable = true) private final String title; @@ -42,4 +43,31 @@ public Book(String isbn, String title, String author, int pages, @VectorStoreRecordVector(dimensions = 4, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.HNSW) private final List summaryEmbedding; + public String getIsbn() { + return isbn; + } + + public String getTitle() { + return title; + } + + public String getAuthor() { + return author; + } + + public int getPages() { + return pages; + } + + public List getTags() { + return tags; + } + + public String getSummary() { + return summary; + } + + public List getSummaryEmbedding() { + return summaryEmbedding; + } } \ No newline at end of file From f282705f7ab335809e72e03d829d3d6e092f4ae6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Jun 2025 14:45:36 +0000 Subject: [PATCH 179/277] Bump org.postgresql:postgresql in /semantickernel-experimental Bumps [org.postgresql:postgresql](https://github.com/pgjdbc/pgjdbc) from 42.7.4 to 42.7.7. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.7.4...REL42.7.7) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-version: 42.7.7 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- semantickernel-experimental/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 58f99e03..98ce0918 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -110,7 +110,7 @@ org.postgresql postgresql - 42.7.4 + 42.7.7 org.xerial From 2e013bf21aa396905eb993609ac6721473ad248c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 13 Jun 2025 03:19:54 +0000 Subject: [PATCH 180/277] Bump org.postgresql:postgresql in /data/semantickernel-data-jdbc Bumps [org.postgresql:postgresql](https://github.com/pgjdbc/pgjdbc) from 42.7.4 to 42.7.7. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.7.4...REL42.7.7) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-version: 42.7.7 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- data/semantickernel-data-jdbc/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 5d0883eb..4cf608b5 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -56,7 +56,7 @@ org.postgresql postgresql - 42.7.4 + 42.7.7 org.xerial From c3db0367affd318079e94b9fe6c1b004f05173c2 Mon Sep 17 00:00:00 2001 From: Kaiyuan Li Date: Sat, 14 Jun 2025 00:50:03 -0700 Subject: [PATCH 181/277] add test for key type --- ...OracleVectorStoreRecordCollectionTest.java | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 4ec1599d..05c174a2 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -1,5 +1,7 @@ package com.microsoft.semantickernel.data.jdbc.oracle; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollection; import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; @@ -16,6 +18,7 @@ import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; + import oracle.jdbc.OracleConnection; import oracle.jdbc.datasource.impl.OracleDataSource; import org.junit.jupiter.api.BeforeAll; @@ -33,7 +36,9 @@ import java.sql.Statement; import java.time.Duration; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.UUID; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -334,6 +339,69 @@ public void searchWithTagFilter() { assertEquals(hotels.get(1).getId(), results.get(0).getRecord().getId()); } + @ParameterizedTest + @MethodSource("supportedKeyTypes") + void testKeyTypes(String suffix, Class keyType, Object keyValue) { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(keyType) + .build(); + + VectorStoreRecordDataField dummyField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(String.class) + .build(); + + VectorStoreRecordVectorField dummyVector = VectorStoreRecordVectorField.builder() + .withName("vec") + .withStorageName("vec") + .withFieldType(List.class) + .withDimensions(2) + .withDistanceFunction(DistanceFunction.EUCLIDEAN_DISTANCE) + .withIndexKind(IndexKind.UNDEFINED) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dummyField, dummyVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "test_keytype_" + suffix; + + VectorStoreRecordCollection collectionRaw = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(DummyRecordForKeyTypes.class) + .withRecordDefinition(definition) + .build()); + + VectorStoreRecordCollection collection = + (VectorStoreRecordCollection) collectionRaw; + + collection.createCollectionAsync().block(); + + DummyRecordForKeyTypes record = new DummyRecordForKeyTypes(keyValue, "dummyValue", Arrays.asList(1.0f, 2.0f)); + collection.upsertAsync(record, null).block(); + + DummyRecordForKeyTypes result = collection.getAsync(keyValue, null).block(); + assertNotNull(result); + assertEquals("dummyValue", result.getDummy()); + + collection.deleteCollectionAsync().block(); + } + @Nested class HNSWIndexTests { @Test @@ -502,4 +570,48 @@ private static Stream parametersExactSearch() { Arguments.of (DistanceFunction.UNDEFINED, Arrays.asList(0.1000d, 18.9081d, 19.9669d)) ); } + + // commented out temporarily because only String type key is supported in + // JDBCVectorStoreRecordCollection#getKeyFromRecord: + // ... + // return (String) keyField.get(data); + // ... + // thus upsertAync/getAsync won't work + private static Stream supportedKeyTypes() { + return Stream.of( + Arguments.of("string", String.class, "asd123")/*, + Arguments.of("integer", Integer.class, 321), + Arguments.of("long", Long.class, 5L), + Arguments.of("short", Short.class, (short) 3), + Arguments.of("uuid", UUID.class, UUID.randomUUID())*/ + ); + } + + private static class DummyRecordForKeyTypes { + private final Object id; + private final String dummy; + private final List vec; + @JsonCreator + public DummyRecordForKeyTypes( + @JsonProperty("id")Object id, + @JsonProperty("dummy") String dummy, + @JsonProperty("vec") List vec) { + this.id = id; + this.dummy = dummy; + this.vec = vec; + } + + public Object getId() { + return id; + } + + public String getDummy() { + return dummy; + } + + @Override + public String toString() { + return String.valueOf(id); + } + } } From 2306e210f5f3645248313ced01283c663c5ed7a7 Mon Sep 17 00:00:00 2001 From: Kaiyuan Li Date: Sat, 14 Jun 2025 00:59:38 -0700 Subject: [PATCH 182/277] enable test for COSINE_DISTANCE, COSINE_SIMILARITY, DOT_PRODUCT in Hotel.java --- .../semantickernel/data/jdbc/oracle/Hotel.java | 6 +++--- .../OracleVectorStoreRecordCollectionTest.java | 15 ++++++++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java index 197cb8c3..667d771e 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java @@ -77,9 +77,9 @@ protected Hotel( this.tags = tags; this.description = description; this.euclidean = euclidean; - this.cosineDistance = euclidean; - this.cosineSimilarity = euclidean; - this.dotProduct = euclidean; + this.cosineDistance = cosineDistance; + this.cosineSimilarity = cosineSimilarity; + this.dotProduct = dotProduct; this.rating = rating; } diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 05c174a2..e71a9655 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -141,21 +141,26 @@ public void clearCollection() { } private static List getHotels() { + List vec1 = Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f); + List vec2 = Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f); + List vec3 = Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f); + List vec4 = Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f); + List vec5 =Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f); return Arrays.asList( new Hotel("id_1", "Hotel 1", 1, 1.49d, Arrays.asList("one", "two"), "Hotel 1 description", - Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), null, null, null, + vec1, vec1, vec1, vec1, 4.0), new Hotel("id_2", "Hotel 2", 2, 1.44d, Arrays.asList("three", "four"), "Hotel 2 description with free-text search", - Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), null, null, null, + vec2, vec2, vec2, vec2, 4.0), new Hotel("id_3", "Hotel 3", 3, 1.53d, Arrays.asList("five", "six"), "Hotel 3 description", - Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), null, null, null, + vec3, vec3, vec3, vec3, 5.0), new Hotel("id_4", "Hotel 4", 4, 1.35d, Arrays.asList("seven", "eight"), "Hotel 4 description", - Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), null, null, null, + vec4, vec4, vec4, vec4, 4.0), new Hotel("id_5", "Hotel 5", 5, 1.89d, Arrays.asList("nine", "ten"),"Hotel 5 description", - Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), null, null, null, + vec5, vec5, vec5, vec5, 4.0)); } From 0bf9199c280d88c254b5c46a24100133592cd088 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Tue, 17 Jun 2025 21:11:59 +0200 Subject: [PATCH 183/277] Refactoring --- .../jdbc/JDBCVectorStoreRecordCollection.java | 6 +- .../jdbc/oracle/OracleDataTypesMapping.java | 4 + .../oracle/OracleVectorStoreFieldHelper.java | 307 ++++++++++++++++++ .../OracleVectorStoreQueryProvider.java | 297 ++++++----------- .../data/jdbc/oracle/Hotel.java | 22 +- ...OracleVectorStoreRecordCollectionTest.java | 25 +- .../VectorStoreRecordDefinition.java | 2 +- .../VectorStoreRecordVectorField.java | 6 +- 8 files changed, 446 insertions(+), 223 deletions(-) create mode 100644 data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java index 2d75af14..81395589 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -34,10 +34,10 @@ public class JDBCVectorStoreRecordCollection implements SQLVectorStoreRecordCollection { private final String collectionName; - private final VectorStoreRecordDefinition recordDefinition; - private final VectorStoreRecordMapper vectorStoreRecordMapper; + protected final VectorStoreRecordDefinition recordDefinition; + protected final VectorStoreRecordMapper vectorStoreRecordMapper; private final JDBCVectorStoreRecordCollectionOptions options; - private final SQLVectorStoreQueryProvider queryProvider; + protected final SQLVectorStoreQueryProvider queryProvider; /** * Creates a new instance of the {@link JDBCVectorStoreRecordCollection}. diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java index f441a440..4b74527b 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java @@ -1,5 +1,8 @@ package com.microsoft.semantickernel.data.jdbc.oracle; +/** + * Defines oracle database type constants for supported field types. + */ public class OracleDataTypesMapping { public static final String STRING_VARCHAR = "NVARCHAR2(%s)"; public static final String STRING_CLOB = "CLOB"; @@ -15,4 +18,5 @@ public class OracleDataTypesMapping { public static final String OFFSET_DATE_TIME = "TIMESTAMP(7) WITH TIME ZONE"; public static final String UUID = "RAW(16)"; public static final String JSON = "JSON"; + public static final String VECTOR_FLOAT = "VECTOR(%s, FLOAT32)"; } diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java new file mode 100644 index 00000000..3ae09ed4 --- /dev/null +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java @@ -0,0 +1,307 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider.StringTypeMapping; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import oracle.jdbc.OracleTypes; +import java.math.BigDecimal; +import java.time.OffsetDateTime; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Helper class for field operations. + */ +public class OracleVectorStoreFieldHelper { + private static final Logger LOGGER = Logger.getLogger(OracleVectorStoreQueryProvider.class.getName()); + + /** + * Maps supported key java classes to Oracle database types + */ + private static final HashMap, String> supportedKeyTypes = new HashMap() { + { + put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, 255)); + put(short.class, OracleDataTypesMapping.SHORT); + put(Short.class, OracleDataTypesMapping.SHORT); + put(int.class, OracleDataTypesMapping.INTEGER); + put(Integer.class, OracleDataTypesMapping.INTEGER); + put(long.class, OracleDataTypesMapping.LONG); + put(Long.class, OracleDataTypesMapping.LONG); + put(UUID .class, OracleDataTypesMapping.UUID); + } + }; + + /** + * Maps supported vector java classes to Oracle database types + */ + private static final Map, String> supportedVectorTypes = new HashMap() { + { + put(String.class, OracleDataTypesMapping.VECTOR_FLOAT); + put(List.class, OracleDataTypesMapping.VECTOR_FLOAT); + put(Collection.class, OracleDataTypesMapping.VECTOR_FLOAT); + put(float[].class, OracleDataTypesMapping.VECTOR_FLOAT); + put(Float[].class, OracleDataTypesMapping.VECTOR_FLOAT); +/* + put(byte[].class,"VECTOR(%s, INT8)"); + put(Byte[].class,"VECTOR(%s, INT8)"); + put(double[].class,"VECTOR(%s, FLOAT64)"); + put(Double[].class,"VECTOR(%s, FLOAT64)"); + put(boolean[].class,"VECTOR(%s, BINARY)"); + put(Boolean[].class,"VECTOR(%s, BINARY)"); + */ + } + }; + + /** + * Maps supported data java classes to Oracle database types + */ + private static final HashMap, String> supportedDataTypes = new HashMap() { + { + put(byte.class, OracleDataTypesMapping.BYTE); + put(Byte.class, OracleDataTypesMapping.BYTE); + put(short.class, OracleDataTypesMapping.SHORT); + put(Short.class, OracleDataTypesMapping.SHORT); + put(int.class, OracleDataTypesMapping.INTEGER); + put(Integer.class, OracleDataTypesMapping.INTEGER); + put(long.class, OracleDataTypesMapping.LONG); + put(Long.class, OracleDataTypesMapping.LONG); + put(Float.class, OracleDataTypesMapping.FLOAT); + put(float.class, OracleDataTypesMapping.FLOAT); + put(Double.class, OracleDataTypesMapping.DOUBLE); + put(double.class, OracleDataTypesMapping.DOUBLE); + put(BigDecimal.class, OracleDataTypesMapping.DECIMAL); + put(Boolean.class, OracleDataTypesMapping.BOOLEAN); + put(boolean.class, OracleDataTypesMapping.BOOLEAN); + put(OffsetDateTime.class, OracleDataTypesMapping.OFFSET_DATE_TIME); + put(UUID.class, OracleDataTypesMapping.UUID); + put(byte[].class, OracleDataTypesMapping.BYTE_ARRAY); + put(List.class, OracleDataTypesMapping.JSON); + } + + }; + + /** + * Maps vector type to OracleTypes. Only needed if types other than FLOAT_32 are supported. + */ + private static final Map, Integer> mapOracleTypeToVector = new HashMap() { + { + put(float[].class, OracleTypes.VECTOR_FLOAT32); + put(Float[].class, OracleTypes.VECTOR_FLOAT32); +/* + put(byte[].class, OracleTypes.VECTOR_INT8); + put(Byte[].class, OracleTypes.VECTOR_INT8); + put(Double[].class, OracleTypes.VECTOR_FLOAT64); + put(double[].class, OracleTypes.VECTOR_FLOAT64); + put(Boolean[].class, OracleTypes.VECTOR_BINARY); + put(boolean[].class, OracleTypes.VECTOR_BINARY); +*/ + } + }; + + /** + * Gets the mapping between the supported Java key types and the Oracle database type. + * + * @return the mapping between the supported Java key types and the Oracle database type. + */ + public static HashMap, String> getSupportedKeyTypes() { + return supportedKeyTypes; + } + + /** + * Gets the mapping between the supported Java data types and the Oracle database type. + * + * @return the mapping between the supported Java data types and the Oracle database type. + */ + public static Map, String> getSupportedDataTypes( + StringTypeMapping stringTypeMapping, int defaultVarCharLength) { + + if (stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR)) { + supportedDataTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, defaultVarCharLength)); + } else { + supportedDataTypes.put(String.class, OracleDataTypesMapping.STRING_CLOB); + } + return supportedDataTypes; + } + + /** + * Gets the mapping between the supported Java data types and the Oracle database type. + * + * @return the mapping between the supported Java data types and the Oracle database type. + */ + public static Map, String> getSupportedVectorTypes() { + return supportedVectorTypes; + } + + /** + * Generates the statement to create the index according to the vector field definition. + * + * @return the CREATE VECTOR INDEX statement to create the index according to the vector + * field definition. + */ + public static String getCreateVectorIndexStatement(VectorStoreRecordVectorField field, String collectionTableName) { + switch (field.getIndexKind()) { + case IVFFLAT: + return "CREATE VECTOR INDEX IF NOT EXISTS " + + getIndexName(field.getEffectiveStorageName()) + + " ON " + + collectionTableName + "( " + field.getEffectiveStorageName() + " ) " + + " ORGANIZATION NEIGHBOR PARTITIONS " + + " WITH DISTANCE COSINE " + + "PARAMETERS ( TYPE IVF )"; + case HNSW: + return "CREATE VECTOR INDEX IF NOT EXISTS " + getIndexName(field.getEffectiveStorageName()) + + " ON " + + collectionTableName + "( " + field.getEffectiveStorageName() + " ) " + + "ORGANIZATION INMEMORY GRAPH " + + "WITH DISTANCE COSINE " + + "PARAMETERS (TYPE HNSW)"; + case UNDEFINED: + return null; + default: + LOGGER.warning("Unsupported index kind: " + field.getIndexKind()); + return null; + } + } + + /** + * Generates the statement to create the index according to the field definition. + * + * @return the CREATE INDEX statement to create the index according to the field definition. + */ + public static String createIndexForDataField(String collectionTableName, VectorStoreRecordDataField dataField, Map, String> supportedDataTypes) { + if (supportedDataTypes.get(dataField.getFieldType()) == "JSON") { + String dataFieldIndex = "CREATE MULTIVALUE INDEX %s ON %s t (t.%s.%s)"; + return String.format(dataFieldIndex, + collectionTableName + "_" + dataField.getEffectiveStorageName(), + collectionTableName, + dataField.getEffectiveStorageName(), + getFunctionForType(supportedDataTypes.get(dataField.getFieldSubType()))); + } else { + String dataFieldIndex = "CREATE INDEX %s ON %s (%s ASC)"; + return String.format(dataFieldIndex, + collectionTableName + "_" + dataField.getEffectiveStorageName(), + collectionTableName, + dataField.getEffectiveStorageName() + ); + } + } + + /** + * Gets the function that allows to return the function that converts the JSON value to the + * data type. + * @param jdbcType The JDBC type. + * @return the function that allows to return the function that converts the JSON value to the + * data type. + */ + private static String getFunctionForType(String jdbcType) { + switch (jdbcType) { + case OracleDataTypesMapping.BOOLEAN: + return "boolean()"; + case OracleDataTypesMapping.BYTE: + case OracleDataTypesMapping.SHORT: + case OracleDataTypesMapping.INTEGER: + case OracleDataTypesMapping.LONG: + case OracleDataTypesMapping.FLOAT: + case OracleDataTypesMapping.DOUBLE: + case OracleDataTypesMapping.DECIMAL: + return "numberOnly()"; + case OracleDataTypesMapping.OFFSET_DATE_TIME: + return "timestamp()"; + default: + return "string()"; + } + } + + /** + * Gets the type of the vector given the field definition. This method is not needed if only + * + * @param field the vector field definition. + * @return returns the type of vector for the given field type. + */ + public static String getTypeForVectorField(VectorStoreRecordVectorField field) { + String dimension = field.getDimensions() > 0 ? String.valueOf(field.getDimensions()) : "*"; + return String.format(supportedVectorTypes.get(field.getFieldType()), dimension); +/* Not needed since all types are FLOAT32 + if (field.getFieldSubType() != null) { + String vectorType; + switch (field.getFieldSubType().getName()) { + case "java.lang.Double": + vectorType = "FLOAT64"; + break; + case "java.lang.Byte": + vectorType = "INT8"; + break; + case "java.lang.Boolean": + vectorType = "BINARY"; + break; + default: + vectorType = "FLOAT32"; + } + return String.format(supportedVectorTypes.get(field.getFieldType()), dimension, vectorType); + } else { + return String.format(supportedVectorTypes.get(field.getFieldType()), dimension); + } + */ + } + + /** + * Gets the JDBC oracle of the vector field definition. + * @param field the vector field definition. + * @return the JDBC oracle type. + */ + public static int getOracleTypeForField(VectorStoreRecordVectorField field) { + if (field.getFieldSubType() == null) { + return mapOracleTypeToVector.get(field.getFieldType()).intValue(); + } else { + switch (field.getFieldSubType().getName()) { + case "java.lang.Double": + return OracleTypes.VECTOR_FLOAT64; + case "java.lang.Byte": + return OracleTypes.VECTOR_INT8; + case "java.lang.Boolean": + return OracleTypes.VECTOR_BINARY; + default: + return OracleTypes.VECTOR_FLOAT32; + } + } + } + + /** + * Generates the index name given the field name. by suffixing "_VECTOR_INDEX" to the field name. + * @param effectiveStorageName the field name. + * @return the index name. + */ + private static String getIndexName(String effectiveStorageName) { + return effectiveStorageName + "_VECTOR_INDEX"; + } + + /** + * Returns vector columns names and types for CREATE TABLE statement + * @param fields list of vector record fields. + * @return comma separated list of columns and types for CREATE TABLE statement. + */ + public static String getVectorColumnNamesAndTypes(List fields) { + List columns = fields.stream() + .map(field -> field.getEffectiveStorageName() + " " + + OracleVectorStoreFieldHelper.getTypeForVectorField(field) + ).collect(Collectors.toList()); + + return String.join(", ", columns); + } + + /** + * Returns key column names and type for key column for CREATE TABLE statement + * @param field the key field. + * @return column name and type of the key field for CREATE TABLE statement. + */ + public static String getKeyColumnNameAndType(VectorStoreRecordKeyField field) { + return field.getEffectiveStorageName() + " " + supportedKeyTypes.get(field.getFieldType()); + } + +} diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index 12466a6f..b32e2a82 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -15,38 +15,33 @@ import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import oracle.jdbc.OracleResultSet; import oracle.jdbc.OracleStatement; -import oracle.jdbc.OracleType; import oracle.jdbc.OracleTypes; import javax.annotation.Nonnull; +import javax.annotation.concurrent.GuardedBy; import javax.sql.DataSource; -import java.math.BigDecimal; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.Statement; import java.sql.SQLException; -import java.time.OffsetDateTime; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.UUID; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.StreamSupport; +/** + * JDBC Vector Store for the Oracle Database + */ public class OracleVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider { // This could be removed if super.collectionTable made protected @@ -57,7 +52,7 @@ public class OracleVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider private static final Object dbCreationLock = new Object(); - Logger logger = Logger.getLogger(OracleVectorStoreQueryProvider.class.getName()); + private static final Logger logger = Logger.getLogger(OracleVectorStoreQueryProvider.class.getName()); public enum StringTypeMapping { /** @@ -70,6 +65,15 @@ public enum StringTypeMapping { USE_VARCHAR } + /** + * Constructor + * @param dataSource the datasiyrce + * @param collectionsTable the collections table name + * @param prefixForCollectionTables the prefix for the collection table name + * @param defaultVarcharSize the size of VARCHAR columns + * @param stringTypeMapping the storage type of string columns (VARCHAR or CLOB) + * @param objectMapper the object mapper. + */ private OracleVectorStoreQueryProvider( @Nonnull DataSource dataSource, @Nonnull String collectionsTable, @@ -81,113 +85,16 @@ private OracleVectorStoreQueryProvider( dataSource, collectionsTable, prefixForCollectionTables, - buildSupportedKeyTypes(), - buildSupportedDataTypes(stringTypeMapping, defaultVarcharSize), - buildSupportedVectorTypes()); + OracleVectorStoreFieldHelper.getSupportedKeyTypes(), + OracleVectorStoreFieldHelper.getSupportedDataTypes(stringTypeMapping, defaultVarcharSize), + OracleVectorStoreFieldHelper.getSupportedVectorTypes()); this.collectionsTable = collectionsTable; this.objectMapper = objectMapper; } - private static HashMap, String> buildSupportedKeyTypes() { - HashMap, String> supportedKeyTypes = new HashMap<>(); - supportedKeyTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, 255)); - supportedKeyTypes.put(short.class, OracleDataTypesMapping.SHORT); - supportedKeyTypes.put(Short.class, OracleDataTypesMapping.SHORT); - supportedKeyTypes.put(int.class, OracleDataTypesMapping.INTEGER); - supportedKeyTypes.put(Integer.class, OracleDataTypesMapping.INTEGER); - supportedKeyTypes.put(long.class, OracleDataTypesMapping.LONG); - supportedKeyTypes.put(Long.class, OracleDataTypesMapping.LONG); - supportedKeyTypes.put(UUID.class, OracleDataTypesMapping.UUID); - - return supportedKeyTypes; - } - private static Map, String> buildSupportedVectorTypes() { - HashMap, String> supportedVectorTypes = new HashMap<>(); - supportedVectorTypes.put(String.class, "VECTOR(%s)"); - supportedVectorTypes.put(List.class, "VECTOR(%s)"); - supportedVectorTypes.put(Collection.class, "VECTOR(%s)"); - return supportedVectorTypes; - } - - private static Map, String> buildSupportedDataTypes(StringTypeMapping stringTypeMapping, int defaultVarCharLength) { - HashMap, String> supportedDataTypes = new HashMap<>(); - if (stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR)) { - supportedDataTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, defaultVarCharLength)); - } else { - supportedDataTypes.put(String.class, OracleDataTypesMapping.STRING_CLOB); - } - supportedDataTypes.put(byte.class, OracleDataTypesMapping.BYTE); - supportedDataTypes.put(Byte.class, OracleDataTypesMapping.BYTE); - supportedDataTypes.put(short.class, OracleDataTypesMapping.SHORT); - supportedDataTypes.put(Short.class, OracleDataTypesMapping.SHORT); - supportedDataTypes.put(int.class, OracleDataTypesMapping.INTEGER); - supportedDataTypes.put(Integer.class, OracleDataTypesMapping.INTEGER); - supportedDataTypes.put(long.class, OracleDataTypesMapping.LONG); - supportedDataTypes.put(Long.class, OracleDataTypesMapping.LONG); - supportedDataTypes.put(Float.class, OracleDataTypesMapping.FLOAT); - supportedDataTypes.put(float.class, OracleDataTypesMapping.FLOAT); - supportedDataTypes.put(Double.class, OracleDataTypesMapping.DOUBLE); - supportedDataTypes.put(double.class, OracleDataTypesMapping.DOUBLE); - supportedDataTypes.put(BigDecimal.class, OracleDataTypesMapping.DECIMAL); - supportedDataTypes.put(Boolean.class, OracleDataTypesMapping.BOOLEAN); - supportedDataTypes.put(boolean.class, OracleDataTypesMapping.BOOLEAN); - supportedDataTypes.put(OffsetDateTime.class, OracleDataTypesMapping.OFFSET_DATE_TIME); - supportedDataTypes.put(UUID.class, OracleDataTypesMapping.UUID); - supportedDataTypes.put(byte[].class, OracleDataTypesMapping.BYTE_ARRAY); - supportedDataTypes.put(List.class, OracleDataTypesMapping.JSON); - return supportedDataTypes; - } - - private String createIndexForVectorField(String collectionName, VectorStoreRecordVectorField vectorField) { - switch (vectorField.getIndexKind()) { - case IVFFLAT: - return "CREATE VECTOR INDEX IF NOT EXISTS " - + getIndexName(vectorField.getEffectiveStorageName()) - + " ON " - + getCollectionTableName(collectionName) + "( " + vectorField.getEffectiveStorageName() + " ) " - + " ORGANIZATION NEIGHBOR PARTITIONS " - + " WITH DISTANCE COSINE " - + "PARAMETERS ( TYPE IVF )"; - case HNSW: - return "CREATE VECTOR INDEX IF NOT EXISTS " + getIndexName(vectorField.getEffectiveStorageName()) - + " ON " - + getCollectionTableName(collectionName) + "( " + vectorField.getEffectiveStorageName() + " ) " - + "ORGANIZATION INMEMORY GRAPH " - + "WITH DISTANCE COSINE " - + "PARAMETERS (TYPE HNSW)"; - case UNDEFINED: - return null; - default: - logger.warning("Unsupported index kind: " + vectorField.getIndexKind()); - return null; - } - } - - private String getIndexName(String effectiveStorageName) { - return effectiveStorageName + "_VECTOR_INDEX"; - } - - - protected String getVectorColumnNamesAndTypes(List fields, - Map, String> types) { - List columns = fields.stream() - .map(field -> validateSQLidentifier(field.getEffectiveStorageName()) + " " - + String.format(types.get(field.getFieldType()), field.getDimensions() > 0 ? field.getDimensions() + ", FLOAT32" : "FLOAT32")) - .collect(Collectors.toList()); - - return String.join(", ", columns); - } - - @Override - protected String getInsertCollectionQuery(String collectionsTable) { - return formatQuery( - "MERGE INTO %s existing "+ - "USING (SELECT ? AS collectionId FROM DUAL) new ON (existing.collectionId = new.collectionId) " + - "WHEN NOT MATCHED THEN INSERT (existing.collectionId) VALUES (new.collectionId)", - collectionsTable); - } - @Override + @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") + @GuardedBy("dbCreationLock") public void createCollection(String collectionName, VectorStoreRecordDefinition recordDefinition) { @@ -199,11 +106,11 @@ public void createCollection(String collectionName, + "%s, " + "%s)", getCollectionTableName(collectionName), - getKeyColumnNameAndType(recordDefinition.getKeyField(), getSupportedDataTypes()), + OracleVectorStoreFieldHelper.getKeyColumnNameAndType(recordDefinition.getKeyField()), getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), getSupportedDataTypes()), - getVectorColumnNamesAndTypes(new ArrayList<>(vectorFields), - getSupportedVectorTypes())); + OracleVectorStoreFieldHelper.getVectorColumnNamesAndTypes( + new ArrayList<>(vectorFields))); String insertCollectionQuery = this.getInsertCollectionQuery(collectionsTable); @@ -213,28 +120,28 @@ public void createCollection(String collectionName, try (Statement statement = connection.createStatement()) { // Create table System.out.println(createStorageTable); - statement.addBatch(createStorageTable); + statement.execute(createStorageTable); // Index filterable columns for (VectorStoreRecordDataField dataField : recordDefinition.getDataFields()) { if (dataField.isFilterable()) { - String dataFieldIndex = createIndexForDataField(collectionName, dataField); + String dataFieldIndex = OracleVectorStoreFieldHelper.createIndexForDataField( + getCollectionTableName(collectionName), dataField, supportedDataTypes); System.out.println(dataFieldIndex); - statement.addBatch(dataFieldIndex); + statement.execute(dataFieldIndex); } } // Create indexed for vectorFields for (VectorStoreRecordVectorField vectorField : vectorFields) { - String createVectorIndex = createIndexForVectorField(collectionName, - vectorField); - + String createVectorIndex = OracleVectorStoreFieldHelper.getCreateVectorIndexStatement( + vectorField, getCollectionTableName(collectionName)); if (createVectorIndex != null) { System.out.println(createVectorIndex); - statement.addBatch(createVectorIndex); + statement.execute(createVectorIndex); } } - statement.executeBatch(); + //statement.executeBatch(); try (PreparedStatement insert = connection.prepareStatement( insertCollectionQuery)) { @@ -254,60 +161,48 @@ public void createCollection(String collectionName, } } - private String getKeyColumnNameAndType(VectorStoreRecordKeyField field, Map, String> types) { - return validateSQLidentifier(field.getEffectiveStorageName()) + " " - + types.get(field.getFieldType()); + @Override + protected String getInsertCollectionQuery(String collectionsTable) { + return formatQuery( + "MERGE INTO %s existing "+ + "USING (SELECT ? AS collectionId FROM DUAL) new ON (existing.collectionId = new.collectionId) " + + "WHEN NOT MATCHED THEN INSERT (existing.collectionId) VALUES (new.collectionId)", + collectionsTable); } - private String createIndexForDataField(String collectionName, VectorStoreRecordDataField dataField) { - if (supportedDataTypes.get(dataField.getFieldType()) == "JSON") { - String dataFieldIndex = "CREATE MULTIVALUE INDEX %s ON %s t (t.%s.%s)"; - return formatQuery(dataFieldIndex, - getCollectionTableName(collectionName) + "_" + dataField.getEffectiveStorageName(), - getCollectionTableName(collectionName), - dataField.getEffectiveStorageName(), - getFunctionForType(supportedDataTypes.get(dataField.getFieldSubType()))); - } else { - String dataFieldIndex = "CREATE INDEX %s ON %s (%s ASC)"; - return formatQuery(dataFieldIndex, - getCollectionTableName(collectionName) + "_" + dataField.getEffectiveStorageName(), - getCollectionTableName(collectionName), - dataField.getEffectiveStorageName() - ); - } + @Override + public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { - } + final String NEW_VALUE = "new"; + final String EXISTING_VALUE = "existing"; - private String getFunctionForType(String jdbcType) { - switch (jdbcType) { - case "BOOLEAN": - return "boolean()"; - case "INTEGER": - case "LONG": - case "REAL": - case "DOUBLE PRECISION": - return "numberOnly()"; - case "TIMESTAMPTZ": - return "timestamp()"; - default: - return "string()"; - } - } + String insertNewFieldList = recordDefinition.getAllFields().stream() + .map(f -> NEW_VALUE + "." + f.getEffectiveStorageName()) + .collect(Collectors.joining(", ")); - @Override - public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { + String insertExistingFieldList = recordDefinition.getAllFields().stream() + .map(f -> EXISTING_VALUE + "." + f.getEffectiveStorageName()) + .collect(Collectors.joining(", ")); + + String updateFieldList = recordDefinition.getAllFields().stream() + .filter(f -> f != recordDefinition.getKeyField()) + .map(f -> EXISTING_VALUE + "." + f.getEffectiveStorageName() + " = " + NEW_VALUE + "." + f.getEffectiveStorageName()) + .collect(Collectors.joining(", ")); + + String namedWildcard = recordDefinition.getAllFields().stream().map(f -> "? " + f.getEffectiveStorageName()) + .collect(Collectors.joining(", ")); String upsertQuery = formatQuery("MERGE INTO %s existing "+ "USING (SELECT %s FROM DUAL) new ON (existing.%s = new.%s) " + "WHEN MATCHED THEN UPDATE SET %s " + "WHEN NOT MATCHED THEN INSERT (%s) VALUES (%s)", getCollectionTableName(collectionName), - getNamedWildcard(recordDefinition.getAllFields()), + namedWildcard, getKeyColumnName(recordDefinition.getKeyField()), getKeyColumnName(recordDefinition.getKeyField()), - getUpdateFieldList(recordDefinition.getKeyField(), recordDefinition.getAllFields(), "existing", "new"), - getInsertFieldList(recordDefinition.getKeyField(), recordDefinition.getAllFields(), "existing"), - getInsertFieldList(recordDefinition.getKeyField(), recordDefinition.getAllFields(), "new")); + updateFieldList, + insertExistingFieldList, + insertNewFieldList); System.out.println(upsertQuery); try (Connection connection = dataSource.getConnection(); @@ -335,8 +230,13 @@ private void setUpsertStatementValues(PreparedStatement statement, Object record if (field instanceof VectorStoreRecordVectorField) { // Convert the vector field to a string if (!field.getFieldType().equals(String.class)) { - double[] values = valueNode == null ? null : StreamSupport.stream(((ArrayNode)valueNode).spliterator(), false).mapToDouble(d -> d.asDouble()).toArray(); - statement.setObject(i + 1, values, OracleType.VECTOR_FLOAT64); + double[] values = valueNode.isNull() + ? null + : StreamSupport.stream(( + (ArrayNode)valueNode).spliterator(), false) + .mapToDouble(d -> d.asDouble()).toArray(); + statement.setObject(i + 1, values, + OracleVectorStoreFieldHelper.getOracleTypeForField((VectorStoreRecordVectorField)field)); System.out.println("Set values: " + values); continue; } @@ -357,23 +257,6 @@ private void setUpsertStatementValues(PreparedStatement statement, Object record } } } - private String getInsertFieldList(VectorStoreRecordKeyField key, List fields, String alias) { - return fields.stream().map(f -> alias + "." + f.getEffectiveStorageName()) - .collect(Collectors.joining(", ")); - } - - private String getUpdateFieldList(VectorStoreRecordKeyField key, List fields, String oldAlias, String newAlias) { - return fields.stream().filter(f -> f != key).map(f -> oldAlias + "." + f.getEffectiveStorageName() + " = " + - newAlias + "." + f.getEffectiveStorageName()) - .collect(Collectors.joining(", ")); - - } - - - private String getNamedWildcard(List fields) { - return fields.stream().map(f -> "? " + f.getEffectiveStorageName()) - .collect(Collectors.joining(", ")); - } @Override public VectorSearchResults search(String collectionName, List vector, @@ -430,10 +313,12 @@ public VectorSearchResults search(String collectionName, List VectorSearchResults search(String collectionName, List(records); } + private void defineDataColumnType(int columnIndex, OracleStatement statement, Class fieldType) throws SQLException { // swich between supported classes and define the column type on the statement switch (supportedDataTypes.get(fieldType)) { - case "CLOB": + case OracleDataTypesMapping.STRING_CLOB: statement.defineColumnType(columnIndex, OracleTypes.CLOB, Integer.MAX_VALUE); break; - case "INTEGER": + case OracleDataTypesMapping.BYTE: + statement.defineColumnType(columnIndex, OracleTypes.NUMBER); + break; + case OracleDataTypesMapping.SHORT: + statement.defineColumnType(columnIndex, OracleTypes.NUMBER); + break; + case OracleDataTypesMapping.INTEGER: statement.defineColumnType(columnIndex, OracleTypes.INTEGER); break; - case "LONG": + case OracleDataTypesMapping.LONG: statement.defineColumnType(columnIndex, OracleTypes.BIGINT); break; - case "REAL": - statement.defineColumnType(columnIndex, OracleTypes.REAL); + case OracleDataTypesMapping.FLOAT: + statement.defineColumnType(columnIndex, OracleTypes.BINARY_FLOAT); break; - case "DOUBLE PRECISION": + case OracleDataTypesMapping.DOUBLE: statement.defineColumnType(columnIndex, OracleTypes.BINARY_DOUBLE); break; - case "BOOLEAN": + case OracleDataTypesMapping.DECIMAL: + statement.defineColumnType(columnIndex, OracleTypes.BINARY_DOUBLE); + break; + case OracleDataTypesMapping.BOOLEAN: statement.defineColumnType(columnIndex, OracleTypes.BOOLEAN); break; - case "TIMESTAMPTZ": + case OracleDataTypesMapping.OFFSET_DATE_TIME: statement.defineColumnType(columnIndex, OracleTypes.TIMESTAMPTZ); break; - case "JSON": + case OracleDataTypesMapping.JSON: statement.defineColumnType(columnIndex, OracleTypes.JSON, Integer.MAX_VALUE); break; + case OracleDataTypesMapping.UUID: + case OracleDataTypesMapping.BYTE_ARRAY: + statement.defineColumnType(columnIndex, OracleTypes.RAW); default: statement.defineColumnType(columnIndex, OracleTypes.VARCHAR); } @@ -542,11 +438,6 @@ public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { return String.format("JSON_EXISTS(%s, '$[*]?(@ == $v_%s)' PASSING ? AS \"v_%s\")", fieldName, fieldName, fieldName); } - - - public static Builder builder() { - return new Builder(); - } @Override public VectorStoreRecordMapper getVectorStoreRecordMapper( @@ -559,6 +450,10 @@ public VectorStoreRecordMapper getVectorStoreRecordM .build(); } + public static Builder builder() { + return new Builder(); + } + public static class Builder extends JDBCVectorStoreQueryProvider.Builder { diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java index 197cb8c3..d1c848a6 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java @@ -40,15 +40,15 @@ public class Hotel { @JsonProperty("summaryEmbedding2") @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.HNSW) - private final List cosineDistance; + private final float[] cosineDistance; @JsonProperty("summaryEmbedding3") @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_SIMILARITY, indexKind = IndexKind.IVFFLAT) - private final List cosineSimilarity; + private final float[] cosineSimilarity; @JsonProperty("summaryEmbedding4") @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.DOT_PRODUCT, indexKind = IndexKind.IVFFLAT) - private final List dotProduct; + private final Float[] dotProduct; @VectorStoreRecordData private double rating; @@ -66,9 +66,9 @@ protected Hotel( @JsonProperty("tags") List tags, @JsonProperty("summary") String description, @JsonProperty("summaryEmbedding1") List euclidean, - @JsonProperty("summaryEmbedding2") List cosineDistance, - @JsonProperty("summaryEmbedding3") List cosineSimilarity, - @JsonProperty("summaryEmbedding4") List dotProduct, + @JsonProperty("summaryEmbedding2") float[] cosineDistance, + @JsonProperty("summaryEmbedding3") float[] cosineSimilarity, + @JsonProperty("summaryEmbedding4") Float[] dotProduct, @JsonProperty("rating") double rating) { this.id = id; this.name = name; @@ -77,9 +77,9 @@ protected Hotel( this.tags = tags; this.description = description; this.euclidean = euclidean; - this.cosineDistance = euclidean; - this.cosineSimilarity = euclidean; - this.dotProduct = euclidean; + this.cosineDistance = cosineDistance; + this.cosineSimilarity = cosineSimilarity; + this.dotProduct = dotProduct; this.rating = rating; } @@ -107,11 +107,11 @@ public List getEuclidean() { return euclidean; } - public List getCosineDistance() { + public float[] getCosineDistance() { return cosineDistance; } - public List getDotProduct() { + public Float[] getDotProduct() { return dotProduct; } diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 72ed3524..22d0259f 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -128,19 +128,34 @@ public void clearCollection() { private static List getHotels() { return Arrays.asList( new Hotel("id_1", "Hotel 1", 1, 1.49d, Arrays.asList("one", "two"), "Hotel 1 description", - Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), null, null, null, + Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), + new float[] {0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f}, + new float[] {0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f}, + new Float[] {0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f}, 4.0), new Hotel("id_2", "Hotel 2", 2, 1.44d, Arrays.asList("three", "four"), "Hotel 2 description with free-text search", - Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), null, null, null, + Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), + new float[] {-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f}, + new float[] {-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f}, + new Float[] {-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f}, 4.0), new Hotel("id_3", "Hotel 3", 3, 1.53d, Arrays.asList("five", "six"), "Hotel 3 description", - Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), null, null, null, + Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), + new float[] {4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f}, + new float[] {4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f}, + new Float[] {4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f}, 5.0), new Hotel("id_4", "Hotel 4", 4, 1.35d, Arrays.asList("seven", "eight"), "Hotel 4 description", - Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), null, null, null, + Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), + new float[] {7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f}, + new float[] {7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f}, + new Float[] {7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f}, 4.0), new Hotel("id_5", "Hotel 5", 5, 1.89d, Arrays.asList("nine", "ten"),"Hotel 5 description", - Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), null, null, null, + Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), + new float[] {-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f}, + new float[] {-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f}, + new Float[] {-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f}, 4.0)); } diff --git a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java index 8f8d6212..e769bb6f 100644 --- a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java +++ b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java @@ -210,7 +210,7 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) vectorFields.add(VectorStoreRecordVectorField.builder() .withName(field.getName()) .withStorageName(storageName) - .withFieldType(field.getType()) + .withFieldType(field.getType(), List.class.equals(field.getType()) ? (Class)((ParameterizedType) field.getGenericType()).getActualTypeArguments()[0] : null) .withDimensions(vectorAttribute.dimensions()) .withIndexKind(vectorAttribute.indexKind()) .withDistanceFunction(vectorAttribute.distanceFunction()) diff --git a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java index 00b7627a..b708d2fb 100644 --- a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java +++ b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java @@ -34,10 +34,11 @@ public VectorStoreRecordVectorField( @Nonnull String name, @Nullable String storageName, @Nonnull Class fieldType, + Class fieldSubType, int dimensions, @Nullable IndexKind indexKind, @Nullable DistanceFunction distanceFunction) { - super(name, storageName, fieldType); + super(name, storageName, fieldType, fieldSubType); this.dimensions = dimensions; this.indexKind = indexKind == null ? IndexKind.UNDEFINED : indexKind; this.distanceFunction = distanceFunction == null ? DistanceFunction.UNDEFINED @@ -130,7 +131,8 @@ public VectorStoreRecordVectorField build() { throw new IllegalArgumentException("dimensions must be greater than 0"); } - return new VectorStoreRecordVectorField(name, storageName, fieldType, dimensions, + return new VectorStoreRecordVectorField(name, storageName, fieldType, fieldSubType, + dimensions, indexKind, distanceFunction); } From 434b9e66918db86f0f2333aa140a6ec612a05d2d Mon Sep 17 00:00:00 2001 From: Kaiyuan Li Date: Tue, 17 Jun 2025 23:17:44 -0700 Subject: [PATCH 184/277] Add test for supported data type(timestamp and uuid not included) --- ...OracleVectorStoreRecordCollectionTest.java | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index e71a9655..d1505f7e 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -29,20 +29,25 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.time.Duration; +import java.time.OffsetDateTime; import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -407,6 +412,87 @@ void testKeyTypes(String suffix, Class keyType, Object keyValue) { collection.deleteCollectionAsync().block(); } + @ParameterizedTest + @MethodSource("supportedDataTypes") + void testDataTypes(String dataFieldName, Class dataFieldType, Object dataFieldValue, Class fieldSubType) { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + VectorStoreRecordDataField dataField; + if (fieldSubType != null) { + dataField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(dataFieldType, fieldSubType) + .isFilterable(true) + .build(); + } else { + dataField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(dataFieldType) + .isFilterable(true) + .build(); + } + + VectorStoreRecordVectorField dummyVector = VectorStoreRecordVectorField.builder() + .withName("vec") + .withStorageName("vec") + .withFieldType(List.class) + .withDimensions(2) + .withDistanceFunction(DistanceFunction.EUCLIDEAN_DISTANCE) + .withIndexKind(IndexKind.UNDEFINED) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dataField, dummyVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "test_datatype_" + dataFieldName; + + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions. builder() + .withRecordClass(DummyRecordForDataTypes.class) + .withRecordDefinition(definition).build()); + + collection.createCollectionAsync().block(); + + String key = "testid"; + + DummyRecordForDataTypes record = + new DummyRecordForDataTypes(key, dataFieldValue, Arrays.asList(1.0f, 2.0f)); + + collection.upsertAsync(record, null).block(); + + DummyRecordForDataTypes result = collection.getAsync(key, null).block(); + assertNotNull(result); + + if (dataFieldValue instanceof Number && result.getDummy() instanceof Number) { + assertEquals(((Number) dataFieldValue).doubleValue(), ((Number) result.getDummy()).doubleValue()); + } else if (dataFieldValue instanceof byte[]) { + assertArrayEquals((byte[]) dataFieldValue, (byte[]) result.getDummy()); + } else { + assertEquals(dataFieldValue, result.getDummy()); + } + + collection.deleteCollectionAsync().block(); + } + @Nested class HNSWIndexTests { @Test @@ -592,6 +678,25 @@ private static Stream supportedKeyTypes() { ); } + private static Stream supportedDataTypes() { + return Stream.of( + Arguments.of("string", String.class, "asd123", null), + Arguments.of("boolean_true", Boolean.class, true, null), + Arguments.of("boolean_false", Boolean.class, false, null), + Arguments.of("byte", Byte.class, (byte) 127, null), + Arguments.of("short", Short.class, (short) 3, null), + Arguments.of("integer", Integer.class, 321, null), + Arguments.of("long", Long.class, 5L, null), + Arguments.of("float", Float.class, 3.14f, null), + Arguments.of("double", double.class, 3.14159265358d, null), + Arguments.of("decimal", BigDecimal.class, new BigDecimal("12345.67"), null), + //Arguments.of("timestamp", OffsetDateTime.class, OffsetDateTime.now(), null) + //Arguments.of("uuid", UUID.class, UUID.randomUUID(), null) + Arguments.of("byte_array", byte[].class, "abc".getBytes(StandardCharsets.UTF_8), null), + Arguments.of("json", List.class, Arrays.asList("a", "s", "d"), String.class) + ); + } + private static class DummyRecordForKeyTypes { private final Object id; private final String dummy; @@ -619,4 +724,32 @@ public String toString() { return String.valueOf(id); } } + + private static class DummyRecordForDataTypes { + private final String id; + private final Object dummy; + private final List vec; + @JsonCreator + public DummyRecordForDataTypes( + @JsonProperty("id") String id, + @JsonProperty("dummy") Object dummy, + @JsonProperty("vec") List vec) { + this.id = id; + this.dummy = dummy; + this.vec = vec; + } + + public String getId() { + return id; + } + + public Object getDummy() { + return dummy; + } + + @Override + public String toString() { + return String.valueOf(id); + } + } } From 1c83d62a1f8a5da0dad0eca235ecb53f7b1ea6a3 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Fri, 20 Jun 2025 18:46:42 +0200 Subject: [PATCH 185/277] Test and bug fixes --- .../OracleVectorStoreQueryProvider.java | 83 ++++- .../OracleVectorStoreDataTypeSearchTest.java | 297 ++++++++++++++++++ .../oracle/OracleVectorStoreDataTypeTest.java | 1 + 3 files changed, 375 insertions(+), 6 deletions(-) create mode 100644 data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index 3c3e4fcc..f7a32fd5 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -34,6 +34,7 @@ import javax.annotation.Nonnull; import javax.annotation.concurrent.GuardedBy; import javax.sql.DataSource; +import java.math.BigDecimal; import java.nio.ByteBuffer; import java.sql.Connection; import java.sql.PreparedStatement; @@ -331,7 +332,8 @@ public VectorSearchResults search(String collectionName, List parameters = getFilterParameters(options.getVectorSearchFilter()); String selectQuery = "SELECT " - + formatQuery("VECTOR_DISTANCE(%s, ?, %s) distance, ", vectorField.getEffectiveStorageName(), toOracleDistanceFunction(distanceFunction)) + + (vector == null ? "0 as distance, " : + formatQuery("VECTOR_DISTANCE(%s, ?, %s) distance, ", vectorField.getEffectiveStorageName(), toOracleDistanceFunction(distanceFunction))) + getQueryColumnsFromFields(fields) + " FROM " + getCollectionTableName(collectionName) + (filter != null && !filter.isEmpty() ? " WHERE " + filter : "") @@ -345,12 +347,16 @@ public VectorSearchResults search(String collectionName, List VectorSearchResults search(String collectionName, List(records); } + private void setSearchParameter(PreparedStatement statement, int index, Class type, Object value) { + + try { + if (List.class.equals(type)) { + statement.setObject(index, objectMapper.writeValueAsString(value)); + System.out.println( + "Set values: " + objectMapper.writeValueAsString(value)); + return; + } + if (UUID.class.equals(type)) { + if (value == null) { + statement.setNull(index, OracleTypes.RAW); + } else { + UUID uuid = (UUID)value; + ByteBuffer bb = ByteBuffer.allocate(16); + bb.putLong(uuid.getMostSignificantBits()); + bb.putLong(uuid.getLeastSignificantBits()); + statement.setBytes(index, bb.array()); + System.out.println("Set values: " + uuid); + } + return; + } + if (OffsetDateTime.class.equals(type)) { + if (value == null) { + statement.setNull(index, OracleTypes.TIMESTAMPTZ); + } else { + OffsetDateTime offsetDateTime = (OffsetDateTime) value; + ((OraclePreparedStatement) statement).setTIMESTAMPTZ(index, + TIMESTAMPTZ.of(offsetDateTime)); + System.out.println("Set values: " + offsetDateTime); + } + return; + } + if (BigDecimal.class.equals(type)) { + if (value == null) { + statement.setNull(index, OracleTypes.DECIMAL); + } else { + BigDecimal bigDecimal = (BigDecimal) value; + ((OraclePreparedStatement) statement).setBigDecimal(index, + bigDecimal); + System.out.println("Set values: " + bigDecimal); + } + return; + } + System.out.println("Set parameter " + index + " to: " + value); + statement.setObject(index, value); + + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + private void defineDataColumnType(int columnIndex, OracleStatement statement, Class fieldType) throws SQLException { // swich between supported classes and define the column type on the statement @@ -481,6 +539,19 @@ public List getFilterParameters(VectorSearchFilter filter) { }).collect(Collectors.toList()); } + @Override + public String getEqualToFilter(EqualToFilterClause filterClause) { + String fieldName = JDBCVectorStoreQueryProvider + .validateSQLidentifier(filterClause.getFieldName()); + Object value = filterClause.getValue(); + + if (value == null) { + return String.format("%s is NULL", fieldName); + } else { + return String.format("%s = ?", fieldName); + } + } + @Override public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { String fieldName = JDBCVectorStoreQueryProvider diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java new file mode 100644 index 00000000..4ce739a8 --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java @@ -0,0 +1,297 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.util.Arrays; +import java.util.UUID; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class OracleVectorStoreDataTypeSearchTest extends OracleCommonVectorStoreRecordCollectionTest { + private static final double MIN_NUMBER = 1.0E-130; + private static final BigDecimal BIG_NUMBER = BigDecimal.valueOf(9999999999999999.99); + + + + @ParameterizedTest + @MethodSource("supportedDataTypes") + void testDataTypesSearch (ClassWithAllBoxedTypes record) { + VectorStoreRecordCollection collection = setupBoxed(); + + collection.upsertAsync(record, null).block(); + + // boolean + VectorSearchResults results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("booleanValue", record.getBooleanValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getBooleanValue(), results.getResults().get(0).getRecord().getBooleanValue()); + + // byte + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("byteValue", record.getByteValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getByteValue(), results.getResults().get(0).getRecord().getByteValue()); + + // short + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("shortValue", record.getShortValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getShortValue(), results.getResults().get(0).getRecord().getShortValue()); + + // integer + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("integerValue", record.getIntegerValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getIntegerValue(), results.getResults().get(0).getRecord().getIntegerValue()); + + // long + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("longValue", record.getLongValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getLongValue(), results.getResults().get(0).getRecord().getLongValue()); + + // float + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("floatValue", record.getFloatValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getFloatValue(), results.getResults().get(0).getRecord().getFloatValue()); + + // double + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("doubleValue", record.getDoubleValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getDoubleValue(), results.getResults().get(0).getRecord().getDoubleValue()); + + // decimal + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("decimalValue", record.getDecimalValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + if (record.getDecimalValue() != null) { + assertEquals(0, record.getDecimalValue() + .compareTo(results.getResults().get(0).getRecord().getDecimalValue())); + } else { + assertEquals(record.getDecimalValue(), + results.getResults().get(0).getRecord().getDecimalValue()); + } + + // offset date time + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("offsetDateTimeValue", record.getOffsetDateTimeValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + if (record.getOffsetDateTimeValue() != null) { + assertTrue(record.getOffsetDateTimeValue() + .isEqual(results.getResults().get(0).getRecord().getOffsetDateTimeValue())); + } else { + assertEquals(record.getOffsetDateTimeValue(), + results.getResults().get(0).getRecord().getOffsetDateTimeValue()); + } + + // UUID + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("uuidValue", record.getUuidValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getUuidValue(), results.getResults().get(0).getRecord().getUuidValue()); + + // byte array + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("byteArrayValue", record.getByteArrayValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertArrayEquals(record.getByteArrayValue(), results.getResults().get(0).getRecord().getByteArrayValue()); + + collection.deleteCollectionAsync().block(); + + } + + + public VectorStoreRecordCollection setupBoxed() { + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + VectorStoreRecordCollection collection = + vectorStore.getCollection("BoxedTypes", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(ClassWithAllBoxedTypes.class) + .build()).createCollectionAsync().block(); + + collection.createCollectionAsync().block(); + + return collection; + } + + + private static Stream supportedDataTypes() { + return Stream.of( + Arguments.of( + new ClassWithAllBoxedTypes( + "ID1", true, (byte) 127, (short) 3, 321, 5L, + 3.14f, 3.14159265358d, new BigDecimal("12345.67"), + OffsetDateTime.now(), UUID.randomUUID(), "abc".getBytes(StandardCharsets.UTF_8), + Arrays.asList(1.0f, 2.6f), + new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } + ) + ), + Arguments.of( + new ClassWithAllBoxedTypes( + "ID2", false, Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, Long.MIN_VALUE, + Float.MIN_VALUE, MIN_NUMBER, BigDecimal.valueOf(MIN_NUMBER), + OffsetDateTime.now(), UUID.randomUUID(), new byte[] {Byte.MIN_VALUE, -10, 0, 10, Byte.MAX_VALUE}, + Arrays.asList(Float.MIN_VALUE, -10f, 0f, 10f, Float.MAX_VALUE), + new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } + ) + ), + Arguments.of( + new ClassWithAllBoxedTypes( + "ID3", false, Byte.MAX_VALUE, Short.MAX_VALUE, Integer.MAX_VALUE, Long.MAX_VALUE, + Float.MAX_VALUE, BIG_NUMBER.doubleValue(), BIG_NUMBER.subtract(BigDecimal.valueOf(0.01d)), + OffsetDateTime.now(), UUID.randomUUID(), null, + null, + new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } + ) + ), + Arguments.of( + new ClassWithAllBoxedTypes( + "ID3", null, null, null, null, null, + null, null, null, + null, null, null, + null, + null + ) + ) + ); + } + + private static Stream supportedDataPrimitiveTypes() { + return Stream.of( + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID1", true, (byte) 127, (short) 3, 321, 5L, + 3.14f, 3.14159265358d, new BigDecimal("12345.67"), + OffsetDateTime.now(), UUID.randomUUID(), "abc".getBytes(StandardCharsets.UTF_8), + Arrays.asList(1.0f, 2.6f), + new float[]{0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f} + ) + ), + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID2", false, Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, + Long.MIN_VALUE, + Float.MIN_VALUE, MIN_NUMBER, BigDecimal.valueOf(MIN_NUMBER), + OffsetDateTime.now(), UUID.randomUUID(), + new byte[]{Byte.MIN_VALUE, -10, 0, 10, Byte.MAX_VALUE}, + Arrays.asList(Float.MIN_VALUE, -10f, 0f, 10f, Float.MAX_VALUE), + new float[]{0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f} + ) + ), + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID3", false, Byte.MAX_VALUE, Short.MAX_VALUE, Integer.MAX_VALUE, + Long.MAX_VALUE, + Float.MAX_VALUE, BIG_NUMBER.doubleValue(), + BIG_NUMBER.subtract(BigDecimal.valueOf(0.01d)), + OffsetDateTime.now(), UUID.randomUUID(), null, + null, + new float[]{0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f} + ) + ), + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID3", false, (byte) 0, (short) 0, 0, 0l, + 0f, 0d, null, + null, null, null, + null, + null + ) + ) + ); + } +} diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java index 07002b23..5c575950 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java @@ -123,6 +123,7 @@ void testPrimitiveDataTypes(ClassWithAllPrimitiveTypes values) { collection.deleteCollectionAsync().block(); } + private static Stream supportedDataTypes() { return Stream.of( Arguments.of( From a6fac68e531fbeb9fa9ac136c22badfa1c222fee Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Sat, 21 Jun 2025 00:20:02 +0200 Subject: [PATCH 186/277] BigDecimal mapping and value range --- .../data/jdbc/oracle/OracleDataTypesMapping.java | 2 +- .../OracleVectorStoreDataTypeSearchTest.java | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java index 4b74527b..0258cb5f 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java @@ -14,7 +14,7 @@ public class OracleDataTypesMapping { public static final String LONG = "NUMBER(19)"; public static final String FLOAT = "BINARY_FLOAT"; public static final String DOUBLE = "BINARY_DOUBLE"; - public static final String DECIMAL = "NUMBER(18,2)"; + public static final String DECIMAL = "NUMBER"; public static final String OFFSET_DATE_TIME = "TIMESTAMP(7) WITH TIME ZONE"; public static final String UUID = "RAW(16)"; public static final String JSON = "JSON"; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java index 4ce739a8..d99234ab 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java @@ -4,11 +4,9 @@ import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; -import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -24,7 +22,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; public class OracleVectorStoreDataTypeSearchTest extends OracleCommonVectorStoreRecordCollectionTest { - private static final double MIN_NUMBER = 1.0E-130; + private static final double MIN_DOUBLE = 1.0E-130; + private static final double MIN_DECIMAL = -1.0E125; private static final BigDecimal BIG_NUMBER = BigDecimal.valueOf(9999999999999999.99); @@ -120,6 +119,11 @@ void testDataTypesSearch (ClassWithAllBoxedTypes record) { assertEquals(1, results.getTotalCount()); assertEquals(record.getDoubleValue(), results.getResults().get(0).getRecord().getDoubleValue()); + System.out.println(record.getDecimalValue()); + System.out.println(record.getDecimalValue().doubleValue()); + System.out.println(results.getResults().get(0).getRecord().getDecimalValue()); + System.out.println(results.getResults().get(0).getRecord().getDecimalValue().doubleValue()); + // decimal results = collection.searchAsync( null, @@ -223,7 +227,7 @@ private static Stream supportedDataTypes() { Arguments.of( new ClassWithAllBoxedTypes( "ID2", false, Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, Long.MIN_VALUE, - Float.MIN_VALUE, MIN_NUMBER, BigDecimal.valueOf(MIN_NUMBER), + Float.MIN_VALUE, MIN_DOUBLE, BigDecimal.valueOf(MIN_DECIMAL), OffsetDateTime.now(), UUID.randomUUID(), new byte[] {Byte.MIN_VALUE, -10, 0, 10, Byte.MAX_VALUE}, Arrays.asList(Float.MIN_VALUE, -10f, 0f, 10f, Float.MAX_VALUE), new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } @@ -265,7 +269,7 @@ private static Stream supportedDataPrimitiveTypes() { new ClassWithAllPrimitiveTypes( "ID2", false, Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, Long.MIN_VALUE, - Float.MIN_VALUE, MIN_NUMBER, BigDecimal.valueOf(MIN_NUMBER), + Float.MIN_VALUE, MIN_DOUBLE, BigDecimal.valueOf(MIN_DECIMAL), OffsetDateTime.now(), UUID.randomUUID(), new byte[]{Byte.MIN_VALUE, -10, 0, 10, Byte.MAX_VALUE}, Arrays.asList(Float.MIN_VALUE, -10f, 0f, 10f, Float.MAX_VALUE), From 2f421a84bca70b39ba8d00b40a06c154cc63b8e5 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Tue, 3 Jun 2025 17:01:00 +0200 Subject: [PATCH 187/277] Any tag filter --- .../OracleVectorStoreQueryProvider.java | 47 +++++++++++++++++-- ...OracleVectorStoreRecordCollectionTest.java | 34 ++++++++++++-- 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index 71da3199..f9ba7414 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -4,7 +4,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; +import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; +import com.microsoft.semantickernel.data.filter.EqualToFilterClause; import com.microsoft.semantickernel.data.jdbc.*; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; @@ -34,6 +37,7 @@ import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -387,13 +391,12 @@ public VectorSearchResults search(String collectionName, List getFilterParameters(VectorSearchFilter filter) { + if (filter == null + || filter.getFilterClauses().isEmpty()) { + return Collections.emptyList(); + } + + return filter.getFilterClauses().stream().map(filterClause -> { + if (filterClause instanceof EqualToFilterClause) { + EqualToFilterClause equalToFilterClause = (EqualToFilterClause) filterClause; + return equalToFilterClause.getValue(); + } else if (filterClause instanceof AnyTagEqualToFilterClause) { + AnyTagEqualToFilterClause anyTagEqualToFilterClause = (AnyTagEqualToFilterClause) filterClause; + return anyTagEqualToFilterClause.getValue(); + } else { + throw new SKException("Unsupported filter clause type '" + + filterClause.getClass().getSimpleName() + "'."); + } + }).collect(Collectors.toList()); + } + + @Override + public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { + String fieldName = JDBCVectorStoreQueryProvider + .validateSQLidentifier(filterClause.getFieldName()); + + return String.format("JSON_EXISTS(%s, '$[*]?(@ == $v_%s)' PASSING ? AS \"v_%s\")", + fieldName, fieldName, fieldName); + } + + public static Builder builder() { return new Builder(); } diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 546e251f..72ed3524 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -127,19 +127,19 @@ public void clearCollection() { private static List getHotels() { return Arrays.asList( - new Hotel("id_1", "Hotel 1", 1, 1.49d, null, "Hotel 1 description", + new Hotel("id_1", "Hotel 1", 1, 1.49d, Arrays.asList("one", "two"), "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), null, null, null, 4.0), - new Hotel("id_2", "Hotel 2", 2, 1.44d, null, "Hotel 2 description with free-text search", + new Hotel("id_2", "Hotel 2", 2, 1.44d, Arrays.asList("three", "four"), "Hotel 2 description with free-text search", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), null, null, null, 4.0), - new Hotel("id_3", "Hotel 3", 3, 1.53d, null, "Hotel 3 description", + new Hotel("id_3", "Hotel 3", 3, 1.53d, Arrays.asList("five", "six"), "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), null, null, null, 5.0), - new Hotel("id_4", "Hotel 4", 4, 1.35d, null, "Hotel 4 description", + new Hotel("id_4", "Hotel 4", 4, 1.35d, Arrays.asList("seven", "eight"), "Hotel 4 description", Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), null, null, null, 4.0), - new Hotel("id_5", "Hotel 5", 5, 1.89d, null,"Hotel 5 description", + new Hotel("id_5", "Hotel 5", 5, 1.89d, Arrays.asList("nine", "ten"),"Hotel 5 description", Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), null, null, null, 4.0)); } @@ -300,6 +300,30 @@ public void searchWithFilter(DistanceFunction distanceFunction, double expectedD assertEquals(results.get(0).getScore(), expectedDistance, 0.0001d); } + + @Test + public void searchWithTagFilter() { + List hotels = getHotels(); + recordCollection.upsertBatchAsync(hotels, null).block(); + + VectorSearchOptions options = VectorSearchOptions.builder() +// .withVectorFieldName("") + .withTop(3) + .withVectorSearchFilter( + VectorSearchFilter.builder() + .anyTagEqualTo("tags", "three") + .build()) + .build(); + + // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned + List> results = recordCollection + .searchAsync(SEARCH_EMBEDDINGS, options).block().getResults(); + assertNotNull(results); + assertEquals(1, results.size()); + // The first hotel should be the most similar + assertEquals(hotels.get(1).getId(), results.get(0).getRecord().getId()); + } + private static Stream distanceFunctionAndDistance() { return Stream.of( Arguments.of (DistanceFunction.COSINE_DISTANCE, 0.8548d), From cf99cf7e77bee1a51828afe1437a1158c32a0fe2 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Tue, 10 Jun 2025 17:55:16 +0200 Subject: [PATCH 188/277] Added constants for type mapping and matched ODP type mapping when possible. --- .../jdbc/oracle/OracleDataTypesMapping.java | 18 ++++++ .../OracleVectorStoreQueryProvider.java | 64 +++++++++++++------ .../oracle/OracleVectorStoreRecordMapper.java | 32 +++++++--- 3 files changed, 84 insertions(+), 30 deletions(-) create mode 100644 data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java new file mode 100644 index 00000000..f441a440 --- /dev/null +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java @@ -0,0 +1,18 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +public class OracleDataTypesMapping { + public static final String STRING_VARCHAR = "NVARCHAR2(%s)"; + public static final String STRING_CLOB = "CLOB"; + public static final String BOOLEAN = "BOOLEAN"; + public static final String BYTE = "NUMBER(3)"; + public static final String BYTE_ARRAY = "RAW(2000)"; + public static final String SHORT = "NUMBER(5)"; + public static final String INTEGER = "NUMBER(10)"; + public static final String LONG = "NUMBER(19)"; + public static final String FLOAT = "BINARY_FLOAT"; + public static final String DOUBLE = "BINARY_DOUBLE"; + public static final String DECIMAL = "NUMBER(18,2)"; + public static final String OFFSET_DATE_TIME = "TIMESTAMP(7) WITH TIME ZONE"; + public static final String UUID = "RAW(16)"; + public static final String JSON = "JSON"; +} diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index f9ba7414..fe51f7b2 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -29,6 +29,7 @@ import javax.annotation.Nonnull; import javax.sql.DataSource; +import java.math.BigDecimal; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -41,6 +42,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -81,17 +83,25 @@ private OracleVectorStoreQueryProvider( prefixForCollectionTables, buildSupportedKeyTypes(), buildSupportedDataTypes(stringTypeMapping, defaultVarcharSize), - buildSupportedVectorTypes(defaultVarcharSize)); + buildSupportedVectorTypes()); this.collectionsTable = collectionsTable; this.objectMapper = objectMapper; } private static HashMap, String> buildSupportedKeyTypes() { HashMap, String> supportedKeyTypes = new HashMap<>(); - supportedKeyTypes.put(String.class, "VARCHAR(255)"); + supportedKeyTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, 255)); + supportedKeyTypes.put(short.class, OracleDataTypesMapping.SHORT); + supportedKeyTypes.put(Short.class, OracleDataTypesMapping.SHORT); + supportedKeyTypes.put(int.class, OracleDataTypesMapping.INTEGER); + supportedKeyTypes.put(Integer.class, OracleDataTypesMapping.INTEGER); + supportedKeyTypes.put(long.class, OracleDataTypesMapping.LONG); + supportedKeyTypes.put(Long.class, OracleDataTypesMapping.LONG); + supportedKeyTypes.put(UUID.class, OracleDataTypesMapping.UUID); + return supportedKeyTypes; } - private static Map, String> buildSupportedVectorTypes(int defaultVarCharLength) { + private static Map, String> buildSupportedVectorTypes() { HashMap, String> supportedVectorTypes = new HashMap<>(); supportedVectorTypes.put(String.class, "VECTOR(%s)"); supportedVectorTypes.put(List.class, "VECTOR(%s)"); @@ -102,22 +112,29 @@ private static Map, String> buildSupportedVectorTypes(int defaultVarCha private static Map, String> buildSupportedDataTypes(StringTypeMapping stringTypeMapping, int defaultVarCharLength) { HashMap, String> supportedDataTypes = new HashMap<>(); if (stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR)) { - supportedDataTypes.put(String.class, "VARCHAR(" + defaultVarCharLength + ")"); + supportedDataTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, defaultVarCharLength)); } else { - supportedDataTypes.put(String.class, "CLOB"); + supportedDataTypes.put(String.class, OracleDataTypesMapping.STRING_CLOB); } - supportedDataTypes.put(Integer.class, "INTEGER"); - supportedDataTypes.put(int.class, "INTEGER"); - supportedDataTypes.put(Long.class, "LONG"); - supportedDataTypes.put(long.class, "LONG"); - supportedDataTypes.put(Float.class, "REAL"); - supportedDataTypes.put(float.class, "REAL"); - supportedDataTypes.put(Double.class, "DOUBLE PRECISION"); - supportedDataTypes.put(double.class, "DOUBLE PRECISION"); - supportedDataTypes.put(Boolean.class, "BOOLEAN"); - supportedDataTypes.put(boolean.class, "BOOLEAN"); - supportedDataTypes.put(OffsetDateTime.class, "TIMESTAMPTZ"); - supportedDataTypes.put(List.class, "JSON"); + supportedDataTypes.put(byte.class, OracleDataTypesMapping.BYTE); + supportedDataTypes.put(Byte.class, OracleDataTypesMapping.BYTE); + supportedDataTypes.put(short.class, OracleDataTypesMapping.SHORT); + supportedDataTypes.put(Short.class, OracleDataTypesMapping.SHORT); + supportedDataTypes.put(int.class, OracleDataTypesMapping.INTEGER); + supportedDataTypes.put(Integer.class, OracleDataTypesMapping.INTEGER); + supportedDataTypes.put(long.class, OracleDataTypesMapping.LONG); + supportedDataTypes.put(Long.class, OracleDataTypesMapping.LONG); + supportedDataTypes.put(Float.class, OracleDataTypesMapping.FLOAT); + supportedDataTypes.put(float.class, OracleDataTypesMapping.FLOAT); + supportedDataTypes.put(Double.class, OracleDataTypesMapping.DOUBLE); + supportedDataTypes.put(double.class, OracleDataTypesMapping.DOUBLE); + supportedDataTypes.put(BigDecimal.class, OracleDataTypesMapping.DECIMAL); + supportedDataTypes.put(Boolean.class, OracleDataTypesMapping.BOOLEAN); + supportedDataTypes.put(boolean.class, OracleDataTypesMapping.BOOLEAN); + supportedDataTypes.put(OffsetDateTime.class, OracleDataTypesMapping.OFFSET_DATE_TIME); + supportedDataTypes.put(UUID.class, OracleDataTypesMapping.UUID); + supportedDataTypes.put(byte[].class, OracleDataTypesMapping.BYTE_ARRAY); + supportedDataTypes.put(List.class, OracleDataTypesMapping.JSON); return supportedDataTypes; } @@ -178,11 +195,11 @@ public void createCollection(String collectionName, List vectorFields = recordDefinition.getVectorFields(); String createStorageTable = formatQuery("CREATE TABLE IF NOT EXISTS %s (" - + "%s VARCHAR(255) PRIMARY KEY, " + + "%s PRIMARY KEY, " + "%s, " + "%s)", getCollectionTableName(collectionName), - getKeyColumnName(recordDefinition.getKeyField()), + getKeyColumnNameAndType(recordDefinition.getKeyField(), getSupportedDataTypes()), getColumnNamesAndTypes(new ArrayList<>(recordDefinition.getDataFields()), getSupportedDataTypes()), getVectorColumnNamesAndTypes(new ArrayList<>(vectorFields), @@ -237,6 +254,11 @@ public void createCollection(String collectionName, } } + private String getKeyColumnNameAndType(VectorStoreRecordKeyField field, Map, String> types) { + return validateSQLidentifier(field.getEffectiveStorageName()) + " " + + types.get(field.getFieldType()); + } + private String createIndexForDataField(String collectionName, VectorStoreRecordDataField dataField) { if (supportedDataTypes.get(dataField.getFieldType()) == "JSON") { String dataFieldIndex = "CREATE MULTIVALUE INDEX %s ON %s t (t.%s.%s)"; @@ -545,7 +567,7 @@ public static class Builder private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; private ObjectMapper objectMapper = new ObjectMapper(); private StringTypeMapping stringTypeMapping = StringTypeMapping.USE_VARCHAR; - private int defaultVarcharSize = 4000; + private int defaultVarcharSize = 2000; @SuppressFBWarnings("EI_EXPOSE_REP2") @@ -594,7 +616,7 @@ public Builder withStringTypeMapping (StringTypeMapping stringTypeMapping) { /** * Sets the default size of the VARHCHAR2 fields. * @param defaultVarcharSize the default size of the VARHCHAR2 fields. By default, the size - * is 4000. + * is 2000. * @return then builder */ public Builder withDefaultVarcharSize (int defaultVarcharSize) { diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index 1ba9fe7a..eaa8e7dc 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -147,29 +147,43 @@ public OracleVectorStoreRecordMapper build() { Object value; switch (supportedDataTypesMapping.get(fieldType)) { - case "CLOB": + case OracleDataTypesMapping.STRING_CLOB: value = resultSet.getString(field.getEffectiveStorageName()); break; - case "INTEGER": - value = resultSet.getInt(field.getEffectiveStorageName()); + case OracleDataTypesMapping.BYTE: + value = resultSet.getByte(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.SHORT: + value = resultSet.getShort(field.getEffectiveStorageName()); break; - case "LONG": + case OracleDataTypesMapping.INTEGER: value = resultSet.getInt(field.getEffectiveStorageName()); break; - case "REAL": + case OracleDataTypesMapping.LONG: + value = resultSet.getLong(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.FLOAT: value = resultSet.getFloat(field.getEffectiveStorageName()); break; - case "DOUBLE PRECISION": + case OracleDataTypesMapping.DOUBLE: value = resultSet.getDouble(field.getEffectiveStorageName()); break; - case "BOOLEAN": + case OracleDataTypesMapping.DECIMAL: + value = resultSet.getBigDecimal(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.BOOLEAN: value = resultSet.getBoolean(field.getEffectiveStorageName()); break; - case "TIMESTAMPTZ": + case OracleDataTypesMapping.OFFSET_DATE_TIME: value = ((OracleResultSet)resultSet).getTIMESTAMPTZ(field.getEffectiveStorageName()) .offsetDateTimeValue(); break; - case "JSON": + case OracleDataTypesMapping.BYTE_ARRAY: + value = resultSet.getBytes(field.getEffectiveStorageName()); + break; + // fallthrough + case OracleDataTypesMapping.UUID: + case OracleDataTypesMapping.JSON: value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); break; default: From a9bc539b62e1e3ac76788308ac2b4db278327adf Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Tue, 17 Jun 2025 21:11:59 +0200 Subject: [PATCH 189/277] Refactoring # Conflicts: # data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java --- .../jdbc/JDBCVectorStoreRecordCollection.java | 6 +- .../jdbc/oracle/OracleDataTypesMapping.java | 4 + .../oracle/OracleVectorStoreFieldHelper.java | 307 ++++++++++++++++++ .../data/jdbc/oracle/Hotel.java | 22 +- ...OracleVectorStoreRecordCollectionTest.java | 25 +- .../VectorStoreRecordDefinition.java | 2 +- .../VectorStoreRecordVectorField.java | 6 +- 7 files changed, 350 insertions(+), 22 deletions(-) create mode 100644 data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java index 2d75af14..81395589 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -34,10 +34,10 @@ public class JDBCVectorStoreRecordCollection implements SQLVectorStoreRecordCollection { private final String collectionName; - private final VectorStoreRecordDefinition recordDefinition; - private final VectorStoreRecordMapper vectorStoreRecordMapper; + protected final VectorStoreRecordDefinition recordDefinition; + protected final VectorStoreRecordMapper vectorStoreRecordMapper; private final JDBCVectorStoreRecordCollectionOptions options; - private final SQLVectorStoreQueryProvider queryProvider; + protected final SQLVectorStoreQueryProvider queryProvider; /** * Creates a new instance of the {@link JDBCVectorStoreRecordCollection}. diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java index f441a440..4b74527b 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java @@ -1,5 +1,8 @@ package com.microsoft.semantickernel.data.jdbc.oracle; +/** + * Defines oracle database type constants for supported field types. + */ public class OracleDataTypesMapping { public static final String STRING_VARCHAR = "NVARCHAR2(%s)"; public static final String STRING_CLOB = "CLOB"; @@ -15,4 +18,5 @@ public class OracleDataTypesMapping { public static final String OFFSET_DATE_TIME = "TIMESTAMP(7) WITH TIME ZONE"; public static final String UUID = "RAW(16)"; public static final String JSON = "JSON"; + public static final String VECTOR_FLOAT = "VECTOR(%s, FLOAT32)"; } diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java new file mode 100644 index 00000000..3ae09ed4 --- /dev/null +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java @@ -0,0 +1,307 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider.StringTypeMapping; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import oracle.jdbc.OracleTypes; +import java.math.BigDecimal; +import java.time.OffsetDateTime; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Helper class for field operations. + */ +public class OracleVectorStoreFieldHelper { + private static final Logger LOGGER = Logger.getLogger(OracleVectorStoreQueryProvider.class.getName()); + + /** + * Maps supported key java classes to Oracle database types + */ + private static final HashMap, String> supportedKeyTypes = new HashMap() { + { + put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, 255)); + put(short.class, OracleDataTypesMapping.SHORT); + put(Short.class, OracleDataTypesMapping.SHORT); + put(int.class, OracleDataTypesMapping.INTEGER); + put(Integer.class, OracleDataTypesMapping.INTEGER); + put(long.class, OracleDataTypesMapping.LONG); + put(Long.class, OracleDataTypesMapping.LONG); + put(UUID .class, OracleDataTypesMapping.UUID); + } + }; + + /** + * Maps supported vector java classes to Oracle database types + */ + private static final Map, String> supportedVectorTypes = new HashMap() { + { + put(String.class, OracleDataTypesMapping.VECTOR_FLOAT); + put(List.class, OracleDataTypesMapping.VECTOR_FLOAT); + put(Collection.class, OracleDataTypesMapping.VECTOR_FLOAT); + put(float[].class, OracleDataTypesMapping.VECTOR_FLOAT); + put(Float[].class, OracleDataTypesMapping.VECTOR_FLOAT); +/* + put(byte[].class,"VECTOR(%s, INT8)"); + put(Byte[].class,"VECTOR(%s, INT8)"); + put(double[].class,"VECTOR(%s, FLOAT64)"); + put(Double[].class,"VECTOR(%s, FLOAT64)"); + put(boolean[].class,"VECTOR(%s, BINARY)"); + put(Boolean[].class,"VECTOR(%s, BINARY)"); + */ + } + }; + + /** + * Maps supported data java classes to Oracle database types + */ + private static final HashMap, String> supportedDataTypes = new HashMap() { + { + put(byte.class, OracleDataTypesMapping.BYTE); + put(Byte.class, OracleDataTypesMapping.BYTE); + put(short.class, OracleDataTypesMapping.SHORT); + put(Short.class, OracleDataTypesMapping.SHORT); + put(int.class, OracleDataTypesMapping.INTEGER); + put(Integer.class, OracleDataTypesMapping.INTEGER); + put(long.class, OracleDataTypesMapping.LONG); + put(Long.class, OracleDataTypesMapping.LONG); + put(Float.class, OracleDataTypesMapping.FLOAT); + put(float.class, OracleDataTypesMapping.FLOAT); + put(Double.class, OracleDataTypesMapping.DOUBLE); + put(double.class, OracleDataTypesMapping.DOUBLE); + put(BigDecimal.class, OracleDataTypesMapping.DECIMAL); + put(Boolean.class, OracleDataTypesMapping.BOOLEAN); + put(boolean.class, OracleDataTypesMapping.BOOLEAN); + put(OffsetDateTime.class, OracleDataTypesMapping.OFFSET_DATE_TIME); + put(UUID.class, OracleDataTypesMapping.UUID); + put(byte[].class, OracleDataTypesMapping.BYTE_ARRAY); + put(List.class, OracleDataTypesMapping.JSON); + } + + }; + + /** + * Maps vector type to OracleTypes. Only needed if types other than FLOAT_32 are supported. + */ + private static final Map, Integer> mapOracleTypeToVector = new HashMap() { + { + put(float[].class, OracleTypes.VECTOR_FLOAT32); + put(Float[].class, OracleTypes.VECTOR_FLOAT32); +/* + put(byte[].class, OracleTypes.VECTOR_INT8); + put(Byte[].class, OracleTypes.VECTOR_INT8); + put(Double[].class, OracleTypes.VECTOR_FLOAT64); + put(double[].class, OracleTypes.VECTOR_FLOAT64); + put(Boolean[].class, OracleTypes.VECTOR_BINARY); + put(boolean[].class, OracleTypes.VECTOR_BINARY); +*/ + } + }; + + /** + * Gets the mapping between the supported Java key types and the Oracle database type. + * + * @return the mapping between the supported Java key types and the Oracle database type. + */ + public static HashMap, String> getSupportedKeyTypes() { + return supportedKeyTypes; + } + + /** + * Gets the mapping between the supported Java data types and the Oracle database type. + * + * @return the mapping between the supported Java data types and the Oracle database type. + */ + public static Map, String> getSupportedDataTypes( + StringTypeMapping stringTypeMapping, int defaultVarCharLength) { + + if (stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR)) { + supportedDataTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, defaultVarCharLength)); + } else { + supportedDataTypes.put(String.class, OracleDataTypesMapping.STRING_CLOB); + } + return supportedDataTypes; + } + + /** + * Gets the mapping between the supported Java data types and the Oracle database type. + * + * @return the mapping between the supported Java data types and the Oracle database type. + */ + public static Map, String> getSupportedVectorTypes() { + return supportedVectorTypes; + } + + /** + * Generates the statement to create the index according to the vector field definition. + * + * @return the CREATE VECTOR INDEX statement to create the index according to the vector + * field definition. + */ + public static String getCreateVectorIndexStatement(VectorStoreRecordVectorField field, String collectionTableName) { + switch (field.getIndexKind()) { + case IVFFLAT: + return "CREATE VECTOR INDEX IF NOT EXISTS " + + getIndexName(field.getEffectiveStorageName()) + + " ON " + + collectionTableName + "( " + field.getEffectiveStorageName() + " ) " + + " ORGANIZATION NEIGHBOR PARTITIONS " + + " WITH DISTANCE COSINE " + + "PARAMETERS ( TYPE IVF )"; + case HNSW: + return "CREATE VECTOR INDEX IF NOT EXISTS " + getIndexName(field.getEffectiveStorageName()) + + " ON " + + collectionTableName + "( " + field.getEffectiveStorageName() + " ) " + + "ORGANIZATION INMEMORY GRAPH " + + "WITH DISTANCE COSINE " + + "PARAMETERS (TYPE HNSW)"; + case UNDEFINED: + return null; + default: + LOGGER.warning("Unsupported index kind: " + field.getIndexKind()); + return null; + } + } + + /** + * Generates the statement to create the index according to the field definition. + * + * @return the CREATE INDEX statement to create the index according to the field definition. + */ + public static String createIndexForDataField(String collectionTableName, VectorStoreRecordDataField dataField, Map, String> supportedDataTypes) { + if (supportedDataTypes.get(dataField.getFieldType()) == "JSON") { + String dataFieldIndex = "CREATE MULTIVALUE INDEX %s ON %s t (t.%s.%s)"; + return String.format(dataFieldIndex, + collectionTableName + "_" + dataField.getEffectiveStorageName(), + collectionTableName, + dataField.getEffectiveStorageName(), + getFunctionForType(supportedDataTypes.get(dataField.getFieldSubType()))); + } else { + String dataFieldIndex = "CREATE INDEX %s ON %s (%s ASC)"; + return String.format(dataFieldIndex, + collectionTableName + "_" + dataField.getEffectiveStorageName(), + collectionTableName, + dataField.getEffectiveStorageName() + ); + } + } + + /** + * Gets the function that allows to return the function that converts the JSON value to the + * data type. + * @param jdbcType The JDBC type. + * @return the function that allows to return the function that converts the JSON value to the + * data type. + */ + private static String getFunctionForType(String jdbcType) { + switch (jdbcType) { + case OracleDataTypesMapping.BOOLEAN: + return "boolean()"; + case OracleDataTypesMapping.BYTE: + case OracleDataTypesMapping.SHORT: + case OracleDataTypesMapping.INTEGER: + case OracleDataTypesMapping.LONG: + case OracleDataTypesMapping.FLOAT: + case OracleDataTypesMapping.DOUBLE: + case OracleDataTypesMapping.DECIMAL: + return "numberOnly()"; + case OracleDataTypesMapping.OFFSET_DATE_TIME: + return "timestamp()"; + default: + return "string()"; + } + } + + /** + * Gets the type of the vector given the field definition. This method is not needed if only + * + * @param field the vector field definition. + * @return returns the type of vector for the given field type. + */ + public static String getTypeForVectorField(VectorStoreRecordVectorField field) { + String dimension = field.getDimensions() > 0 ? String.valueOf(field.getDimensions()) : "*"; + return String.format(supportedVectorTypes.get(field.getFieldType()), dimension); +/* Not needed since all types are FLOAT32 + if (field.getFieldSubType() != null) { + String vectorType; + switch (field.getFieldSubType().getName()) { + case "java.lang.Double": + vectorType = "FLOAT64"; + break; + case "java.lang.Byte": + vectorType = "INT8"; + break; + case "java.lang.Boolean": + vectorType = "BINARY"; + break; + default: + vectorType = "FLOAT32"; + } + return String.format(supportedVectorTypes.get(field.getFieldType()), dimension, vectorType); + } else { + return String.format(supportedVectorTypes.get(field.getFieldType()), dimension); + } + */ + } + + /** + * Gets the JDBC oracle of the vector field definition. + * @param field the vector field definition. + * @return the JDBC oracle type. + */ + public static int getOracleTypeForField(VectorStoreRecordVectorField field) { + if (field.getFieldSubType() == null) { + return mapOracleTypeToVector.get(field.getFieldType()).intValue(); + } else { + switch (field.getFieldSubType().getName()) { + case "java.lang.Double": + return OracleTypes.VECTOR_FLOAT64; + case "java.lang.Byte": + return OracleTypes.VECTOR_INT8; + case "java.lang.Boolean": + return OracleTypes.VECTOR_BINARY; + default: + return OracleTypes.VECTOR_FLOAT32; + } + } + } + + /** + * Generates the index name given the field name. by suffixing "_VECTOR_INDEX" to the field name. + * @param effectiveStorageName the field name. + * @return the index name. + */ + private static String getIndexName(String effectiveStorageName) { + return effectiveStorageName + "_VECTOR_INDEX"; + } + + /** + * Returns vector columns names and types for CREATE TABLE statement + * @param fields list of vector record fields. + * @return comma separated list of columns and types for CREATE TABLE statement. + */ + public static String getVectorColumnNamesAndTypes(List fields) { + List columns = fields.stream() + .map(field -> field.getEffectiveStorageName() + " " + + OracleVectorStoreFieldHelper.getTypeForVectorField(field) + ).collect(Collectors.toList()); + + return String.join(", ", columns); + } + + /** + * Returns key column names and type for key column for CREATE TABLE statement + * @param field the key field. + * @return column name and type of the key field for CREATE TABLE statement. + */ + public static String getKeyColumnNameAndType(VectorStoreRecordKeyField field) { + return field.getEffectiveStorageName() + " " + supportedKeyTypes.get(field.getFieldType()); + } + +} diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java index 197cb8c3..d1c848a6 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java @@ -40,15 +40,15 @@ public class Hotel { @JsonProperty("summaryEmbedding2") @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.HNSW) - private final List cosineDistance; + private final float[] cosineDistance; @JsonProperty("summaryEmbedding3") @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_SIMILARITY, indexKind = IndexKind.IVFFLAT) - private final List cosineSimilarity; + private final float[] cosineSimilarity; @JsonProperty("summaryEmbedding4") @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.DOT_PRODUCT, indexKind = IndexKind.IVFFLAT) - private final List dotProduct; + private final Float[] dotProduct; @VectorStoreRecordData private double rating; @@ -66,9 +66,9 @@ protected Hotel( @JsonProperty("tags") List tags, @JsonProperty("summary") String description, @JsonProperty("summaryEmbedding1") List euclidean, - @JsonProperty("summaryEmbedding2") List cosineDistance, - @JsonProperty("summaryEmbedding3") List cosineSimilarity, - @JsonProperty("summaryEmbedding4") List dotProduct, + @JsonProperty("summaryEmbedding2") float[] cosineDistance, + @JsonProperty("summaryEmbedding3") float[] cosineSimilarity, + @JsonProperty("summaryEmbedding4") Float[] dotProduct, @JsonProperty("rating") double rating) { this.id = id; this.name = name; @@ -77,9 +77,9 @@ protected Hotel( this.tags = tags; this.description = description; this.euclidean = euclidean; - this.cosineDistance = euclidean; - this.cosineSimilarity = euclidean; - this.dotProduct = euclidean; + this.cosineDistance = cosineDistance; + this.cosineSimilarity = cosineSimilarity; + this.dotProduct = dotProduct; this.rating = rating; } @@ -107,11 +107,11 @@ public List getEuclidean() { return euclidean; } - public List getCosineDistance() { + public float[] getCosineDistance() { return cosineDistance; } - public List getDotProduct() { + public Float[] getDotProduct() { return dotProduct; } diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 72ed3524..22d0259f 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -128,19 +128,34 @@ public void clearCollection() { private static List getHotels() { return Arrays.asList( new Hotel("id_1", "Hotel 1", 1, 1.49d, Arrays.asList("one", "two"), "Hotel 1 description", - Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), null, null, null, + Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), + new float[] {0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f}, + new float[] {0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f}, + new Float[] {0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f}, 4.0), new Hotel("id_2", "Hotel 2", 2, 1.44d, Arrays.asList("three", "four"), "Hotel 2 description with free-text search", - Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), null, null, null, + Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), + new float[] {-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f}, + new float[] {-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f}, + new Float[] {-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f}, 4.0), new Hotel("id_3", "Hotel 3", 3, 1.53d, Arrays.asList("five", "six"), "Hotel 3 description", - Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), null, null, null, + Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), + new float[] {4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f}, + new float[] {4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f}, + new Float[] {4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f}, 5.0), new Hotel("id_4", "Hotel 4", 4, 1.35d, Arrays.asList("seven", "eight"), "Hotel 4 description", - Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), null, null, null, + Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), + new float[] {7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f}, + new float[] {7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f}, + new Float[] {7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f}, 4.0), new Hotel("id_5", "Hotel 5", 5, 1.89d, Arrays.asList("nine", "ten"),"Hotel 5 description", - Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), null, null, null, + Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), + new float[] {-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f}, + new float[] {-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f}, + new Float[] {-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f}, 4.0)); } diff --git a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java index 8f8d6212..e769bb6f 100644 --- a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java +++ b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java @@ -210,7 +210,7 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) vectorFields.add(VectorStoreRecordVectorField.builder() .withName(field.getName()) .withStorageName(storageName) - .withFieldType(field.getType()) + .withFieldType(field.getType(), List.class.equals(field.getType()) ? (Class)((ParameterizedType) field.getGenericType()).getActualTypeArguments()[0] : null) .withDimensions(vectorAttribute.dimensions()) .withIndexKind(vectorAttribute.indexKind()) .withDistanceFunction(vectorAttribute.distanceFunction()) diff --git a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java index 00b7627a..b708d2fb 100644 --- a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java +++ b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordVectorField.java @@ -34,10 +34,11 @@ public VectorStoreRecordVectorField( @Nonnull String name, @Nullable String storageName, @Nonnull Class fieldType, + Class fieldSubType, int dimensions, @Nullable IndexKind indexKind, @Nullable DistanceFunction distanceFunction) { - super(name, storageName, fieldType); + super(name, storageName, fieldType, fieldSubType); this.dimensions = dimensions; this.indexKind = indexKind == null ? IndexKind.UNDEFINED : indexKind; this.distanceFunction = distanceFunction == null ? DistanceFunction.UNDEFINED @@ -130,7 +131,8 @@ public VectorStoreRecordVectorField build() { throw new IllegalArgumentException("dimensions must be greater than 0"); } - return new VectorStoreRecordVectorField(name, storageName, fieldType, dimensions, + return new VectorStoreRecordVectorField(name, storageName, fieldType, fieldSubType, + dimensions, indexKind, distanceFunction); } From 17fce47b46131503a7c5c4d8874f46d8576013cb Mon Sep 17 00:00:00 2001 From: Kaiyuan Li Date: Sat, 7 Jun 2025 00:52:21 -0700 Subject: [PATCH 190/277] Add test for IndexKind.UNDEFINED and IndexKind.HNSW --- ...OracleVectorStoreRecordCollectionTest.java | 161 +++++++++++++++++- 1 file changed, 160 insertions(+), 1 deletion(-) diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 22d0259f..06e8c8d6 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -9,17 +9,28 @@ import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import oracle.jdbc.OracleConnection; import oracle.jdbc.datasource.impl.OracleDataSource; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.time.Duration; import java.util.Arrays; import java.util.List; @@ -42,7 +53,6 @@ public class OracleVectorStoreRecordCollectionTest { private static final OracleDataSource DATA_SOURCE; private static final OracleDataSource SYSDBA_DATA_SOURCE; - static { try { @@ -339,6 +349,155 @@ public void searchWithTagFilter() { assertEquals(hotels.get(1).getId(), results.get(0).getRecord().getId()); } + @Nested + class HNSWIndexTests { + @Test + void testHNSWIndexIsCreatedSuccessfully() throws Exception { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + VectorStoreRecordDataField dummyField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(String.class) + .isFilterable(false) + .build(); + + VectorStoreRecordVectorField hnswVector= VectorStoreRecordVectorField.builder() + .withName("hnsw") + .withStorageName("hnsw") + .withFieldType(List.class) + .withDimensions(8) + .withDistanceFunction(DistanceFunction.COSINE_SIMILARITY) + .withIndexKind(IndexKind.HNSW) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dummyField, hnswVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "skhotels_hnsw"; + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Object.class) + .withRecordDefinition(definition) + .build()); + + // create collection + collection.createCollectionAsync().block(); + + String expectedIndexName = hnswVector.getEffectiveStorageName().toUpperCase() + "_VECTOR_INDEX"; + + // check if index exist + try (Connection conn = DATA_SOURCE.getConnection(); + PreparedStatement stmt = conn.prepareStatement( + "SELECT COUNT(*) FROM USER_INDEXES WHERE INDEX_NAME=?")) { + stmt.setString(1, expectedIndexName); + ResultSet rs = stmt.executeQuery(); + rs.next(); + int count = rs.getInt(1); + + assertEquals(1, count, "hnsw vector index should have been created"); + } finally { + // clean up + try (Connection conn = DATA_SOURCE.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE " + "SKCOLLECTION_" + collectionName); + } + } + } + } + + @Nested + class UndefinedIndexTests { + @Test + void testNoIndexIsCreatedForUndefined() throws Exception { + // create key field + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + // create vector field, set IndexKind to UNDEFINED + VectorStoreRecordVectorField undefinedVector= VectorStoreRecordVectorField.builder() + .withName("undef") + .withStorageName("undef") + .withFieldType(List.class) + .withDimensions(8) + .withDistanceFunction(DistanceFunction.COSINE_SIMILARITY) + .withIndexKind(IndexKind.UNDEFINED) + .build(); + + VectorStoreRecordDataField dummyField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(String.class) + .isFilterable(false) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dummyField, undefinedVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "skhotels_undefined"; + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Object.class) + .withRecordDefinition(definition) + .build()); + + // create collection + collection.createCollectionAsync().block(); + + // check if index exist + String expectedIndexName = undefinedVector.getEffectiveStorageName().toUpperCase() + "_VETCOR_INDEX"; + try (Connection conn = DATA_SOURCE.getConnection(); + PreparedStatement stmt = conn.prepareStatement( + "SELECT COUNT(*) FROM USER_INDEXES WHERE INDEX_NAME = ?")) { + stmt.setString(1, expectedIndexName); + ResultSet rs = stmt.executeQuery(); + rs.next(); + int count = rs.getInt(1); + + assertEquals(0,count,"Vector index should not be created for IndexKind.UNDEFINED"); + } finally { + // clean up + try (Connection conn = DATA_SOURCE.getConnection(); + Statement stmt = conn.createStatement()) { + stmt.executeUpdate("DROP TABLE " + "SKCOLLECTION_" + collectionName); + } + } + } + } + private static Stream distanceFunctionAndDistance() { return Stream.of( Arguments.of (DistanceFunction.COSINE_DISTANCE, 0.8548d), From a9e82132cd15f50eaa09b30a19fac7449a1946ca Mon Sep 17 00:00:00 2001 From: Kaiyuan Li Date: Sat, 14 Jun 2025 00:50:03 -0700 Subject: [PATCH 191/277] add test for key type --- ...OracleVectorStoreRecordCollectionTest.java | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 06e8c8d6..8ad16702 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -1,5 +1,7 @@ package com.microsoft.semantickernel.data.jdbc.oracle; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollection; import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; @@ -16,6 +18,7 @@ import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; + import oracle.jdbc.OracleConnection; import oracle.jdbc.datasource.impl.OracleDataSource; import org.junit.jupiter.api.BeforeAll; @@ -33,7 +36,9 @@ import java.sql.Statement; import java.time.Duration; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.UUID; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -349,6 +354,69 @@ public void searchWithTagFilter() { assertEquals(hotels.get(1).getId(), results.get(0).getRecord().getId()); } + @ParameterizedTest + @MethodSource("supportedKeyTypes") + void testKeyTypes(String suffix, Class keyType, Object keyValue) { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(keyType) + .build(); + + VectorStoreRecordDataField dummyField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(String.class) + .build(); + + VectorStoreRecordVectorField dummyVector = VectorStoreRecordVectorField.builder() + .withName("vec") + .withStorageName("vec") + .withFieldType(List.class) + .withDimensions(2) + .withDistanceFunction(DistanceFunction.EUCLIDEAN_DISTANCE) + .withIndexKind(IndexKind.UNDEFINED) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dummyField, dummyVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "test_keytype_" + suffix; + + VectorStoreRecordCollection collectionRaw = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(DummyRecordForKeyTypes.class) + .withRecordDefinition(definition) + .build()); + + VectorStoreRecordCollection collection = + (VectorStoreRecordCollection) collectionRaw; + + collection.createCollectionAsync().block(); + + DummyRecordForKeyTypes record = new DummyRecordForKeyTypes(keyValue, "dummyValue", Arrays.asList(1.0f, 2.0f)); + collection.upsertAsync(record, null).block(); + + DummyRecordForKeyTypes result = collection.getAsync(keyValue, null).block(); + assertNotNull(result); + assertEquals("dummyValue", result.getDummy()); + + collection.deleteCollectionAsync().block(); + } + @Nested class HNSWIndexTests { @Test @@ -517,4 +585,48 @@ private static Stream parametersExactSearch() { Arguments.of (DistanceFunction.UNDEFINED, Arrays.asList(0.1000d, 18.9081d, 19.9669d)) ); } + + // commented out temporarily because only String type key is supported in + // JDBCVectorStoreRecordCollection#getKeyFromRecord: + // ... + // return (String) keyField.get(data); + // ... + // thus upsertAync/getAsync won't work + private static Stream supportedKeyTypes() { + return Stream.of( + Arguments.of("string", String.class, "asd123")/*, + Arguments.of("integer", Integer.class, 321), + Arguments.of("long", Long.class, 5L), + Arguments.of("short", Short.class, (short) 3), + Arguments.of("uuid", UUID.class, UUID.randomUUID())*/ + ); + } + + private static class DummyRecordForKeyTypes { + private final Object id; + private final String dummy; + private final List vec; + @JsonCreator + public DummyRecordForKeyTypes( + @JsonProperty("id")Object id, + @JsonProperty("dummy") String dummy, + @JsonProperty("vec") List vec) { + this.id = id; + this.dummy = dummy; + this.vec = vec; + } + + public Object getId() { + return id; + } + + public String getDummy() { + return dummy; + } + + @Override + public String toString() { + return String.valueOf(id); + } + } } From b3e1f4bf646b56326d12213c1a62ea996894206c Mon Sep 17 00:00:00 2001 From: Kaiyuan Li Date: Tue, 17 Jun 2025 23:17:44 -0700 Subject: [PATCH 192/277] Add test for supported data type(timestamp and uuid not included) --- ...OracleVectorStoreRecordCollectionTest.java | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 8ad16702..20d49292 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -29,20 +29,25 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.time.Duration; +import java.time.OffsetDateTime; import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -417,6 +422,87 @@ void testKeyTypes(String suffix, Class keyType, Object keyValue) { collection.deleteCollectionAsync().block(); } + @ParameterizedTest + @MethodSource("supportedDataTypes") + void testDataTypes(String dataFieldName, Class dataFieldType, Object dataFieldValue, Class fieldSubType) { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + VectorStoreRecordDataField dataField; + if (fieldSubType != null) { + dataField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(dataFieldType, fieldSubType) + .isFilterable(true) + .build(); + } else { + dataField = VectorStoreRecordDataField.builder() + .withName("dummy") + .withStorageName("dummy") + .withFieldType(dataFieldType) + .isFilterable(true) + .build(); + } + + VectorStoreRecordVectorField dummyVector = VectorStoreRecordVectorField.builder() + .withName("vec") + .withStorageName("vec") + .withFieldType(List.class) + .withDimensions(2) + .withDistanceFunction(DistanceFunction.EUCLIDEAN_DISTANCE) + .withIndexKind(IndexKind.UNDEFINED) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dataField, dummyVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "test_datatype_" + dataFieldName; + + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions. builder() + .withRecordClass(DummyRecordForDataTypes.class) + .withRecordDefinition(definition).build()); + + collection.createCollectionAsync().block(); + + String key = "testid"; + + DummyRecordForDataTypes record = + new DummyRecordForDataTypes(key, dataFieldValue, Arrays.asList(1.0f, 2.0f)); + + collection.upsertAsync(record, null).block(); + + DummyRecordForDataTypes result = collection.getAsync(key, null).block(); + assertNotNull(result); + + if (dataFieldValue instanceof Number && result.getDummy() instanceof Number) { + assertEquals(((Number) dataFieldValue).doubleValue(), ((Number) result.getDummy()).doubleValue()); + } else if (dataFieldValue instanceof byte[]) { + assertArrayEquals((byte[]) dataFieldValue, (byte[]) result.getDummy()); + } else { + assertEquals(dataFieldValue, result.getDummy()); + } + + collection.deleteCollectionAsync().block(); + } + @Nested class HNSWIndexTests { @Test @@ -602,6 +688,25 @@ private static Stream supportedKeyTypes() { ); } + private static Stream supportedDataTypes() { + return Stream.of( + Arguments.of("string", String.class, "asd123", null), + Arguments.of("boolean_true", Boolean.class, true, null), + Arguments.of("boolean_false", Boolean.class, false, null), + Arguments.of("byte", Byte.class, (byte) 127, null), + Arguments.of("short", Short.class, (short) 3, null), + Arguments.of("integer", Integer.class, 321, null), + Arguments.of("long", Long.class, 5L, null), + Arguments.of("float", Float.class, 3.14f, null), + Arguments.of("double", double.class, 3.14159265358d, null), + Arguments.of("decimal", BigDecimal.class, new BigDecimal("12345.67"), null), + //Arguments.of("timestamp", OffsetDateTime.class, OffsetDateTime.now(), null) + //Arguments.of("uuid", UUID.class, UUID.randomUUID(), null) + Arguments.of("byte_array", byte[].class, "abc".getBytes(StandardCharsets.UTF_8), null), + Arguments.of("json", List.class, Arrays.asList("a", "s", "d"), String.class) + ); + } + private static class DummyRecordForKeyTypes { private final Object id; private final String dummy; @@ -629,4 +734,32 @@ public String toString() { return String.valueOf(id); } } + + private static class DummyRecordForDataTypes { + private final String id; + private final Object dummy; + private final List vec; + @JsonCreator + public DummyRecordForDataTypes( + @JsonProperty("id") String id, + @JsonProperty("dummy") Object dummy, + @JsonProperty("vec") List vec) { + this.id = id; + this.dummy = dummy; + this.vec = vec; + } + + public String getId() { + return id; + } + + public Object getDummy() { + return dummy; + } + + @Override + public String toString() { + return String.valueOf(id); + } + } } From ec313762fe796b5a5da0e9c50db5cf0eef0803ce Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Fri, 20 Jun 2025 18:46:42 +0200 Subject: [PATCH 193/277] Test and bug fixes --- .../OracleVectorStoreDataTypeSearchTest.java | 297 ++++++++++++++++++ 1 file changed, 297 insertions(+) create mode 100644 data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java new file mode 100644 index 00000000..4ce739a8 --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java @@ -0,0 +1,297 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.util.Arrays; +import java.util.UUID; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class OracleVectorStoreDataTypeSearchTest extends OracleCommonVectorStoreRecordCollectionTest { + private static final double MIN_NUMBER = 1.0E-130; + private static final BigDecimal BIG_NUMBER = BigDecimal.valueOf(9999999999999999.99); + + + + @ParameterizedTest + @MethodSource("supportedDataTypes") + void testDataTypesSearch (ClassWithAllBoxedTypes record) { + VectorStoreRecordCollection collection = setupBoxed(); + + collection.upsertAsync(record, null).block(); + + // boolean + VectorSearchResults results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("booleanValue", record.getBooleanValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getBooleanValue(), results.getResults().get(0).getRecord().getBooleanValue()); + + // byte + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("byteValue", record.getByteValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getByteValue(), results.getResults().get(0).getRecord().getByteValue()); + + // short + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("shortValue", record.getShortValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getShortValue(), results.getResults().get(0).getRecord().getShortValue()); + + // integer + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("integerValue", record.getIntegerValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getIntegerValue(), results.getResults().get(0).getRecord().getIntegerValue()); + + // long + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("longValue", record.getLongValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getLongValue(), results.getResults().get(0).getRecord().getLongValue()); + + // float + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("floatValue", record.getFloatValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getFloatValue(), results.getResults().get(0).getRecord().getFloatValue()); + + // double + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("doubleValue", record.getDoubleValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getDoubleValue(), results.getResults().get(0).getRecord().getDoubleValue()); + + // decimal + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("decimalValue", record.getDecimalValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + if (record.getDecimalValue() != null) { + assertEquals(0, record.getDecimalValue() + .compareTo(results.getResults().get(0).getRecord().getDecimalValue())); + } else { + assertEquals(record.getDecimalValue(), + results.getResults().get(0).getRecord().getDecimalValue()); + } + + // offset date time + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("offsetDateTimeValue", record.getOffsetDateTimeValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + if (record.getOffsetDateTimeValue() != null) { + assertTrue(record.getOffsetDateTimeValue() + .isEqual(results.getResults().get(0).getRecord().getOffsetDateTimeValue())); + } else { + assertEquals(record.getOffsetDateTimeValue(), + results.getResults().get(0).getRecord().getOffsetDateTimeValue()); + } + + // UUID + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("uuidValue", record.getUuidValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals(record.getUuidValue(), results.getResults().get(0).getRecord().getUuidValue()); + + // byte array + results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("byteArrayValue", record.getByteArrayValue()).build() + ).build()).block(); + + assertEquals(1, results.getTotalCount()); + assertArrayEquals(record.getByteArrayValue(), results.getResults().get(0).getRecord().getByteArrayValue()); + + collection.deleteCollectionAsync().block(); + + } + + + public VectorStoreRecordCollection setupBoxed() { + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + VectorStoreRecordCollection collection = + vectorStore.getCollection("BoxedTypes", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(ClassWithAllBoxedTypes.class) + .build()).createCollectionAsync().block(); + + collection.createCollectionAsync().block(); + + return collection; + } + + + private static Stream supportedDataTypes() { + return Stream.of( + Arguments.of( + new ClassWithAllBoxedTypes( + "ID1", true, (byte) 127, (short) 3, 321, 5L, + 3.14f, 3.14159265358d, new BigDecimal("12345.67"), + OffsetDateTime.now(), UUID.randomUUID(), "abc".getBytes(StandardCharsets.UTF_8), + Arrays.asList(1.0f, 2.6f), + new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } + ) + ), + Arguments.of( + new ClassWithAllBoxedTypes( + "ID2", false, Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, Long.MIN_VALUE, + Float.MIN_VALUE, MIN_NUMBER, BigDecimal.valueOf(MIN_NUMBER), + OffsetDateTime.now(), UUID.randomUUID(), new byte[] {Byte.MIN_VALUE, -10, 0, 10, Byte.MAX_VALUE}, + Arrays.asList(Float.MIN_VALUE, -10f, 0f, 10f, Float.MAX_VALUE), + new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } + ) + ), + Arguments.of( + new ClassWithAllBoxedTypes( + "ID3", false, Byte.MAX_VALUE, Short.MAX_VALUE, Integer.MAX_VALUE, Long.MAX_VALUE, + Float.MAX_VALUE, BIG_NUMBER.doubleValue(), BIG_NUMBER.subtract(BigDecimal.valueOf(0.01d)), + OffsetDateTime.now(), UUID.randomUUID(), null, + null, + new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } + ) + ), + Arguments.of( + new ClassWithAllBoxedTypes( + "ID3", null, null, null, null, null, + null, null, null, + null, null, null, + null, + null + ) + ) + ); + } + + private static Stream supportedDataPrimitiveTypes() { + return Stream.of( + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID1", true, (byte) 127, (short) 3, 321, 5L, + 3.14f, 3.14159265358d, new BigDecimal("12345.67"), + OffsetDateTime.now(), UUID.randomUUID(), "abc".getBytes(StandardCharsets.UTF_8), + Arrays.asList(1.0f, 2.6f), + new float[]{0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f} + ) + ), + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID2", false, Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, + Long.MIN_VALUE, + Float.MIN_VALUE, MIN_NUMBER, BigDecimal.valueOf(MIN_NUMBER), + OffsetDateTime.now(), UUID.randomUUID(), + new byte[]{Byte.MIN_VALUE, -10, 0, 10, Byte.MAX_VALUE}, + Arrays.asList(Float.MIN_VALUE, -10f, 0f, 10f, Float.MAX_VALUE), + new float[]{0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f} + ) + ), + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID3", false, Byte.MAX_VALUE, Short.MAX_VALUE, Integer.MAX_VALUE, + Long.MAX_VALUE, + Float.MAX_VALUE, BIG_NUMBER.doubleValue(), + BIG_NUMBER.subtract(BigDecimal.valueOf(0.01d)), + OffsetDateTime.now(), UUID.randomUUID(), null, + null, + new float[]{0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f} + ) + ), + Arguments.of( + new ClassWithAllPrimitiveTypes( + "ID3", false, (byte) 0, (short) 0, 0, 0l, + 0f, 0d, null, + null, null, null, + null, + null + ) + ) + ); + } +} From cb4f72692c7e6be0c139d53524c9d2198edb2287 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Sat, 21 Jun 2025 00:20:02 +0200 Subject: [PATCH 194/277] BigDecimal mapping and value range --- .../data/jdbc/oracle/OracleDataTypesMapping.java | 2 +- .../OracleVectorStoreDataTypeSearchTest.java | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java index 4b74527b..0258cb5f 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java @@ -14,7 +14,7 @@ public class OracleDataTypesMapping { public static final String LONG = "NUMBER(19)"; public static final String FLOAT = "BINARY_FLOAT"; public static final String DOUBLE = "BINARY_DOUBLE"; - public static final String DECIMAL = "NUMBER(18,2)"; + public static final String DECIMAL = "NUMBER"; public static final String OFFSET_DATE_TIME = "TIMESTAMP(7) WITH TIME ZONE"; public static final String UUID = "RAW(16)"; public static final String JSON = "JSON"; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java index 4ce739a8..d99234ab 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java @@ -4,11 +4,9 @@ import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; -import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -24,7 +22,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; public class OracleVectorStoreDataTypeSearchTest extends OracleCommonVectorStoreRecordCollectionTest { - private static final double MIN_NUMBER = 1.0E-130; + private static final double MIN_DOUBLE = 1.0E-130; + private static final double MIN_DECIMAL = -1.0E125; private static final BigDecimal BIG_NUMBER = BigDecimal.valueOf(9999999999999999.99); @@ -120,6 +119,11 @@ void testDataTypesSearch (ClassWithAllBoxedTypes record) { assertEquals(1, results.getTotalCount()); assertEquals(record.getDoubleValue(), results.getResults().get(0).getRecord().getDoubleValue()); + System.out.println(record.getDecimalValue()); + System.out.println(record.getDecimalValue().doubleValue()); + System.out.println(results.getResults().get(0).getRecord().getDecimalValue()); + System.out.println(results.getResults().get(0).getRecord().getDecimalValue().doubleValue()); + // decimal results = collection.searchAsync( null, @@ -223,7 +227,7 @@ private static Stream supportedDataTypes() { Arguments.of( new ClassWithAllBoxedTypes( "ID2", false, Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, Long.MIN_VALUE, - Float.MIN_VALUE, MIN_NUMBER, BigDecimal.valueOf(MIN_NUMBER), + Float.MIN_VALUE, MIN_DOUBLE, BigDecimal.valueOf(MIN_DECIMAL), OffsetDateTime.now(), UUID.randomUUID(), new byte[] {Byte.MIN_VALUE, -10, 0, 10, Byte.MAX_VALUE}, Arrays.asList(Float.MIN_VALUE, -10f, 0f, 10f, Float.MAX_VALUE), new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f } @@ -265,7 +269,7 @@ private static Stream supportedDataPrimitiveTypes() { new ClassWithAllPrimitiveTypes( "ID2", false, Byte.MIN_VALUE, Short.MIN_VALUE, Integer.MIN_VALUE, Long.MIN_VALUE, - Float.MIN_VALUE, MIN_NUMBER, BigDecimal.valueOf(MIN_NUMBER), + Float.MIN_VALUE, MIN_DOUBLE, BigDecimal.valueOf(MIN_DECIMAL), OffsetDateTime.now(), UUID.randomUUID(), new byte[]{Byte.MIN_VALUE, -10, 0, 10, Byte.MAX_VALUE}, Arrays.asList(Float.MIN_VALUE, -10f, 0f, 10f, Float.MAX_VALUE), From 6bd3a8e646a9c8383487ea3f4548b1e37d3e1687 Mon Sep 17 00:00:00 2001 From: psilberk Date: Fri, 25 Apr 2025 16:20:54 -0700 Subject: [PATCH 195/277] Upsert and Main sample --- data/semantickernel-data-jdbc/pom.xml | 43 +++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 7cd77f3f..1f2e1a28 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-SNAPSHOT ../../pom.xml @@ -14,12 +14,13 @@ - org.slf4j - slf4j-api + com.microsoft.semantic-kernel + semantickernel-api + - com.microsoft.semantic-kernel - semantickernel-api-data + org.slf4j + slf4j-api com.fasterxml.jackson.core @@ -31,9 +32,41 @@ jackson-core compile + + com.github.jknack + handlebars + + + com.google.code.findbugs + jsr305 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + compile + com.github.spotbugs spotbugs-annotations + + org.apache.commons + commons-text + + + org.postgresql + postgresql + 42.7.4 + + + org.xerial + sqlite-jdbc + 3.47.0.0 + + + com.oracle.database.jdbc + ojdbc11 + 23.7.0.25.01 + \ No newline at end of file From 2e3133390f8d808e6e7bf1abf9261fc20ce36bc9 Mon Sep 17 00:00:00 2001 From: psilberk Date: Mon, 23 Jun 2025 14:57:40 -0700 Subject: [PATCH 196/277] Books demo with vector similarity search --- .../data/vectorstores/oracle/Book.java | 49 ++++++++++++++----- .../data/vectorstores/oracle/Main.java | 26 +++++++--- 2 files changed, 56 insertions(+), 19 deletions(-) diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java index aeaff6e8..87104cf8 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java @@ -4,12 +4,12 @@ import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; -import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; -import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; import java.util.List; public class Book { + public Book() {} + public Book(String isbn, String title, String author, int pages, List tags, String summary, List summaryEmbedding) { this.isbn = isbn; @@ -21,27 +21,26 @@ public Book(String isbn, String title, String author, int pages, this.summaryEmbedding = summaryEmbedding; } - @VectorStoreRecordKey - private final String isbn; + private String isbn; @VectorStoreRecordData(isFilterable = true) - private final String title; + private String title; @VectorStoreRecordData(isFilterable = true) - private final String author; + private String author; @VectorStoreRecordData - private final int pages; + private int pages; @VectorStoreRecordData(isFilterable = true) - private final List tags; + private List tags; @VectorStoreRecordData( isFilterable = true, isFullTextSearchable = true ) - private final String summary; + private String summary; - @VectorStoreRecordVector(dimensions = 4, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.HNSW) - private final List summaryEmbedding; + @VectorStoreRecordVector(dimensions = 2) + private List summaryEmbedding; public String getIsbn() { return isbn; @@ -70,4 +69,32 @@ public String getSummary() { public List getSummaryEmbedding() { return summaryEmbedding; } + + public void setIsbn(String isbn) { + this.isbn = isbn; + } + + public void setTitle(String title) { + this.title = title; + } + + public void setAuthor(String author) { + this.author = author; + } + + public void setPages(int pages) { + this.pages = pages; + } + + public void setTags(List tags) { + this.tags = tags; + } + + public void setSummaryEmbedding(List summaryEmbedding) { + this.summaryEmbedding = summaryEmbedding; + } + + public void setSummary(String summary) { + this.summary = summary; + } } \ No newline at end of file diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java index d71a0f7e..babc381f 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java @@ -5,11 +5,14 @@ import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; import java.sql.SQLException; import java.util.Arrays; import java.util.List; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import oracle.jdbc.datasource.impl.OracleDataSource; +import reactor.core.publisher.Mono; public class Main { public static void main(String[] args) throws SQLException { @@ -45,23 +48,30 @@ public static void main(String[] args) throws SQLException { collection.upsertBatchAsync(books, null).block(); // Retrieve the upserted record. - //var retrievedBook = collection.getAsync("1", null).block(); + Book retrievedBook = collection.getAsync("2", null).block(); + + System.out.println(retrievedBook.getAuthor()); // Generate a vector for your search text, using your chosen embedding generation implementation. // Just showing a placeholder method here for brevity. - // var searchVector = generateEmbeddingsAsync( - // "I'm looking for a Book where customer happiness is the priority.").block(); + List searchVector = generateEmbeddingsAsync( + "I'm looking for a horror book.").block(); // Do the search. - // var searchResult = collection.searchAsync(searchVector, VectorSearchOptions.builder() - // .withTop(1).build()).block(); + VectorSearchResults searchResult = collection.searchAsync( + searchVector, VectorSearchOptions.builder().withTop(1).build()).block(); - // Book record = searchResult.getResults().get(0).getRecord(); - // System.out.printf("Found Book description: %s\n", record.getDescription()); + retrievedBook = searchResult.getResults().get(0).getRecord(); + System.out.println("Found Book: " + retrievedBook.getIsbn()); } static List books = Arrays.asList( - new Book("1", "one", "sking", 0, null, "sum", null)); + new Book("1", "one", "sking", 0, null, "horror", List.of(1f, 1f)), + new Book("2", "two", "squeen", 0, null, "non-fiction", List.of(-11f, -11f))); + + private static Mono> generateEmbeddingsAsync(String text) { + return Mono.just(List.of(-0.9f, -0.9f)); + } } \ No newline at end of file From 32f3ddce32d65d9520cf139c1d581d87c591153b Mon Sep 17 00:00:00 2001 From: psilberk Date: Mon, 23 Jun 2025 15:44:34 -0700 Subject: [PATCH 197/277] Fix pom for demo --- .../semantickernel-syntax-examples/pom.xml | 12 ++++++++++++ samples/semantickernel-learn-resources/pom.xml | 6 ++++++ .../data/vectorstores/oracle/Main.java | 4 ++-- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 35efaffc..d5471730 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -147,6 +147,18 @@ 1.4.4-SNAPSHOT compile + + com.microsoft.semantic-kernel + semantickernel-data-postgres + 1.4.4-RC2-SNAPSHOT + compile + + + com.microsoft.semantic-kernel + semantickernel-data-oracle + 1.4.4-RC2-SNAPSHOT + compile + diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index b8a1ea0b..8d3cef89 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -94,6 +94,12 @@ 9.0.0 compile + + com.microsoft.semantic-kernel + semantickernel-data-postgres + 1.4.4-RC2-SNAPSHOT + compile + diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java index babc381f..570dfc60 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java @@ -68,10 +68,10 @@ public static void main(String[] args) throws SQLException { static List books = Arrays.asList( new Book("1", "one", "sking", 0, null, "horror", List.of(1f, 1f)), - new Book("2", "two", "squeen", 0, null, "non-fiction", List.of(-11f, -11f))); + new Book("2", "two", "squeen", 0, null, "non-fiction", List.of(-1f, -1f))); private static Mono> generateEmbeddingsAsync(String text) { - return Mono.just(List.of(-0.9f, -0.9f)); + return Mono.just(List.of(-0.1f, -0.1f)); } } \ No newline at end of file From b7d5ab3102ccadf9fe52672e1174627fb83aee70 Mon Sep 17 00:00:00 2001 From: psilberk Date: Tue, 24 Jun 2025 08:12:45 -0700 Subject: [PATCH 198/277] Fixing samples poms --- .../semantickernel-syntax-examples/pom.xml | 6 ++++++ samples/semantickernel-learn-resources/pom.xml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index ffc2adae..53b09d38 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -136,6 +136,12 @@ com.github.victools jsonschema-module-jackson + + com.microsoft.semantic-kernel + semantickernel-data-postgres + 1.4.4-RC2-SNAPSHOT + compile + diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 68062bf4..9d0cdb1b 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -85,6 +85,12 @@ 9.0.0 compile + + com.microsoft.semantic-kernel + semantickernel-data-postgres + 1.4.4-RC2-SNAPSHOT + compile + From 67aec2b3a8c317a11ff88dd0909e14e69df01990 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Wed, 25 Jun 2025 20:06:27 +0200 Subject: [PATCH 199/277] JsonTypeInfo annotation --- .../OracleVectorStoreQueryProvider.java | 18 +- .../oracle/OracleVectorStoreRecordMapper.java | 28 +++ .../jdbc/oracle/ClassWithAnnotatedTypes.java | 65 ++++++ .../OracleVectorStoreAnnotatedTypeTest.java | 200 ++++++++++++++++++ 4 files changed, 309 insertions(+), 2 deletions(-) create mode 100644 data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java create mode 100644 data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index f7a32fd5..fa606e64 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -51,6 +51,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -67,6 +68,8 @@ public class OracleVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider // This could be common to all query providers private final ObjectMapper objectMapper; + private final Map, String> annotatedTypeMapping; + private static final Object dbCreationLock = new Object(); private static final Logger logger = Logger.getLogger(OracleVectorStoreQueryProvider.class.getName()); @@ -97,7 +100,8 @@ private OracleVectorStoreQueryProvider( @Nonnull String prefixForCollectionTables, int defaultVarcharSize, @Nonnull StringTypeMapping stringTypeMapping, - ObjectMapper objectMapper) { + ObjectMapper objectMapper, + Map, String> annotatedTypeMapping) { super( dataSource, collectionsTable, @@ -108,6 +112,7 @@ private OracleVectorStoreQueryProvider( this.collectionsTable = collectionsTable; this.objectMapper = objectMapper; this.objectMapper.registerModule(new JavaTimeModule()); + this.annotatedTypeMapping = annotatedTypeMapping; } @Override @@ -569,6 +574,7 @@ public VectorStoreRecordMapper getVectorStoreRecordM .withRecordClass(recordClass) .withVectorStoreRecordDefinition(vectorStoreRecordDefinition) .withSupportedDataTypesMapping(getSupportedDataTypes()) + .withAnnotatedTypeMapping(annotatedTypeMapping) .build(); } @@ -586,6 +592,8 @@ public static class Builder private StringTypeMapping stringTypeMapping = StringTypeMapping.USE_VARCHAR; private int defaultVarcharSize = 2000; + private Map, String> annotatedTypeMapping = null; + @SuppressFBWarnings("EI_EXPOSE_REP2") public Builder withDataSource(DataSource dataSource) { @@ -619,6 +627,11 @@ public Builder withObjectMapper( return this; } + public Builder withAnnotatedTypeMapping(Map, String> annotatedTypeMapping) { + this.annotatedTypeMapping = annotatedTypeMapping; + return this; + } + /** * Sets the desired String type mapping. * @param stringTypeMapping the desired String type mapping. The default value is @@ -644,7 +657,8 @@ public Builder withDefaultVarcharSize (int defaultVarcharSize) { @Override public OracleVectorStoreQueryProvider build() { return new OracleVectorStoreQueryProvider(dataSource, collectionsTable, - prefixForCollectionTables, defaultVarcharSize, stringTypeMapping, objectMapper); + prefixForCollectionTables, defaultVarcharSize, stringTypeMapping, objectMapper, + annotatedTypeMapping); } } } \ No newline at end of file diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index a7b42745..e6e7bf0a 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -1,9 +1,11 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data.jdbc.oracle; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; @@ -69,6 +71,7 @@ public static class Builder private VectorStoreRecordDefinition vectorStoreRecordDefinition; private Map, String> supportedDataTypesMapping; private ObjectMapper objectMapper = new ObjectMapper(); + private Map, String> annotatedTypeMapping; /** * Sets the record class. @@ -118,6 +121,11 @@ public Builder withSupportedDataTypesMapping( return this; } + public Builder withAnnotatedTypeMapping(Map, String> annotatedTypeMapping) { + this.annotatedTypeMapping = annotatedTypeMapping; + return this; + } + /** * Builds the {@link OracleVectorStoreRecordMapper}. * @@ -142,6 +150,14 @@ public OracleVectorStoreRecordMapper build() { for (VectorStoreRecordField field : vectorStoreRecordDefinition.getNonVectorFields()) { Class fieldType = field.getFieldType(); + boolean isAnnotated = false; + try { + isAnnotated = recordClass.getDeclaredField(field.getName()) + .isAnnotationPresent(JsonTypeInfo.class); + } catch (NoSuchFieldException e) { + // ignore exception, assume the field is not annotated + } + Object value; switch (supportedDataTypesMapping.get(fieldType)) { case OracleDataTypesMapping.STRING_CLOB: @@ -204,6 +220,18 @@ public OracleVectorStoreRecordMapper build() { JsonNode genericNode = objectMapper.valueToTree(value); objectNode.set(field.getEffectiveStorageName(), genericNode); + if (isAnnotated) { + if (annotatedTypeMapping != null && annotatedTypeMapping.containsKey(field.getFieldType())) { + objectNode.set(field.getEffectiveStorageName() + "_type", + TextNode.valueOf( + annotatedTypeMapping.get(field.getFieldType()))); + } else { + objectNode.set(field.getEffectiveStorageName() + "_type", + TextNode.valueOf( + field.getFieldType().getName())); + + } + } } if (options != null && options.isIncludeVectors()) { for (VectorStoreRecordVectorField field : vectorStoreRecordDefinition.getVectorFields()) { diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java new file mode 100644 index 00000000..318911d8 --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java @@ -0,0 +1,65 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo.As; +import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; + +import java.math.BigDecimal; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.UUID; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class ClassWithAnnotatedTypes { + + private final String id; + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = As.EXTERNAL_PROPERTY , property = "value_type") + @JsonSubTypes({ + @JsonSubTypes.Type(value = String.class, name="java.lang.String"), + @JsonSubTypes.Type(value = Boolean.class, name="java.lang.Boolean"), + @JsonSubTypes.Type(value = Byte.class, name="java.lang.Byte"), + @JsonSubTypes.Type(value = Short.class, name="java.lang.Short"), + @JsonSubTypes.Type(value = Integer.class, name="java.lang.Integer"), + @JsonSubTypes.Type(value = Long.class, name="java.lang.Long"), + @JsonSubTypes.Type(value = Float.class, name="java.lang.Float"), + @JsonSubTypes.Type(value = Double.class, name="java.lang.Double"), + @JsonSubTypes.Type(value = BigDecimal.class, name="java.math.BigDecimal"), + @JsonSubTypes.Type(value = OffsetDateTime.class, name="java.time.OffsetDateTime"), + @JsonSubTypes.Type(value = UUID.class, name="java.util.UUID"), + @JsonSubTypes.Type(value = byte[].class, name="byte_array"), + @JsonSubTypes.Type(value = List.class, name="listOfStrings") + }) + private Object value; + + private final Float[] vectorValue; + + + public ClassWithAnnotatedTypes() { + this(null, null, null); + }; + public ClassWithAnnotatedTypes(String id, Object value, Float[] vectorValue) { + this.id = id; + this.value = value; + this.vectorValue = vectorValue; + } + + public String getId() { + return id; + } + + public Object getValue() { + return value; + } + + public Float[] getVectorValue() { + return vectorValue; + } +} diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java new file mode 100644 index 00000000..9650652e --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java @@ -0,0 +1,200 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; + +public class OracleVectorStoreAnnotatedTypeTest extends OracleCommonVectorStoreRecordCollectionTest { + + @ParameterizedTest + @MethodSource("supportedDataTypes") + void testDataTypes(String dataFieldName, Class dataFieldType, Object dataFieldValue, Class fieldSubType) { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + VectorStoreRecordDataField dataField; + if (fieldSubType != null) { + dataField = VectorStoreRecordDataField.builder() + .withName("value") + .withStorageName("value") + .withFieldType(dataFieldType, fieldSubType) + .isFilterable(true) + .build(); + } else { + dataField = VectorStoreRecordDataField.builder() + .withName("value") + .withStorageName("value") + .withFieldType(dataFieldType) + .isFilterable(true) + .build(); + } + + VectorStoreRecordVectorField dummyVector = VectorStoreRecordVectorField.builder() + .withName("vectorValue") + .withStorageName("vectorValue") + .withFieldType(Float[].class) + .withDimensions(8) + .withDistanceFunction(DistanceFunction.COSINE_DISTANCE) + .withIndexKind(IndexKind.IVFFLAT) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dataField, dummyVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .withAnnotatedTypeMapping(new HashMap() {{ + put(List.class, "listOfStrings"); + }}) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "test_datatype_" + dataFieldName; + + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions. builder() + .withRecordClass(ClassWithAnnotatedTypes.class) + .withRecordDefinition(definition).build()); + + collection.createCollectionAsync().block(); + + String key = "testid"; + + ClassWithAnnotatedTypes record = + new ClassWithAnnotatedTypes(key, dataFieldValue, new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f }); + + collection.upsertAsync(record, null).block(); + + ClassWithAnnotatedTypes result = collection.getAsync(key, null).block(); + assertNotNull(result); + if (record.getValue().getClass().equals(OffsetDateTime.class)) { + assertTrue(((OffsetDateTime)dataFieldValue).isEqual((OffsetDateTime)record.getValue())); + } else { + assertEquals(dataFieldValue, result.getValue()); + } + + collection.deleteCollectionAsync().block(); + } + + @Test + void testByteArrayType() { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + VectorStoreRecordDataField dataField; + dataField = VectorStoreRecordDataField.builder() + .withName("value") + .withStorageName("value") + .withFieldType(byte[].class) + .isFilterable(true) + .build(); + + VectorStoreRecordVectorField dummyVector = VectorStoreRecordVectorField.builder() + .withName("vectorValue") + .withStorageName("vectorValue") + .withFieldType(Float[].class) + .withDimensions(8) + .withDistanceFunction(DistanceFunction.COSINE_DISTANCE) + .withIndexKind(IndexKind.IVFFLAT) + .build(); + + VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField, dataField, dummyVector) + ); + + OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE) + .withAnnotatedTypeMapping(new HashMap() {{ + put(byte[].class, "byte_array"); + }}) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "test_datatype_bytearray"; + + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions. builder() + .withRecordClass(ClassWithAnnotatedTypes.class) + .withRecordDefinition(definition).build()); + + collection.createCollectionAsync().block(); + + String key = "testid"; + + byte[] dataFieldValue = new byte[] {1, 2, 3}; + ClassWithAnnotatedTypes record = + new ClassWithAnnotatedTypes(key, dataFieldValue, new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f }); + + collection.upsertAsync(record, null).block(); + + ClassWithAnnotatedTypes result = collection.getAsync(key, null).block(); + assertNotNull(result); + assertArrayEquals(dataFieldValue, (byte[])result.getValue()); + + collection.deleteCollectionAsync().block(); + } + + + + private static Stream supportedDataTypes() { + return Stream.of( + Arguments.of("string", String.class, "asd123", null), + Arguments.of("boolean_true", Boolean.class, true, null), + Arguments.of("boolean_false", Boolean.class, false, null), + Arguments.of("byte", Byte.class, (byte) 127, null), + Arguments.of("short", Short.class, (short) 3, null), + Arguments.of("integer", Integer.class, 321, null), + Arguments.of("long", Long.class, 5L, null), + Arguments.of("float", Float.class, 3.14f, null), + Arguments.of("double", Double.class, 3.14159265358d, null), + Arguments.of("decimal", BigDecimal.class, new BigDecimal("12345.67"), null), + Arguments.of("timestamp", OffsetDateTime.class, OffsetDateTime.now(), null), + Arguments.of("uuid", UUID.class, UUID.randomUUID(), null), + Arguments.of("json", List.class, Arrays.asList("a", "s", "d"), String.class) + ); + } + +} From 0e044a08e52ecbb4bd694c2eabff7f501467f3b8 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Thu, 26 Jun 2025 18:59:06 +0200 Subject: [PATCH 200/277] Javadod and comments + map UUID to string and other small changes --- data/semantickernel-data-oracle/pom.xml | 30 +- .../jdbc/oracle/OracleDataTypesMapping.java | 58 ++- .../oracle/OracleVectorStoreFieldHelper.java | 206 +++------- .../OracleVectorStoreQueryProvider.java | 381 ++++++++++++------ .../oracle/OracleVectorStoreRecordMapper.java | 41 +- .../jdbc/oracle/ClassWithAllBoxedTypes.java | 6 + .../oracle/ClassWithAllPrimitiveTypes.java | 6 + .../jdbc/oracle/ClassWithAnnotatedTypes.java | 49 ++- .../data/jdbc/oracle/Hotel.java | 3 + ...CommonVectorStoreRecordCollectionTest.java | 6 + .../OracleVectorStoreAnnotatedTypeTest.java | 102 ++--- .../OracleVectorStoreDataTypeSearchTest.java | 11 +- .../oracle/OracleVectorStoreDataTypeTest.java | 6 + ...OracleVectorStoreRecordCollectionTest.java | 6 + 14 files changed, 482 insertions(+), 429 deletions(-) diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml index 2a642271..9f7223f3 100644 --- a/data/semantickernel-data-oracle/pom.xml +++ b/data/semantickernel-data-oracle/pom.xml @@ -56,38 +56,10 @@ jackson-datatype-jsr310 2.18.0 - - com.github.jknack - handlebars - - - com.google.code.findbugs - jsr305 - provided - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - compile - com.github.spotbugs spotbugs-annotations - - org.apache.commons - commons-text - - - org.postgresql - postgresql - 42.7.4 - - - org.xerial - sqlite-jdbc - 3.47.0.0 - com.oracle.database.jdbc ojdbc11 @@ -98,6 +70,7 @@ ojdbc-provider-jackson-oson 1.0.4 + org.junit.jupiter junit-jupiter @@ -108,7 +81,6 @@ junit-jupiter-api test - org.testcontainers testcontainers diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java index 0258cb5f..cfca9ecc 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java @@ -1,22 +1,74 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; /** - * Defines oracle database type constants for supported field types. + * Defines oracle database type constants for supported java types. */ public class OracleDataTypesMapping { - public static final String STRING_VARCHAR = "NVARCHAR2(%s)"; + + /** + * Oracle database type used when strings are mapped to VARCHAR + */ + public static final String STRING_VARCHAR = "VARCHAR2(%s)"; + /** + * Oracle database type used when strings are mapped to CLOB + */ public static final String STRING_CLOB = "CLOB"; + /** + * Oracle database type used to map booleans + */ public static final String BOOLEAN = "BOOLEAN"; + /** + * Oracle database type used to map bytes + */ public static final String BYTE = "NUMBER(3)"; + /** + * Oracle database type used to map byte arrays + */ public static final String BYTE_ARRAY = "RAW(2000)"; + /** + * Oracle database type used to map shorts + */ public static final String SHORT = "NUMBER(5)"; + /** + * Oracle database type used to map ints + */ public static final String INTEGER = "NUMBER(10)"; + /** + * Oracle database type used to map longs + */ public static final String LONG = "NUMBER(19)"; + /** + * Oracle database type used to map float + */ public static final String FLOAT = "BINARY_FLOAT"; + /** + * Oracle database type used to map double + */ public static final String DOUBLE = "BINARY_DOUBLE"; + /** + * Oracle database type used to map BigDecimal + */ public static final String DECIMAL = "NUMBER"; + /** + * Oracle database type used to map offset date time + */ public static final String OFFSET_DATE_TIME = "TIMESTAMP(7) WITH TIME ZONE"; - public static final String UUID = "RAW(16)"; + /** + * Oracle database type used to map UUID + */ + public static final String UUID = "VARCHAR2(36)"; + /** + * Oracle database type used to map lists + */ public static final String JSON = "JSON"; + /** + * Oracle database type used to map vectors (the parameter is the dimension of the vector) + */ public static final String VECTOR_FLOAT = "VECTOR(%s, FLOAT32)"; } diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java index dbc77178..c177131f 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java @@ -1,3 +1,9 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider.StringTypeMapping; @@ -17,10 +23,15 @@ import java.util.stream.Collectors; /** - * Helper class for field operations. + * Helper class for field operations. Handles mapping between field java types to DB types and + * generating SQL statement to create field indexes. */ -public class OracleVectorStoreFieldHelper { - private static final Logger LOGGER = Logger.getLogger(OracleVectorStoreQueryProvider.class.getName()); +class OracleVectorStoreFieldHelper { + + /** + * The logger + */ + private static final Logger LOGGER = Logger.getLogger(OracleVectorStoreFieldHelper.class.getName()); /** * Maps supported key java classes to Oracle database types @@ -28,13 +39,6 @@ public class OracleVectorStoreFieldHelper { private static final HashMap, String> supportedKeyTypes = new HashMap() { { put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, 255)); - put(short.class, OracleDataTypesMapping.SHORT); - put(Short.class, OracleDataTypesMapping.SHORT); - put(int.class, OracleDataTypesMapping.INTEGER); - put(Integer.class, OracleDataTypesMapping.INTEGER); - put(long.class, OracleDataTypesMapping.LONG); - put(Long.class, OracleDataTypesMapping.LONG); - put(UUID .class, OracleDataTypesMapping.UUID); } }; @@ -48,14 +52,6 @@ public class OracleVectorStoreFieldHelper { put(Collection.class, OracleDataTypesMapping.VECTOR_FLOAT); put(float[].class, OracleDataTypesMapping.VECTOR_FLOAT); put(Float[].class, OracleDataTypesMapping.VECTOR_FLOAT); -/* - put(byte[].class,"VECTOR(%s, INT8)"); - put(Byte[].class,"VECTOR(%s, INT8)"); - put(double[].class,"VECTOR(%s, FLOAT64)"); - put(Double[].class,"VECTOR(%s, FLOAT64)"); - put(boolean[].class,"VECTOR(%s, BINARY)"); - put(Boolean[].class,"VECTOR(%s, BINARY)"); - */ } }; @@ -84,26 +80,12 @@ public class OracleVectorStoreFieldHelper { put(byte[].class, OracleDataTypesMapping.BYTE_ARRAY); put(List.class, OracleDataTypesMapping.JSON); } - }; /** - * Maps vector type to OracleTypes. Only needed if types other than FLOAT_32 are supported. + * Suffix added to the effective column name to generate the index name for a vector column. */ - private static final Map, Integer> mapOracleTypeToVector = new HashMap() { - { - put(float[].class, OracleTypes.VECTOR_FLOAT32); - put(Float[].class, OracleTypes.VECTOR_FLOAT32); -/* - put(byte[].class, OracleTypes.VECTOR_INT8); - put(Byte[].class, OracleTypes.VECTOR_INT8); - put(Double[].class, OracleTypes.VECTOR_FLOAT64); - put(double[].class, OracleTypes.VECTOR_FLOAT64); - put(Boolean[].class, OracleTypes.VECTOR_BINARY); - put(boolean[].class, OracleTypes.VECTOR_BINARY); -*/ - } - }; + public static final String VECTOR_INDEX_SUFFIX = "_VECTOR_INDEX"; /** * Gets the mapping between the supported Java key types and the Oracle database type. @@ -121,12 +103,11 @@ public static HashMap, String> getSupportedKeyTypes() { */ public static Map, String> getSupportedDataTypes( StringTypeMapping stringTypeMapping, int defaultVarCharLength) { - - if (stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR)) { - supportedDataTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, defaultVarCharLength)); - } else { - supportedDataTypes.put(String.class, OracleDataTypesMapping.STRING_CLOB); - } + String stringType = stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR) + ? String.format(OracleDataTypesMapping.STRING_VARCHAR, defaultVarCharLength) + : OracleDataTypesMapping.STRING_CLOB; + supportedDataTypes.put(String.class, stringType); + LOGGER.finest("Mapping String columns to " + stringType); return supportedDataTypes; } @@ -177,14 +158,14 @@ public static String getCreateVectorIndexStatement(VectorStoreRecordVectorField */ public static String createIndexForDataField(String collectionTableName, VectorStoreRecordDataField dataField, Map, String> supportedDataTypes) { if (supportedDataTypes.get(dataField.getFieldType()) == "JSON") { - String dataFieldIndex = "CREATE MULTIVALUE INDEX %s ON %s t (t.%s.%s)"; + String dataFieldIndex = "CREATE MULTIVALUE INDEX IF NOT EXISTS %s ON %s t (t.%s.%s)"; return String.format(dataFieldIndex, collectionTableName + "_" + dataField.getEffectiveStorageName(), collectionTableName, dataField.getEffectiveStorageName(), getFunctionForType(supportedDataTypes.get(dataField.getFieldSubType()))); } else { - String dataFieldIndex = "CREATE INDEX %s ON %s (%s ASC)"; + String dataFieldIndex = "CREATE INDEX IF NOT EXISTS %s ON %s (%s ASC)"; return String.format(dataFieldIndex, collectionTableName + "_" + dataField.getEffectiveStorageName(), collectionTableName, @@ -194,95 +175,28 @@ public static String createIndexForDataField(String collectionTableName, VectorS } /** - * Gets the function that allows to return the function that converts the JSON value to the - * data type. - * @param jdbcType The JDBC type. - * @return the function that allows to return the function that converts the JSON value to the - * data type. + * Returns vector columns names and types for CREATE TABLE statement + * @param fields list of vector record fields. + * @return comma separated list of columns and types for CREATE TABLE statement. */ - private static String getFunctionForType(String jdbcType) { - switch (jdbcType) { - case OracleDataTypesMapping.BOOLEAN: - return "boolean()"; - case OracleDataTypesMapping.BYTE: - case OracleDataTypesMapping.SHORT: - case OracleDataTypesMapping.INTEGER: - case OracleDataTypesMapping.LONG: - case OracleDataTypesMapping.FLOAT: - case OracleDataTypesMapping.DOUBLE: - case OracleDataTypesMapping.DECIMAL: - return "numberOnly()"; - case OracleDataTypesMapping.OFFSET_DATE_TIME: - return "timestamp()"; - default: - return "string()"; - } - } + public static String getVectorColumnNamesAndTypes(List fields) { + List columns = fields.stream() + .map(field -> field.getEffectiveStorageName() + " " + + OracleVectorStoreFieldHelper.getTypeForVectorField(field) + ).collect(Collectors.toList()); - /** - * Gets the type of the vector given the field definition. This method is not needed if only - * - * @param field the vector field definition. - * @return returns the type of vector for the given field type. - */ - public static String getTypeForVectorField(VectorStoreRecordVectorField field) { - String dimension = field.getDimensions() > 0 ? String.valueOf(field.getDimensions()) : "*"; - return String.format(supportedVectorTypes.get(field.getFieldType()), dimension); -/* Not needed since all types are FLOAT32 - if (field.getFieldSubType() != null) { - String vectorType; - switch (field.getFieldSubType().getName()) { - case "java.lang.Double": - vectorType = "FLOAT64"; - break; - case "java.lang.Byte": - vectorType = "INT8"; - break; - case "java.lang.Boolean": - vectorType = "BINARY"; - break; - default: - vectorType = "FLOAT32"; - } - return String.format(supportedVectorTypes.get(field.getFieldType()), dimension, vectorType); - } else { - return String.format(supportedVectorTypes.get(field.getFieldType()), dimension); - } - */ + return String.join(", ", columns); } /** - * Gets the JDBC oracle of the vector field definition. - * @param field the vector field definition. - * @return the JDBC oracle type. + * Returns key column names and type for key column for CREATE TABLE statement + * @param field the key field. + * @return column name and type of the key field for CREATE TABLE statement. */ - public static int getOracleTypeForVectorField(VectorStoreRecordVectorField field) { - if (field.getFieldSubType() == null) { - Integer oracleType = mapOracleTypeToVector.get(field.getFieldType()); - if (oracleType != null) { - return oracleType.intValue(); - } else { - // field was declared as list with no subtype, assume FLOAT - return OracleTypes.VECTOR_FLOAT32; - } - - } else { - switch (field.getFieldSubType().getName()) { - case "java.lang.Double": - return OracleTypes.VECTOR_FLOAT64; - case "java.lang.Byte": - return OracleTypes.VECTOR_INT8; - case "java.lang.Boolean": - return OracleTypes.VECTOR_BINARY; - default: - return OracleTypes.VECTOR_FLOAT32; - } - } + public static String getKeyColumnNameAndType(VectorStoreRecordKeyField field) { + return field.getEffectiveStorageName() + " " + supportedKeyTypes.get(field.getFieldType()); } - public static boolean isUUID (VectorStoreRecordField field) { - return (field.getFieldType().getName() == "java.util.UUID"); - } /** * Generates the index name given the field name. by suffixing "_VECTOR_INDEX" to the field name. @@ -290,30 +204,44 @@ public static boolean isUUID (VectorStoreRecordField field) { * @return the index name. */ private static String getIndexName(String effectiveStorageName) { - return effectiveStorageName + "_VECTOR_INDEX"; + return effectiveStorageName + VECTOR_INDEX_SUFFIX; } /** - * Returns vector columns names and types for CREATE TABLE statement - * @param fields list of vector record fields. - * @return comma separated list of columns and types for CREATE TABLE statement. + * Gets the type of the vector given the field definition. This method is not needed if only + * + * @param field the vector field definition. + * @return returns the type of vector for the given field type. */ - public static String getVectorColumnNamesAndTypes(List fields) { - List columns = fields.stream() - .map(field -> field.getEffectiveStorageName() + " " + - OracleVectorStoreFieldHelper.getTypeForVectorField(field) - ).collect(Collectors.toList()); - - return String.join(", ", columns); + private static String getTypeForVectorField(VectorStoreRecordVectorField field) { + String dimension = field.getDimensions() > 0 ? String.valueOf(field.getDimensions()) : "*"; + return String.format(supportedVectorTypes.get(field.getFieldType()), dimension); } /** - * Returns key column names and type for key column for CREATE TABLE statement - * @param field the key field. - * @return column name and type of the key field for CREATE TABLE statement. + * Gets the function that allows to return the function that converts the JSON value to the + * data type. + * @param jdbcType The JDBC type. + * @return the function that allows to return the function that converts the JSON value to the + * data type. */ - public static String getKeyColumnNameAndType(VectorStoreRecordKeyField field) { - return field.getEffectiveStorageName() + " " + supportedKeyTypes.get(field.getFieldType()); + private static String getFunctionForType(String jdbcType) { + switch (jdbcType) { + case OracleDataTypesMapping.BOOLEAN: + return "boolean()"; + case OracleDataTypesMapping.BYTE: + case OracleDataTypesMapping.SHORT: + case OracleDataTypesMapping.INTEGER: + case OracleDataTypesMapping.LONG: + case OracleDataTypesMapping.FLOAT: + case OracleDataTypesMapping.DOUBLE: + case OracleDataTypesMapping.DECIMAL: + return "numberOnly()"; + case OracleDataTypesMapping.OFFSET_DATE_TIME: + return "timestamp()"; + default: + return "string()"; + } } } diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index fa606e64..eba71db7 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -1,12 +1,16 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.util.ISO8601DateFormat; -import com.fasterxml.jackson.databind.util.StdDateFormat; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; import com.microsoft.semantickernel.data.filter.EqualToFilterClause; @@ -28,31 +32,24 @@ import oracle.jdbc.OraclePreparedStatement; import oracle.jdbc.OracleStatement; import oracle.jdbc.OracleTypes; -import oracle.sql.TIMESTAMPLTZ; import oracle.sql.TIMESTAMPTZ; import javax.annotation.Nonnull; import javax.annotation.concurrent.GuardedBy; import javax.sql.DataSource; import java.math.BigDecimal; -import java.nio.ByteBuffer; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.sql.SQLType; import java.sql.Statement; import java.sql.SQLException; -import java.sql.Timestamp; -import java.text.SimpleDateFormat; -import java.time.LocalDateTime; import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.locks.ReentrantLock; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -68,11 +65,15 @@ public class OracleVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider // This could be common to all query providers private final ObjectMapper objectMapper; - private final Map, String> annotatedTypeMapping; - - private static final Object dbCreationLock = new Object(); + /** + * Lock used to ensure that only one thread can create a collection at a time. + */ + private static final ReentrantLock dbCreationLock = new ReentrantLock(); - private static final Logger logger = Logger.getLogger(OracleVectorStoreQueryProvider.class.getName()); + /** + * The logger + */ + private static final Logger LOGGER = Logger.getLogger(OracleVectorStoreQueryProvider.class.getName()); public enum StringTypeMapping { /** @@ -86,8 +87,9 @@ public enum StringTypeMapping { } /** - * Constructor - * @param dataSource the datasiyrce + * Create an instance of OracleVectorStoreQueryProvider. + * + * @param dataSource the datasource * @param collectionsTable the collections table name * @param prefixForCollectionTables the prefix for the collection table name * @param defaultVarcharSize the size of VARCHAR columns @@ -100,8 +102,7 @@ private OracleVectorStoreQueryProvider( @Nonnull String prefixForCollectionTables, int defaultVarcharSize, @Nonnull StringTypeMapping stringTypeMapping, - ObjectMapper objectMapper, - Map, String> annotatedTypeMapping) { + ObjectMapper objectMapper) { super( dataSource, collectionsTable, @@ -111,17 +112,33 @@ private OracleVectorStoreQueryProvider( OracleVectorStoreFieldHelper.getSupportedVectorTypes()); this.collectionsTable = collectionsTable; this.objectMapper = objectMapper; + // The JavaTimeModule must be registered to handle OffsetDateTime. To make sure that it is + // registered enable the feature IGNORE_DUPLICATE_MODULE_REGISTRATIONS and register the + // module. + this.objectMapper.enable(MapperFeature.IGNORE_DUPLICATE_MODULE_REGISTRATIONS); this.objectMapper.registerModule(new JavaTimeModule()); - this.annotatedTypeMapping = annotatedTypeMapping; } + /** + *

+ * Creates a collection with the given name and record definition. + *

+ * A collection is represented as a table in an Oracle DB containing columns + * that match the record definition. The table name is the name of the collection + * prefixed by the provided collection prefix. If no prefix was provided the default + * prefix will be used. + *

+ * @param collectionName the name of the collection + * @param recordDefinition the record definition + */ @Override @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") @GuardedBy("dbCreationLock") public void createCollection(String collectionName, VectorStoreRecordDefinition recordDefinition) { - synchronized (dbCreationLock) { + dbCreationLock.lock(); + try { List vectorFields = recordDefinition.getVectorFields(); String createStorageTable = formatQuery("CREATE TABLE IF NOT EXISTS %s (" @@ -138,39 +155,47 @@ public void createCollection(String collectionName, String insertCollectionQuery = this.getInsertCollectionQuery(collectionsTable); try (Connection connection = dataSource.getConnection()) { - connection.createStatement().execute(formatQuery("DROP TABLE IF EXISTS %s", getCollectionTableName(collectionName))); + // set auto commit of, either all statements should be executed or none connection.setAutoCommit(false); try (Statement statement = connection.createStatement()) { // Create table - System.out.println(createStorageTable); statement.addBatch(createStorageTable); + LOGGER.finest("Creating collection " + collectionName + + " using statement: " + createStorageTable); - // Index filterable columns + // Index filterable data columns for (VectorStoreRecordDataField dataField : recordDefinition.getDataFields()) { if (dataField.isFilterable()) { String dataFieldIndex = OracleVectorStoreFieldHelper.createIndexForDataField( getCollectionTableName(collectionName), dataField, supportedDataTypes); - System.out.println(dataFieldIndex); statement.addBatch(dataFieldIndex); + LOGGER.finest("Creating index on column " + + dataField.getEffectiveStorageName() + " using the statement: " + + dataFieldIndex); } } - // Create indexed for vectorFields + // Create index for vectorFields for (VectorStoreRecordVectorField vectorField : vectorFields) { String createVectorIndex = OracleVectorStoreFieldHelper.getCreateVectorIndexStatement( vectorField, getCollectionTableName(collectionName)); if (createVectorIndex != null) { - System.out.println(createVectorIndex); statement.addBatch(createVectorIndex); + LOGGER.finest("Creating index on vector column " + + vectorField.getEffectiveStorageName() + " using the statement: " + + createVectorIndex); + } } statement.executeBatch(); + // Insert the collection to the store (collections table) using MERGE statement try (PreparedStatement insert = connection.prepareStatement( insertCollectionQuery)) { - System.out.println(insertCollectionQuery); insert.setString(1, collectionName); insert.execute(); + LOGGER.finest("Inserting collection to store using statement: " + + insertCollectionQuery); } connection.commit(); @@ -181,161 +206,216 @@ public void createCollection(String collectionName, } catch (SQLException e) { throw new SKException("Failed to create collection", e); } + } finally { + dbCreationLock.unlock(); } } + /** + *

+ * Inserts or updates record of a collection given the collection name, the records, the record + * definition and the upsert options. + *

+ * @Note At the moment {@link UpsertRecordOptions} is an empty class. No options are available. + *

+ * + * @param collectionName the collection name + * @param records the records to update or insert + * @param recordDefinition the record definition + * @param options the options + */ @Override - protected String getInsertCollectionQuery(String collectionsTable) { - return formatQuery( - "MERGE INTO %s existing "+ - "USING (SELECT ? AS collectionId FROM DUAL) new ON (existing.collectionId = new.collectionId) " + - "WHEN NOT MATCHED THEN INSERT (existing.collectionId) VALUES (new.collectionId)", - collectionsTable); - } - - @Override - public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { + public void upsertRecords(String collectionName, + List records, + VectorStoreRecordDefinition recordDefinition, + UpsertRecordOptions options) { final String NEW_VALUE = "new"; final String EXISTING_VALUE = "existing"; + // generate the comma separated list of new fields + // Ex.: new.field1, new.field2 ... new.fieldn String insertNewFieldList = recordDefinition.getAllFields().stream() .map(f -> NEW_VALUE + "." + f.getEffectiveStorageName()) .collect(Collectors.joining(", ")); + // generate the comma separated list of existing fields + // Ex.: existing.field1, existing.field2 ... existing.fieldn String insertExistingFieldList = recordDefinition.getAllFields().stream() .map(f -> EXISTING_VALUE + "." + f.getEffectiveStorageName()) .collect(Collectors.joining(", ")); + // generate the comma separated list for setting new values on fields + // Ex.: new.field1 = existing.field1, new.field2 = existing.field2 ... new.fieldn = existing.fieldn String updateFieldList = recordDefinition.getAllFields().stream() .filter(f -> f != recordDefinition.getKeyField()) .map(f -> EXISTING_VALUE + "." + f.getEffectiveStorageName() + " = " + NEW_VALUE + "." + f.getEffectiveStorageName()) .collect(Collectors.joining(", ")); - String namedWildcard = recordDefinition.getAllFields().stream().map(f -> "? " + f.getEffectiveStorageName()) + // generate the comma separated list of placeholders "?" for each field + // Ex.: ? field1, ? field2 ... ? fieldn + String namedPlaceholders = recordDefinition.getAllFields().stream().map(f -> "? " + f.getEffectiveStorageName()) .collect(Collectors.joining(", ")); - String upsertQuery = formatQuery("MERGE INTO %s existing "+ + // Generate the MERGE statement to perform the upsert. + String upsertStatement = formatQuery("MERGE INTO %s existing "+ "USING (SELECT %s FROM DUAL) new ON (existing.%s = new.%s) " + "WHEN MATCHED THEN UPDATE SET %s " + "WHEN NOT MATCHED THEN INSERT (%s) VALUES (%s)", getCollectionTableName(collectionName), - namedWildcard, + namedPlaceholders, getKeyColumnName(recordDefinition.getKeyField()), getKeyColumnName(recordDefinition.getKeyField()), updateFieldList, insertExistingFieldList, insertNewFieldList); - System.out.println(upsertQuery); + LOGGER.finest("Generated upsert statement: " + upsertStatement); try (Connection connection = dataSource.getConnection(); - PreparedStatement statement = connection.prepareStatement(upsertQuery)) { + PreparedStatement statement = connection.prepareStatement(upsertStatement)) { + // Loop through records, set values and add values to batch for (Object record : records) { setUpsertStatementValues(statement, record, recordDefinition.getAllFields()); statement.addBatch(); } + // Execute the upsert statement statement.executeBatch(); } catch (SQLException e) { throw new SKException("Failed to upsert records", e); } } - private void setUpsertStatementValues(PreparedStatement statement, Object record, + /** + * Generates the MERGE statement to add the given collection to the store. + * + * @param collectionsTable the name of the DB table containing all collections. + * @return a SQL statement that inserts a collection to the store if it does not exist. + */ + @Override + protected String getInsertCollectionQuery(String collectionsTable) { + return formatQuery( + "MERGE INTO %s existing "+ + "USING (SELECT ? AS collectionId FROM DUAL) new ON (existing.collectionId = new.collectionId) " + + "WHEN NOT MATCHED THEN INSERT (existing.collectionId) VALUES (new.collectionId)", + collectionsTable); + } + + /** + * The {@link OracleVectorStoreQueryProvider#upsertRecords(String, List, VectorStoreRecordDefinition, UpsertRecordOptions)} + * method adds a placeholder for each field. This method sets the value of each field on the + * MERGE statement with the value of the record. The placeholder and values are set in the order + * of the fields in the list. + * + * @param upsertStatement the MERGE statement + * @param record the record containing the values + * @param fields the list of fields. + */ + private void setUpsertStatementValues(PreparedStatement upsertStatement, Object record, List fields) { - objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); - String datePattern = "yyyy-MM-dd HH:mm:ss.SSSZ"; - SimpleDateFormat df = new SimpleDateFormat(datePattern); - objectMapper.setDateFormat(df); + + // use the object mapper to convert the record to an equivalent tree mode JsonNode value, + // this allows to retrieve the values using the effective storage name of the fields and + // avoids the use of introspection. JsonNode jsonNode = objectMapper.valueToTree(record); for (int i = 0; i < fields.size(); ++i) { VectorStoreRecordField field = fields.get(i); try { + JsonNode valueNode = jsonNode.get(field.getEffectiveStorageName()); + // Some field types require special treatment to convert the java type to the + // DB type if (field instanceof VectorStoreRecordVectorField) { - // Convert the vector field to a string + // If the vector field is not set as a string convert to an array of doubles + // and set the value if (!field.getFieldType().equals(String.class)) { double[] values = (valueNode == null || valueNode.isNull()) ? null : StreamSupport.stream(( (ArrayNode)valueNode).spliterator(), false) .mapToDouble(d -> d.asDouble()).toArray(); - statement.setObject(i + 1, values, - OracleVectorStoreFieldHelper.getOracleTypeForVectorField((VectorStoreRecordVectorField)field)); - System.out.println("Set values: " + values); + upsertStatement.setObject(i + 1, values, OracleTypes.VECTOR_FLOAT32); continue; } } else if (field instanceof VectorStoreRecordDataField) { - // Convert List field to a string + // Lists are stored as JSON objects, write the list as a JSON string representation + // of the list. if (field.getFieldType().equals(List.class)) { - statement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); - System.out.println( - "Set values: " + objectMapper.writeValueAsString(valueNode)); + upsertStatement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); continue; } - if (OracleVectorStoreFieldHelper.isUUID(field)) { - if (valueNode == null || valueNode.isNull()) { - statement.setNull(i + 1, OracleTypes.RAW); - } else { - UUID uuid = UUID.fromString(valueNode.textValue()); - ByteBuffer bb = ByteBuffer.allocate(16); - bb.putLong(uuid.getMostSignificantBits()); - bb.putLong(uuid.getLeastSignificantBits()); - statement.setBytes(i + 1, bb.array()); - System.out.println("Set values: " + objectMapper.convertValue(valueNode, - field.getFieldType())); - } + // Convert UUID to string before setting the value. + if (field.getFieldType().equals(UUID.class)) { + upsertStatement.setObject(i + 1, valueNode.isNull() ? null : valueNode.asText()); continue; } + // Convert OffsetDateTime to TIMESTAMPTZ before setting the value. if (field.getFieldType().equals(OffsetDateTime.class)) { if (valueNode == null || valueNode.isNull()) { - statement.setNull(i + 1, OracleTypes.TIMESTAMPTZ); + upsertStatement.setNull(i + 1, OracleTypes.TIMESTAMPTZ); } else { - OffsetDateTime offsetDateTime = OffsetDateTime.parse( - valueNode.asText()); - ((OraclePreparedStatement) statement).setTIMESTAMPTZ(i + 1, + OffsetDateTime offsetDateTime = (OffsetDateTime) objectMapper.convertValue(valueNode, field.getFieldType()); + ((OraclePreparedStatement) upsertStatement).setTIMESTAMPTZ(i + 1, TIMESTAMPTZ.of(offsetDateTime)); - System.out.println("Set values: " + objectMapper.convertValue(valueNode, - field.getFieldType())); } continue; } } - statement.setObject(i + 1, + // For all other field type use setObject with the field value + upsertStatement.setObject(i + 1, objectMapper.convertValue(valueNode,field.getFieldType())); - System.out.println("Set values: " + objectMapper.convertValue(valueNode, field.getFieldType())); } catch (SQLException | JsonProcessingException e) { throw new RuntimeException(e); } } } + /** + *

+ * Executes a vector search query, using the search options and returns the results. The results + * are mapped to the specified record type using the provided mapper. The query is executed + * against the specified collection. + *

+ * + *

+ * @param collectionName the collection name + * @param vector the vector to search with + * @param options the search options + * @param recordDefinition the record definition + * @param mapper the mapper, responsible for mapping the result set to the record + * type. + * @return the search results + * @param the record type + */ @Override public VectorSearchResults search(String collectionName, List vector, VectorSearchOptions options, VectorStoreRecordDefinition recordDefinition, VectorStoreRecordMapper mapper) { - VectorStoreRecordVectorField firstVectorField = recordDefinition.getVectorFields() - .get(0); + // Gets the search vector field and its distance function. If not vector field was provided, + // use the first one VectorStoreRecordVectorField vectorField = options.getVectorFieldName() == null - ? firstVectorField + ? recordDefinition.getVectorFields().get(0) : (VectorStoreRecordVectorField) recordDefinition .getField(options.getVectorFieldName()); - DistanceFunction distanceFunction = vectorField.getDistanceFunction(); - - List fields; - if (options.isIncludeVectors()) { - fields = recordDefinition.getAllFields(); - } else { - fields = recordDefinition.getNonVectorFields(); + DistanceFunction distanceFunction = vectorField == null ? null : vectorField.getDistanceFunction(); + if (options.getVectorFieldName() != null && vectorField == null) { + throw new SKException(""); } + // get list of fields that should be returned by the query + List fields = (options.isIncludeVectors()) + ? recordDefinition.getAllFields() + : recordDefinition.getNonVectorFields(); + + // get search filters and get the list of parameters for the filters String filter = getFilter(options.getVectorSearchFilter(), recordDefinition); List parameters = getFilterParameters(options.getVectorSearchFilter()); + // generate SQL statement String selectQuery = "SELECT " + (vector == null ? "0 as distance, " : formatQuery("VECTOR_DISTANCE(%s, ?, %s) distance, ", vectorField.getEffectiveStorageName(), toOracleDistanceFunction(distanceFunction))) @@ -345,19 +425,20 @@ public VectorSearchResults search(String collectionName, List 0 ? " OFFSET " + options.getSkip() + " ROWS" : "") + (options.getTop() > 0 ? " FETCH " + (options.getSkip() > 0 ? "NEXT " : "FIRST ") + options.getTop() + " ROWS ONLY" : ""); + LOGGER.finest("Search using statement: " + selectQuery); - System.out.println(selectQuery); + // Execute the statement List> records = new ArrayList<>(); try (Connection connection = dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(selectQuery)) { // set parameters from filters int parameterIndex = 1; + // if a vector was provided for similarity search set the value of the vector if (vector != null) { - System.out.println("Set vector parameter with index " + parameterIndex + " to: " - + objectMapper.writeValueAsString(vector)); statement.setString(parameterIndex++, objectMapper.writeValueAsString(vector)); } + // set all parameters. for (Object parameter : parameters) { if (parameter != null) { setSearchParameter(statement, parameterIndex++, parameter.getClass(), parameter); @@ -371,18 +452,19 @@ public VectorSearchResults search(String collectionName, List VectorSearchResults search(String collectionName, List(mapper.mapStorageModelToRecord(rs, getRecordOptions), score)); } } } catch (SQLException | JsonProcessingException e) { - logger.info(e.getMessage()); throw new SKException("Search failed", e); } return new VectorSearchResults<>(records); } + /** + * Sets the parameter value + * @param statement the statement + * @param index the parameter index + * @param type the parameter type + * @param value the value + */ private void setSearchParameter(PreparedStatement statement, int index, Class type, Object value) { try { + // Use JSON string to set lists if (List.class.equals(type)) { statement.setObject(index, objectMapper.writeValueAsString(value)); System.out.println( "Set values: " + objectMapper.writeValueAsString(value)); return; } + // convert UUID to string if (UUID.class.equals(type)) { - if (value == null) { - statement.setNull(index, OracleTypes.RAW); - } else { - UUID uuid = (UUID)value; - ByteBuffer bb = ByteBuffer.allocate(16); - bb.putLong(uuid.getMostSignificantBits()); - bb.putLong(uuid.getLeastSignificantBits()); - statement.setBytes(index, bb.array()); - System.out.println("Set values: " + uuid); - } + statement.setString(index, value.toString()); return; } + // convert OffsetDateType to TIMESTAMPTZ if (OffsetDateTime.class.equals(type)) { if (value == null) { statement.setNull(index, OracleTypes.TIMESTAMPTZ); @@ -435,6 +518,7 @@ private void setSearchParameter(PreparedStatement statement, int index, Class } return; } + // use setBigDecimal to set BigDecimal value if (BigDecimal.class.equals(type)) { if (value == null) { statement.setNull(index, OracleTypes.DECIMAL); @@ -446,7 +530,8 @@ private void setSearchParameter(PreparedStatement statement, int index, Class } return; } - System.out.println("Set parameter " + index + " to: " + value); + + // for all other types set object with the given value statement.setObject(index, value); } catch (Exception ex) { @@ -455,6 +540,13 @@ private void setSearchParameter(PreparedStatement statement, int index, Class } + /** + * Defines the type that will be used to retrieve data from a given database table column. + * @param columnIndex the index of the column + * @param statement the statement + * @param fieldType the java field type + * @throws SQLException if an error occurs while defining the column type + */ private void defineDataColumnType(int columnIndex, OracleStatement statement, Class fieldType) throws SQLException { // swich between supported classes and define the column type on the statement switch (supportedDataTypes.get(fieldType)) { @@ -491,9 +583,6 @@ private void defineDataColumnType(int columnIndex, OracleStatement statement, Cl case OracleDataTypesMapping.JSON: statement.defineColumnType(columnIndex, OracleTypes.JSON, Integer.MAX_VALUE); break; - case OracleDataTypesMapping.UUID: - statement.defineColumnType(columnIndex, OracleTypes.RAW); - break; case OracleDataTypesMapping.BYTE_ARRAY: statement.defineColumnType(columnIndex, OracleTypes.RAW); default: @@ -501,7 +590,11 @@ private void defineDataColumnType(int columnIndex, OracleStatement statement, Cl } } - + /** + * Converts a {@link DistanceFunction} to the equivalent Oracle distance function. + * @param distanceFunction the distance function + * @return the Oracle distance function + */ private String toOracleDistanceFunction(DistanceFunction distanceFunction) { switch (distanceFunction) { case DOT_PRODUCT: @@ -525,6 +618,7 @@ private String toOracleDistanceFunction(DistanceFunction distanceFunction) { */ @Override public List getFilterParameters(VectorSearchFilter filter) { + // TODO: this method should be protected, not public if (filter == null || filter.getFilterClauses().isEmpty()) { return Collections.emptyList(); @@ -544,6 +638,11 @@ public List getFilterParameters(VectorSearchFilter filter) { }).collect(Collectors.toList()); } + /** + * Gets the filter clause for an equal to filter + * @param filterClause The equal to filter clause to get the filter string for. + * @return the filter clause + */ @Override public String getEqualToFilter(EqualToFilterClause filterClause) { String fieldName = JDBCVectorStoreQueryProvider @@ -557,6 +656,11 @@ public String getEqualToFilter(EqualToFilterClause filterClause) { } } + /** + * Gets the filter clause for an any tag equal to filter + * @param filterClause The any tag equal to filter clause to get the filter string for. + * @return the filter clause + */ @Override public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { String fieldName = JDBCVectorStoreQueryProvider @@ -566,6 +670,13 @@ public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { fieldName, fieldName, fieldName); } + /** + * Gets the mapper used to map a ResultSet to records + * @param recordClass the record class + * @param vectorStoreRecordDefinition the record definition + * @return the vector store record mapper + * @param the type of the records + */ @Override public VectorStoreRecordMapper getVectorStoreRecordMapper( Class recordClass, @@ -574,25 +685,52 @@ public VectorStoreRecordMapper getVectorStoreRecordM .withRecordClass(recordClass) .withVectorStoreRecordDefinition(vectorStoreRecordDefinition) .withSupportedDataTypesMapping(getSupportedDataTypes()) - .withAnnotatedTypeMapping(annotatedTypeMapping) .build(); } + /** + * Gets a builder that allows to build an OracleVectorStoreQueryProvider + * @return the builder + */ public static Builder builder() { return new Builder(); } + /** + * OracleVectorStoreQueryProvider builder. + */ public static class Builder extends JDBCVectorStoreQueryProvider.Builder { + /** + * The data source + */ private DataSource dataSource; + + /** + * The collections table + */ private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; + + /** + * The prefix for collection table names + */ private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; + + /** + * The object mapper + */ private ObjectMapper objectMapper = new ObjectMapper(); + + /** + * The string type mapping choice + */ private StringTypeMapping stringTypeMapping = StringTypeMapping.USE_VARCHAR; - private int defaultVarcharSize = 2000; - private Map, String> annotatedTypeMapping = null; + /** + * The size of varchar columns + */ + private int defaultVarcharSize = 2000; @SuppressFBWarnings("EI_EXPOSE_REP2") @@ -621,17 +759,17 @@ public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { return this; } + /** + * Sets the object mapper used to map records to and from results + * @param objectMapper the object mapper + * @return the builder + */ public Builder withObjectMapper( ObjectMapper objectMapper) { this.objectMapper = objectMapper; return this; } - public Builder withAnnotatedTypeMapping(Map, String> annotatedTypeMapping) { - this.annotatedTypeMapping = annotatedTypeMapping; - return this; - } - /** * Sets the desired String type mapping. * @param stringTypeMapping the desired String type mapping. The default value is @@ -644,8 +782,8 @@ public Builder withStringTypeMapping (StringTypeMapping stringTypeMapping) { } /** - * Sets the default size of the VARHCHAR2 fields. - * @param defaultVarcharSize the default size of the VARHCHAR2 fields. By default, the size + * Sets the default size of the VARHCHAR fields. + * @param defaultVarcharSize the default size of the VARHCHAR fields. By default, the size * is 2000. * @return then builder */ @@ -654,11 +792,14 @@ public Builder withDefaultVarcharSize (int defaultVarcharSize) { return this; } + /** + * Builds and Oracle vector store query provider. + * @return the query provider + */ @Override public OracleVectorStoreQueryProvider build() { return new OracleVectorStoreQueryProvider(dataSource, collectionsTable, - prefixForCollectionTables, defaultVarcharSize, stringTypeMapping, objectMapper, - annotatedTypeMapping); + prefixForCollectionTables, defaultVarcharSize, stringTypeMapping, objectMapper); } } } \ No newline at end of file diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index e6e7bf0a..fd4f9dbf 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -1,11 +1,14 @@ -// Copyright (c) Microsoft. All rights reserved. +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; -import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; @@ -17,7 +20,6 @@ import oracle.jdbc.OracleResultSet; import oracle.jdbc.provider.oson.OsonModule; import oracle.sql.TIMESTAMPTZ; -import java.nio.ByteBuffer; import java.sql.ResultSet; import java.sql.SQLException; import java.util.Map; @@ -150,14 +152,6 @@ public OracleVectorStoreRecordMapper build() { for (VectorStoreRecordField field : vectorStoreRecordDefinition.getNonVectorFields()) { Class fieldType = field.getFieldType(); - boolean isAnnotated = false; - try { - isAnnotated = recordClass.getDeclaredField(field.getName()) - .isAnnotationPresent(JsonTypeInfo.class); - } catch (NoSuchFieldException e) { - // ignore exception, assume the field is not annotated - } - Object value; switch (supportedDataTypesMapping.get(fieldType)) { case OracleDataTypesMapping.STRING_CLOB: @@ -195,15 +189,8 @@ public OracleVectorStoreRecordMapper build() { value = resultSet.getBytes(field.getEffectiveStorageName()); break; case OracleDataTypesMapping.UUID: - byte[] bytes = resultSet.getBytes(field.getEffectiveStorageName()); - if (bytes != null) { - ByteBuffer bb = ByteBuffer.wrap(bytes); - long firstLong = bb.getLong(); - long secondLong = bb.getLong(); - value = new UUID(firstLong, secondLong); - } else { - value = null; - } + String uuidValue = resultSet.getString(field.getEffectiveStorageName()); + value = uuidValue == null ? null : UUID.fromString(uuidValue); break; case OracleDataTypesMapping.JSON: value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); @@ -220,18 +207,6 @@ public OracleVectorStoreRecordMapper build() { JsonNode genericNode = objectMapper.valueToTree(value); objectNode.set(field.getEffectiveStorageName(), genericNode); - if (isAnnotated) { - if (annotatedTypeMapping != null && annotatedTypeMapping.containsKey(field.getFieldType())) { - objectNode.set(field.getEffectiveStorageName() + "_type", - TextNode.valueOf( - annotatedTypeMapping.get(field.getFieldType()))); - } else { - objectNode.set(field.getEffectiveStorageName() + "_type", - TextNode.valueOf( - field.getFieldType().getName())); - - } - } } if (options != null && options.isIncludeVectors()) { for (VectorStoreRecordVectorField field : vectorStoreRecordDefinition.getVectorFields()) { diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllBoxedTypes.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllBoxedTypes.java index 9f93e03d..307c974d 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllBoxedTypes.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllBoxedTypes.java @@ -1,3 +1,9 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllPrimitiveTypes.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllPrimitiveTypes.java index 6e91485d..0ca52d7b 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllPrimitiveTypes.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllPrimitiveTypes.java @@ -1,3 +1,9 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java index 318911d8..5964cb3c 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java @@ -1,41 +1,45 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.annotation.JsonTypeInfo.As; -import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; -import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; -import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; -import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; -import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; -import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; import java.math.BigDecimal; import java.time.OffsetDateTime; import java.util.List; import java.util.UUID; -@JsonIgnoreProperties(ignoreUnknown = true) public class ClassWithAnnotatedTypes { private final String id; + @JsonProperty("value_type") + private final String valueType; + + @JsonProperty("value_field") @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = As.EXTERNAL_PROPERTY , property = "value_type") @JsonSubTypes({ - @JsonSubTypes.Type(value = String.class, name="java.lang.String"), - @JsonSubTypes.Type(value = Boolean.class, name="java.lang.Boolean"), - @JsonSubTypes.Type(value = Byte.class, name="java.lang.Byte"), - @JsonSubTypes.Type(value = Short.class, name="java.lang.Short"), - @JsonSubTypes.Type(value = Integer.class, name="java.lang.Integer"), - @JsonSubTypes.Type(value = Long.class, name="java.lang.Long"), - @JsonSubTypes.Type(value = Float.class, name="java.lang.Float"), - @JsonSubTypes.Type(value = Double.class, name="java.lang.Double"), - @JsonSubTypes.Type(value = BigDecimal.class, name="java.math.BigDecimal"), - @JsonSubTypes.Type(value = OffsetDateTime.class, name="java.time.OffsetDateTime"), - @JsonSubTypes.Type(value = UUID.class, name="java.util.UUID"), + @JsonSubTypes.Type(value = String.class, name="string"), + @JsonSubTypes.Type(value = Boolean.class, name="boolean"), + @JsonSubTypes.Type(value = Byte.class, name="byte"), + @JsonSubTypes.Type(value = Short.class, name="short"), + @JsonSubTypes.Type(value = Integer.class, name="integer"), + @JsonSubTypes.Type(value = Long.class, name="long"), + @JsonSubTypes.Type(value = Float.class, name="float"), + @JsonSubTypes.Type(value = Double.class, name="double"), + @JsonSubTypes.Type(value = BigDecimal.class, name="decimal"), + @JsonSubTypes.Type(value = OffsetDateTime.class, name="timestamp"), + @JsonSubTypes.Type(value = UUID.class, name="uuid"), @JsonSubTypes.Type(value = byte[].class, name="byte_array"), - @JsonSubTypes.Type(value = List.class, name="listOfStrings") + @JsonSubTypes.Type(value = List.class, name="json") }) private Object value; @@ -43,10 +47,11 @@ public class ClassWithAnnotatedTypes { public ClassWithAnnotatedTypes() { - this(null, null, null); + this(null, null, null, null); }; - public ClassWithAnnotatedTypes(String id, Object value, Float[] vectorValue) { + public ClassWithAnnotatedTypes(String id, String valueType, Object value, Float[] vectorValue) { this.id = id; + this.valueType = valueType; this.value = value; this.vectorValue = vectorValue; } @@ -55,6 +60,8 @@ public String getId() { return id; } + public String getValueType() { return valueType; } + public Object getValue() { return value; } diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java index d1c848a6..dc736ba3 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java @@ -1,4 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. + +// Should we change the copyright from the file we copied? Or should we +// just change the test to use a class that we create? package com.microsoft.semantickernel.data.jdbc.oracle; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleCommonVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleCommonVectorStoreRecordCollectionTest.java index f8656c3a..7549188b 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleCommonVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleCommonVectorStoreRecordCollectionTest.java @@ -1,3 +1,9 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; import oracle.jdbc.OracleConnection; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java index 9650652e..42036826 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java @@ -1,3 +1,9 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; @@ -41,18 +47,26 @@ void testDataTypes(String dataFieldName, Class dataFieldType, Object dataFiel if (fieldSubType != null) { dataField = VectorStoreRecordDataField.builder() .withName("value") - .withStorageName("value") + .withStorageName("value_field") .withFieldType(dataFieldType, fieldSubType) .isFilterable(true) .build(); } else { dataField = VectorStoreRecordDataField.builder() .withName("value") - .withStorageName("value") + .withStorageName("value_field") .withFieldType(dataFieldType) .isFilterable(true) .build(); } + VectorStoreRecordDataField dataTypeField; + dataTypeField = VectorStoreRecordDataField.builder() + .withName("valueType") + .withStorageName("value_type") + .withFieldType(String.class) + .isFilterable(false) + .build(); + VectorStoreRecordVectorField dummyVector = VectorStoreRecordVectorField.builder() .withName("vectorValue") @@ -64,14 +78,11 @@ void testDataTypes(String dataFieldName, Class dataFieldType, Object dataFiel .build(); VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( - Arrays.asList(keyField, dataField, dummyVector) + Arrays.asList(keyField, dataTypeField, dataField, dummyVector) ); OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() .withDataSource(DATA_SOURCE) - .withAnnotatedTypeMapping(new HashMap() {{ - put(List.class, "listOfStrings"); - }}) .build(); JDBCVectorStore vectorStore = JDBCVectorStore.builder() @@ -94,7 +105,7 @@ JDBCVectorStoreRecordCollectionOptions. builder() String key = "testid"; ClassWithAnnotatedTypes record = - new ClassWithAnnotatedTypes(key, dataFieldValue, new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f }); + new ClassWithAnnotatedTypes(key, dataFieldName, dataFieldValue, new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f }); collection.upsertAsync(record, null).block(); @@ -102,6 +113,8 @@ JDBCVectorStoreRecordCollectionOptions. builder() assertNotNull(result); if (record.getValue().getClass().equals(OffsetDateTime.class)) { assertTrue(((OffsetDateTime)dataFieldValue).isEqual((OffsetDateTime)record.getValue())); + } else if (dataFieldName == "byte_array") { + assertArrayEquals((byte[]) dataFieldValue, (byte[])record.getValue()); } else { assertEquals(dataFieldValue, result.getValue()); } @@ -109,81 +122,11 @@ JDBCVectorStoreRecordCollectionOptions. builder() collection.deleteCollectionAsync().block(); } - @Test - void testByteArrayType() { - VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() - .withName("id") - .withStorageName("id") - .withFieldType(String.class) - .build(); - - VectorStoreRecordDataField dataField; - dataField = VectorStoreRecordDataField.builder() - .withName("value") - .withStorageName("value") - .withFieldType(byte[].class) - .isFilterable(true) - .build(); - - VectorStoreRecordVectorField dummyVector = VectorStoreRecordVectorField.builder() - .withName("vectorValue") - .withStorageName("vectorValue") - .withFieldType(Float[].class) - .withDimensions(8) - .withDistanceFunction(DistanceFunction.COSINE_DISTANCE) - .withIndexKind(IndexKind.IVFFLAT) - .build(); - - VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( - Arrays.asList(keyField, dataField, dummyVector) - ); - - OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() - .withDataSource(DATA_SOURCE) - .withAnnotatedTypeMapping(new HashMap() {{ - put(byte[].class, "byte_array"); - }}) - .build(); - - JDBCVectorStore vectorStore = JDBCVectorStore.builder() - .withDataSource(DATA_SOURCE) - .withOptions(JDBCVectorStoreOptions.builder() - .withQueryProvider(queryProvider) - .build()) - .build(); - - String collectionName = "test_datatype_bytearray"; - - VectorStoreRecordCollection collection = - vectorStore.getCollection(collectionName, - JDBCVectorStoreRecordCollectionOptions. builder() - .withRecordClass(ClassWithAnnotatedTypes.class) - .withRecordDefinition(definition).build()); - - collection.createCollectionAsync().block(); - - String key = "testid"; - - byte[] dataFieldValue = new byte[] {1, 2, 3}; - ClassWithAnnotatedTypes record = - new ClassWithAnnotatedTypes(key, dataFieldValue, new Float[] { 0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f }); - - collection.upsertAsync(record, null).block(); - - ClassWithAnnotatedTypes result = collection.getAsync(key, null).block(); - assertNotNull(result); - assertArrayEquals(dataFieldValue, (byte[])result.getValue()); - - collection.deleteCollectionAsync().block(); - } - - - private static Stream supportedDataTypes() { return Stream.of( Arguments.of("string", String.class, "asd123", null), - Arguments.of("boolean_true", Boolean.class, true, null), - Arguments.of("boolean_false", Boolean.class, false, null), + Arguments.of("boolean", Boolean.class, true, null), + Arguments.of("boolean", Boolean.class, false, null), Arguments.of("byte", Byte.class, (byte) 127, null), Arguments.of("short", Short.class, (short) 3, null), Arguments.of("integer", Integer.class, 321, null), @@ -193,6 +136,7 @@ private static Stream supportedDataTypes() { Arguments.of("decimal", BigDecimal.class, new BigDecimal("12345.67"), null), Arguments.of("timestamp", OffsetDateTime.class, OffsetDateTime.now(), null), Arguments.of("uuid", UUID.class, UUID.randomUUID(), null), + Arguments.of("byte_array", byte[].class, new byte[] {1, 2, 3}, String.class), Arguments.of("json", List.class, Arrays.asList("a", "s", "d"), String.class) ); } diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java index d99234ab..f81176b3 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java @@ -1,3 +1,9 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; @@ -119,11 +125,6 @@ void testDataTypesSearch (ClassWithAllBoxedTypes record) { assertEquals(1, results.getTotalCount()); assertEquals(record.getDoubleValue(), results.getResults().get(0).getRecord().getDoubleValue()); - System.out.println(record.getDecimalValue()); - System.out.println(record.getDecimalValue().doubleValue()); - System.out.println(results.getResults().get(0).getRecord().getDecimalValue()); - System.out.println(results.getResults().get(0).getRecord().getDecimalValue().doubleValue()); - // decimal results = collection.searchAsync( null, diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java index 5c575950..cc8d2462 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java @@ -1,3 +1,9 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index b3eb403c..1a7a328a 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -1,3 +1,9 @@ +/* + ** Semantic Kernel Oracle connector version 1.0. + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. + ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.fasterxml.jackson.annotation.JsonCreator; From abd6d42683bf8524353642b3746674686b3c8df4 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Thu, 26 Jun 2025 20:04:22 +0200 Subject: [PATCH 201/277] Fixed merge issues --- .../jdbc/oracle/OracleDataTypesMapping.java | 23 -- .../oracle/OracleVectorStoreFieldHelper.java | 166 ------------- .../oracle/OracleVectorStoreRecordMapper.java | 11 - .../OracleVectorStoreDataTypeSearchTest.java | 11 - ...OracleVectorStoreRecordCollectionTest.java | 229 ------------------ 5 files changed, 440 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java index 17ae9b2d..cfca9ecc 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java @@ -1,4 +1,3 @@ -<<<<<<< add-oracle-store /* ** Semantic Kernel Oracle connector version 1.0. ** @@ -71,27 +70,5 @@ public class OracleDataTypesMapping { /** * Oracle database type used to map vectors (the parameter is the dimension of the vector) */ -======= -package com.microsoft.semantickernel.data.jdbc.oracle; - -/** - * Defines oracle database type constants for supported field types. - */ -public class OracleDataTypesMapping { - public static final String STRING_VARCHAR = "NVARCHAR2(%s)"; - public static final String STRING_CLOB = "CLOB"; - public static final String BOOLEAN = "BOOLEAN"; - public static final String BYTE = "NUMBER(3)"; - public static final String BYTE_ARRAY = "RAW(2000)"; - public static final String SHORT = "NUMBER(5)"; - public static final String INTEGER = "NUMBER(10)"; - public static final String LONG = "NUMBER(19)"; - public static final String FLOAT = "BINARY_FLOAT"; - public static final String DOUBLE = "BINARY_DOUBLE"; - public static final String DECIMAL = "NUMBER"; - public static final String OFFSET_DATE_TIME = "TIMESTAMP(7) WITH TIME ZONE"; - public static final String UUID = "RAW(16)"; - public static final String JSON = "JSON"; ->>>>>>> main public static final String VECTOR_FLOAT = "VECTOR(%s, FLOAT32)"; } diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java index 359f8f50..c177131f 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java @@ -1,20 +1,14 @@ -<<<<<<< add-oracle-store /* ** Semantic Kernel Oracle connector version 1.0. ** ** Copyright (c) 2025 Oracle and/or its affiliates. ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ */ -======= ->>>>>>> main package com.microsoft.semantickernel.data.jdbc.oracle; import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider.StringTypeMapping; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; -<<<<<<< add-oracle-store import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; -======= ->>>>>>> main import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import oracle.jdbc.OracleTypes; @@ -29,7 +23,6 @@ import java.util.stream.Collectors; /** -<<<<<<< add-oracle-store * Helper class for field operations. Handles mapping between field java types to DB types and * generating SQL statement to create field indexes. */ @@ -39,12 +32,6 @@ class OracleVectorStoreFieldHelper { * The logger */ private static final Logger LOGGER = Logger.getLogger(OracleVectorStoreFieldHelper.class.getName()); -======= - * Helper class for field operations. - */ -public class OracleVectorStoreFieldHelper { - private static final Logger LOGGER = Logger.getLogger(OracleVectorStoreQueryProvider.class.getName()); ->>>>>>> main /** * Maps supported key java classes to Oracle database types @@ -52,16 +39,6 @@ public class OracleVectorStoreFieldHelper { private static final HashMap, String> supportedKeyTypes = new HashMap() { { put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, 255)); -<<<<<<< add-oracle-store -======= - put(short.class, OracleDataTypesMapping.SHORT); - put(Short.class, OracleDataTypesMapping.SHORT); - put(int.class, OracleDataTypesMapping.INTEGER); - put(Integer.class, OracleDataTypesMapping.INTEGER); - put(long.class, OracleDataTypesMapping.LONG); - put(Long.class, OracleDataTypesMapping.LONG); - put(UUID .class, OracleDataTypesMapping.UUID); ->>>>>>> main } }; @@ -75,17 +52,6 @@ public class OracleVectorStoreFieldHelper { put(Collection.class, OracleDataTypesMapping.VECTOR_FLOAT); put(float[].class, OracleDataTypesMapping.VECTOR_FLOAT); put(Float[].class, OracleDataTypesMapping.VECTOR_FLOAT); -<<<<<<< add-oracle-store -======= -/* - put(byte[].class,"VECTOR(%s, INT8)"); - put(Byte[].class,"VECTOR(%s, INT8)"); - put(double[].class,"VECTOR(%s, FLOAT64)"); - put(Double[].class,"VECTOR(%s, FLOAT64)"); - put(boolean[].class,"VECTOR(%s, BINARY)"); - put(Boolean[].class,"VECTOR(%s, BINARY)"); - */ ->>>>>>> main } }; @@ -114,35 +80,12 @@ public class OracleVectorStoreFieldHelper { put(byte[].class, OracleDataTypesMapping.BYTE_ARRAY); put(List.class, OracleDataTypesMapping.JSON); } -<<<<<<< add-oracle-store }; /** * Suffix added to the effective column name to generate the index name for a vector column. */ public static final String VECTOR_INDEX_SUFFIX = "_VECTOR_INDEX"; -======= - - }; - - /** - * Maps vector type to OracleTypes. Only needed if types other than FLOAT_32 are supported. - */ - private static final Map, Integer> mapOracleTypeToVector = new HashMap() { - { - put(float[].class, OracleTypes.VECTOR_FLOAT32); - put(Float[].class, OracleTypes.VECTOR_FLOAT32); -/* - put(byte[].class, OracleTypes.VECTOR_INT8); - put(Byte[].class, OracleTypes.VECTOR_INT8); - put(Double[].class, OracleTypes.VECTOR_FLOAT64); - put(double[].class, OracleTypes.VECTOR_FLOAT64); - put(Boolean[].class, OracleTypes.VECTOR_BINARY); - put(boolean[].class, OracleTypes.VECTOR_BINARY); -*/ - } - }; ->>>>>>> main /** * Gets the mapping between the supported Java key types and the Oracle database type. @@ -160,20 +103,11 @@ public static HashMap, String> getSupportedKeyTypes() { */ public static Map, String> getSupportedDataTypes( StringTypeMapping stringTypeMapping, int defaultVarCharLength) { -<<<<<<< add-oracle-store String stringType = stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR) ? String.format(OracleDataTypesMapping.STRING_VARCHAR, defaultVarCharLength) : OracleDataTypesMapping.STRING_CLOB; supportedDataTypes.put(String.class, stringType); LOGGER.finest("Mapping String columns to " + stringType); -======= - - if (stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR)) { - supportedDataTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, defaultVarCharLength)); - } else { - supportedDataTypes.put(String.class, OracleDataTypesMapping.STRING_CLOB); - } ->>>>>>> main return supportedDataTypes; } @@ -224,22 +158,14 @@ public static String getCreateVectorIndexStatement(VectorStoreRecordVectorField */ public static String createIndexForDataField(String collectionTableName, VectorStoreRecordDataField dataField, Map, String> supportedDataTypes) { if (supportedDataTypes.get(dataField.getFieldType()) == "JSON") { -<<<<<<< add-oracle-store String dataFieldIndex = "CREATE MULTIVALUE INDEX IF NOT EXISTS %s ON %s t (t.%s.%s)"; -======= - String dataFieldIndex = "CREATE MULTIVALUE INDEX %s ON %s t (t.%s.%s)"; ->>>>>>> main return String.format(dataFieldIndex, collectionTableName + "_" + dataField.getEffectiveStorageName(), collectionTableName, dataField.getEffectiveStorageName(), getFunctionForType(supportedDataTypes.get(dataField.getFieldSubType()))); } else { -<<<<<<< add-oracle-store String dataFieldIndex = "CREATE INDEX IF NOT EXISTS %s ON %s (%s ASC)"; -======= - String dataFieldIndex = "CREATE INDEX %s ON %s (%s ASC)"; ->>>>>>> main return String.format(dataFieldIndex, collectionTableName + "_" + dataField.getEffectiveStorageName(), collectionTableName, @@ -249,7 +175,6 @@ public static String createIndexForDataField(String collectionTableName, VectorS } /** -<<<<<<< add-oracle-store * Returns vector columns names and types for CREATE TABLE statement * @param fields list of vector record fields. * @return comma separated list of columns and types for CREATE TABLE statement. @@ -294,8 +219,6 @@ private static String getTypeForVectorField(VectorStoreRecordVectorField field) } /** -======= ->>>>>>> main * Gets the function that allows to return the function that converts the JSON value to the * data type. * @param jdbcType The JDBC type. @@ -321,93 +244,4 @@ private static String getFunctionForType(String jdbcType) { } } -<<<<<<< add-oracle-store -======= - /** - * Gets the type of the vector given the field definition. This method is not needed if only - * - * @param field the vector field definition. - * @return returns the type of vector for the given field type. - */ - public static String getTypeForVectorField(VectorStoreRecordVectorField field) { - String dimension = field.getDimensions() > 0 ? String.valueOf(field.getDimensions()) : "*"; - return String.format(supportedVectorTypes.get(field.getFieldType()), dimension); -/* Not needed since all types are FLOAT32 - if (field.getFieldSubType() != null) { - String vectorType; - switch (field.getFieldSubType().getName()) { - case "java.lang.Double": - vectorType = "FLOAT64"; - break; - case "java.lang.Byte": - vectorType = "INT8"; - break; - case "java.lang.Boolean": - vectorType = "BINARY"; - break; - default: - vectorType = "FLOAT32"; - } - return String.format(supportedVectorTypes.get(field.getFieldType()), dimension, vectorType); - } else { - return String.format(supportedVectorTypes.get(field.getFieldType()), dimension); - } - */ - } - - /** - * Gets the JDBC oracle of the vector field definition. - * @param field the vector field definition. - * @return the JDBC oracle type. - */ - public static int getOracleTypeForField(VectorStoreRecordVectorField field) { - if (field.getFieldSubType() == null) { - return mapOracleTypeToVector.get(field.getFieldType()).intValue(); - } else { - switch (field.getFieldSubType().getName()) { - case "java.lang.Double": - return OracleTypes.VECTOR_FLOAT64; - case "java.lang.Byte": - return OracleTypes.VECTOR_INT8; - case "java.lang.Boolean": - return OracleTypes.VECTOR_BINARY; - default: - return OracleTypes.VECTOR_FLOAT32; - } - } - } - - /** - * Generates the index name given the field name. by suffixing "_VECTOR_INDEX" to the field name. - * @param effectiveStorageName the field name. - * @return the index name. - */ - private static String getIndexName(String effectiveStorageName) { - return effectiveStorageName + "_VECTOR_INDEX"; - } - - /** - * Returns vector columns names and types for CREATE TABLE statement - * @param fields list of vector record fields. - * @return comma separated list of columns and types for CREATE TABLE statement. - */ - public static String getVectorColumnNamesAndTypes(List fields) { - List columns = fields.stream() - .map(field -> field.getEffectiveStorageName() + " " + - OracleVectorStoreFieldHelper.getTypeForVectorField(field) - ).collect(Collectors.toList()); - - return String.join(", ", columns); - } - - /** - * Returns key column names and type for key column for CREATE TABLE statement - * @param field the key field. - * @return column name and type of the key field for CREATE TABLE statement. - */ - public static String getKeyColumnNameAndType(VectorStoreRecordKeyField field) { - return field.getEffectiveStorageName() + " " + supportedKeyTypes.get(field.getFieldType()); - } - ->>>>>>> main } diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index 9386a744..fd4f9dbf 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -182,7 +182,6 @@ public OracleVectorStoreRecordMapper build() { value = resultSet.getBoolean(field.getEffectiveStorageName()); break; case OracleDataTypesMapping.OFFSET_DATE_TIME: -<<<<<<< add-oracle-store TIMESTAMPTZ timestamptz = ((OracleResultSet)resultSet).getTIMESTAMPTZ(field.getEffectiveStorageName()); value = timestamptz != null ? timestamptz.offsetDateTimeValue() : null; break; @@ -193,16 +192,6 @@ public OracleVectorStoreRecordMapper build() { String uuidValue = resultSet.getString(field.getEffectiveStorageName()); value = uuidValue == null ? null : UUID.fromString(uuidValue); break; -======= - value = ((OracleResultSet)resultSet).getTIMESTAMPTZ(field.getEffectiveStorageName()) - .offsetDateTimeValue(); - break; - case OracleDataTypesMapping.BYTE_ARRAY: - value = resultSet.getBytes(field.getEffectiveStorageName()); - break; - // fallthrough - case OracleDataTypesMapping.UUID: ->>>>>>> main case OracleDataTypesMapping.JSON: value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); break; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java index e0759c4f..f81176b3 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java @@ -1,12 +1,9 @@ -<<<<<<< add-oracle-store /* ** Semantic Kernel Oracle connector version 1.0. ** ** Copyright (c) 2025 Oracle and/or its affiliates. ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ */ -======= ->>>>>>> main package com.microsoft.semantickernel.data.jdbc.oracle; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; @@ -128,14 +125,6 @@ void testDataTypesSearch (ClassWithAllBoxedTypes record) { assertEquals(1, results.getTotalCount()); assertEquals(record.getDoubleValue(), results.getResults().get(0).getRecord().getDoubleValue()); -<<<<<<< add-oracle-store -======= - System.out.println(record.getDecimalValue()); - System.out.println(record.getDecimalValue().doubleValue()); - System.out.println(results.getResults().get(0).getRecord().getDecimalValue()); - System.out.println(results.getResults().get(0).getRecord().getDecimalValue().doubleValue()); - ->>>>>>> main // decimal results = collection.searchAsync( null, diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 88df2cf9..1d66a8e3 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -8,11 +8,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -<<<<<<< add-oracle-store -======= -import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollection; -import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollectionOptions; ->>>>>>> main import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; @@ -23,19 +18,10 @@ import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; -<<<<<<< add-oracle-store -======= -import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; ->>>>>>> main import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; -<<<<<<< add-oracle-store -======= -import oracle.jdbc.OracleConnection; -import oracle.jdbc.datasource.impl.OracleDataSource; ->>>>>>> main import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; @@ -43,31 +29,13 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -<<<<<<< add-oracle-store import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.Statement; -======= -import java.math.BigDecimal; -import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.time.Duration; -import java.time.OffsetDateTime; ->>>>>>> main import java.util.Arrays; import java.util.HashMap; import java.util.List; -<<<<<<< add-oracle-store -======= -import java.util.Map; -import java.util.UUID; -import java.util.logging.Logger; ->>>>>>> main import java.util.stream.Collectors; import java.util.stream.Stream; @@ -80,63 +48,6 @@ public class OracleVectorStoreRecordCollectionTest extends OracleCommonVectorStoreRecordCollectionTest { private static VectorStoreRecordCollection recordCollection; -<<<<<<< add-oracle-store -======= - private static final String ORACLE_IMAGE_NAME = "gvenzl/oracle-free:23.7-slim-faststart"; - private static final OracleDataSource DATA_SOURCE; - private static final OracleDataSource SYSDBA_DATA_SOURCE; - - static { - - try { - DATA_SOURCE = new oracle.jdbc.datasource.impl.OracleDataSource(); - SYSDBA_DATA_SOURCE = new oracle.jdbc.datasource.impl.OracleDataSource(); - String urlFromEnv = System.getenv("ORACLE_JDBC_URL"); - - if (urlFromEnv == null) { - // The Ryuk component is relied upon to stop this container. - OracleContainer oracleContainer = new OracleContainer(ORACLE_IMAGE_NAME) - .withCopyFileToContainer(MountableFile.forClasspathResource("/initialize.sql"), - "/container-entrypoint-initdb.d/initialize.sql") - .withStartupTimeout(Duration.ofSeconds(600)) - .withConnectTimeoutSeconds(600) - .withDatabaseName("pdb1") - .withUsername("testuser") - .withPassword("testpwd"); - oracleContainer.start(); - - initDataSource( - DATA_SOURCE, - oracleContainer.getJdbcUrl(), - oracleContainer.getUsername(), - oracleContainer.getPassword()); - initDataSource(SYSDBA_DATA_SOURCE, oracleContainer.getJdbcUrl(), "sys", oracleContainer.getPassword()); - } else { - initDataSource( - DATA_SOURCE, - urlFromEnv, - System.getenv("ORACLE_JDBC_USER"), - System.getenv("ORACLE_JDBC_PASSWORD")); - initDataSource( - SYSDBA_DATA_SOURCE, - urlFromEnv, - System.getenv("ORACLE_JDBC_USER"), - System.getenv("ORACLE_JDBC_PASSWORD")); - } - SYSDBA_DATA_SOURCE.setConnectionProperty(OracleConnection.CONNECTION_PROPERTY_INTERNAL_LOGON, "SYSDBA"); - - } catch (SQLException sqlException) { - throw new AssertionError(sqlException); - } - } - - static void initDataSource(OracleDataSource dataSource, String url, String username, String password) { - dataSource.setURL(url); - dataSource.setUser(username); - dataSource.setPassword(password); - } - ->>>>>>> main @BeforeAll public static void setup() throws Exception { @@ -187,7 +98,6 @@ private static List getHotels() { Float[] arrayF5 = new Float[] { -3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f }; return Arrays.asList( new Hotel("id_1", "Hotel 1", 1, 1.49d, Arrays.asList("one", "two"), "Hotel 1 description", -<<<<<<< add-oracle-store vec1, arrayf1, arrayf1, arrayF1, 4.0), new Hotel("id_2", "Hotel 2", 2, 1.44d, Arrays.asList("three", "four"), "Hotel 2 description with free-text search", @@ -201,36 +111,6 @@ private static List getHotels() { 4.0), new Hotel("id_5", "Hotel 5", 5, 1.89d, Arrays.asList("nine", "ten"),"Hotel 5 description", vec5, arrayf5, arrayf5, arrayF5, -======= - Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f), - new float[] {0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f}, - new float[] {0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f}, - new Float[] {0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f}, - 4.0), - new Hotel("id_2", "Hotel 2", 2, 1.44d, Arrays.asList("three", "four"), "Hotel 2 description with free-text search", - Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f), - new float[] {-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f}, - new float[] {-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f}, - new Float[] {-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f}, - 4.0), - new Hotel("id_3", "Hotel 3", 3, 1.53d, Arrays.asList("five", "six"), "Hotel 3 description", - Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f), - new float[] {4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f}, - new float[] {4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f}, - new Float[] {4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f}, - 5.0), - new Hotel("id_4", "Hotel 4", 4, 1.35d, Arrays.asList("seven", "eight"), "Hotel 4 description", - Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f), - new float[] {7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f}, - new float[] {7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f}, - new Float[] {7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f}, - 4.0), - new Hotel("id_5", "Hotel 5", 5, 1.89d, Arrays.asList("nine", "ten"),"Hotel 5 description", - Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f), - new float[] {-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f}, - new float[] {-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f}, - new Float[] {-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f}, ->>>>>>> main 4.0)); } @@ -477,89 +357,6 @@ void testKeyTypes(String suffix, Class keyType, Object keyValue) { collection.deleteCollectionAsync().block(); } -<<<<<<< add-oracle-store -======= - @ParameterizedTest - @MethodSource("supportedDataTypes") - void testDataTypes(String dataFieldName, Class dataFieldType, Object dataFieldValue, Class fieldSubType) { - VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() - .withName("id") - .withStorageName("id") - .withFieldType(String.class) - .build(); - - VectorStoreRecordDataField dataField; - if (fieldSubType != null) { - dataField = VectorStoreRecordDataField.builder() - .withName("dummy") - .withStorageName("dummy") - .withFieldType(dataFieldType, fieldSubType) - .isFilterable(true) - .build(); - } else { - dataField = VectorStoreRecordDataField.builder() - .withName("dummy") - .withStorageName("dummy") - .withFieldType(dataFieldType) - .isFilterable(true) - .build(); - } - - VectorStoreRecordVectorField dummyVector = VectorStoreRecordVectorField.builder() - .withName("vec") - .withStorageName("vec") - .withFieldType(List.class) - .withDimensions(2) - .withDistanceFunction(DistanceFunction.EUCLIDEAN_DISTANCE) - .withIndexKind(IndexKind.UNDEFINED) - .build(); - - VectorStoreRecordDefinition definition = VectorStoreRecordDefinition.fromFields( - Arrays.asList(keyField, dataField, dummyVector) - ); - - OracleVectorStoreQueryProvider queryProvider = OracleVectorStoreQueryProvider.builder() - .withDataSource(DATA_SOURCE) - .build(); - - JDBCVectorStore vectorStore = JDBCVectorStore.builder() - .withDataSource(DATA_SOURCE) - .withOptions(JDBCVectorStoreOptions.builder() - .withQueryProvider(queryProvider) - .build()) - .build(); - - String collectionName = "test_datatype_" + dataFieldName; - - VectorStoreRecordCollection collection = - vectorStore.getCollection(collectionName, - JDBCVectorStoreRecordCollectionOptions. builder() - .withRecordClass(DummyRecordForDataTypes.class) - .withRecordDefinition(definition).build()); - - collection.createCollectionAsync().block(); - - String key = "testid"; - - DummyRecordForDataTypes record = - new DummyRecordForDataTypes(key, dataFieldValue, Arrays.asList(1.0f, 2.0f)); - - collection.upsertAsync(record, null).block(); - - DummyRecordForDataTypes result = collection.getAsync(key, null).block(); - assertNotNull(result); - - if (dataFieldValue instanceof Number && result.getDummy() instanceof Number) { - assertEquals(((Number) dataFieldValue).doubleValue(), ((Number) result.getDummy()).doubleValue()); - } else if (dataFieldValue instanceof byte[]) { - assertArrayEquals((byte[]) dataFieldValue, (byte[]) result.getDummy()); - } else { - assertEquals(dataFieldValue, result.getDummy()); - } - - collection.deleteCollectionAsync().block(); - } ->>>>>>> main @Nested class HNSWIndexTests { @@ -738,11 +535,7 @@ private static Stream parametersExactSearch() { // thus upsertAync/getAsync won't work private static Stream supportedKeyTypes() { return Stream.of( -<<<<<<< add-oracle-store Arguments.of("string", String.class, "asd123") /*, -======= - Arguments.of("string", String.class, "asd123")/*, ->>>>>>> main Arguments.of("integer", Integer.class, 321), Arguments.of("long", Long.class, 5L), Arguments.of("short", Short.class, (short) 3), @@ -750,28 +543,6 @@ private static Stream supportedKeyTypes() { ); } -<<<<<<< add-oracle-store -======= - private static Stream supportedDataTypes() { - return Stream.of( - Arguments.of("string", String.class, "asd123", null), - Arguments.of("boolean_true", Boolean.class, true, null), - Arguments.of("boolean_false", Boolean.class, false, null), - Arguments.of("byte", Byte.class, (byte) 127, null), - Arguments.of("short", Short.class, (short) 3, null), - Arguments.of("integer", Integer.class, 321, null), - Arguments.of("long", Long.class, 5L, null), - Arguments.of("float", Float.class, 3.14f, null), - Arguments.of("double", double.class, 3.14159265358d, null), - Arguments.of("decimal", BigDecimal.class, new BigDecimal("12345.67"), null), - //Arguments.of("timestamp", OffsetDateTime.class, OffsetDateTime.now(), null) - //Arguments.of("uuid", UUID.class, UUID.randomUUID(), null) - Arguments.of("byte_array", byte[].class, "abc".getBytes(StandardCharsets.UTF_8), null), - Arguments.of("json", List.class, Arrays.asList("a", "s", "d"), String.class) - ); - } - ->>>>>>> main private static class DummyRecordForKeyTypes { private final Object id; private final String dummy; From 219a4d8f439b7318a35d75f48b31ef0ac567862f Mon Sep 17 00:00:00 2001 From: Kaiyuan Li Date: Thu, 26 Jun 2025 22:42:24 -0700 Subject: [PATCH 202/277] add extended test --- .../oracle/OracleVectorStoreExtendedTest.java | 419 ++++++++++++++++++ 1 file changed, 419 insertions(+) create mode 100644 data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java new file mode 100644 index 00000000..7fe66229 --- /dev/null +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java @@ -0,0 +1,419 @@ +package com.microsoft.semantickernel.data.jdbc.oracle; + +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; +import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; +import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordKey; +import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; +import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; +import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; +import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; +import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; +import com.microsoft.semantickernel.exceptions.SKException; +import org.junit.jupiter.api.Test; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertIterableEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class OracleVectorStoreExtendedTest extends OracleCommonVectorStoreRecordCollectionTest { + + // Test vector types + @Test + void testUseStringVec() { + VectorStoreRecordCollection collection = + createCollection( + "use_string_vec", + DummyRecordForVecString.class, + null); + + DummyRecordForVecString d1 = new DummyRecordForVecString("id1", "description1", "[1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8]"); + DummyRecordForVecString d2 = new DummyRecordForVecString("id2", "description2", "[1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8]"); + + collection.upsertBatchAsync(Arrays.asList(d1,d2), null).block(); + + DummyRecordForVecString rec = collection.getAsync("id1", + GetRecordOptions.builder().includeVectors(true).build()).block(); + + assertNotNull(rec); + assertEquals("[1.1,2.2,3.3,4.4,5.5,6.6,7.7,8.8]", rec.getVec()); + + collection.deleteCollectionAsync().block(); + } + + @Test + void testUseCollectionVec() { + VectorStoreRecordCollection collection = + createCollection( + "use_collection_vec", + DummyRecordForVecCollection.class, + null); + + List v1 = Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10f, -1.3f, 5.5f); + List v2 = Arrays.asList(-2f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f); + DummyRecordForVecCollection d1 = new DummyRecordForVecCollection("id1", "", v1); + DummyRecordForVecCollection d2 = new DummyRecordForVecCollection("id2", "", v2); + + collection.upsertBatchAsync(Arrays.asList(d1,d2), null).block(); + + DummyRecordForVecCollection rec = collection.getAsync("id1", + GetRecordOptions.builder().includeVectors(true).build()).block(); + + assertNotNull(rec); + assertEquals(8, rec.getVec().size()); + assertIterableEquals(v1, rec.getVec()); + + collection.deleteCollectionAsync().block(); + } + + // Test corner-case + @Test + void testUseCLOB() { + VectorStoreRecordCollection collection = + createCollection( + "use_clob", + DummyRecordForCLOB.class, + OracleVectorStoreQueryProvider.StringTypeMapping.USE_CLOB); + + DummyRecordForCLOB d1 = new DummyRecordForCLOB("id1", "clob-description", null); + DummyRecordForCLOB d2 = new DummyRecordForCLOB("id2", "clob-description2", vec(0)); + + collection.upsertBatchAsync(Arrays.asList(d1,d2), null).block(); + + try (Connection c = DATA_SOURCE.getConnection()) { + PreparedStatement st = c.prepareStatement( + "SELECT DATA_TYPE FROM USER_TAB_COLUMNS " + + "WHERE TABLE_NAME = 'SKCOLLECTION_USE_CLOB' AND COLUMN_NAME = 'DESCRIPTION'" + ); + ResultSet rs = st.executeQuery(); + rs.next(); + assertEquals("CLOB", rs.getString(1)); + } catch (SQLException e) { + throw new RuntimeException(e); + } finally { + collection.deleteCollectionAsync().block(); + } + } + + @Test + void testClobLongText() { + VectorStoreRecordCollection collection = + createCollection( + "clob_long_text", + DummyRecordForCLOB.class, + OracleVectorStoreQueryProvider.StringTypeMapping.USE_CLOB); + + String longText = String.join("", java.util.Collections.nCopies(6000, "a")); + DummyRecordForCLOB r = new DummyRecordForCLOB("big", longText, vec(0)); + collection.upsertAsync(r, null).block(); + + DummyRecordForCLOB out = collection.getAsync("big", null).block(); + assertEquals(longText.length(), out.getDescription().length()); + assertTrue(out.getDescription().startsWith("aaaa")); + + collection.deleteCollectionAsync().block(); + } + + @Test + void testMultipleFilter() { + VectorStoreRecordCollection collection = + createCollection( + "multiple_filter", + DummyRecordForMultipleFilter.class, + null); + + DummyRecordForMultipleFilter d1 = new DummyRecordForMultipleFilter("id1", 4, 120, floatVec(0f)); + DummyRecordForMultipleFilter d2 = new DummyRecordForMultipleFilter("id2", 4, 100, floatVec(0f)); + DummyRecordForMultipleFilter d3 = new DummyRecordForMultipleFilter("id3", 3, 100, floatVec(0f)); + + collection.upsertBatchAsync(Arrays.asList(d1,d2,d3), null).block(); + + VectorSearchFilter filter = VectorSearchFilter.builder() + .equalTo("price",100) + .equalTo("stars", 4) + .build(); + + VectorSearchResults results = + collection.searchAsync(null, + VectorSearchOptions.builder() + .withVectorSearchFilter(filter) + .build() + ).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals("id2", results.getResults().get(0).getRecord().getId()); + + collection.deleteCollectionAsync().block(); + } + + @Test + void testVectorDimensionMismatch() { + VectorStoreRecordCollection collection = + createCollection( + "vector_dimension_mismatch", + DummyRecord.class, + null); + + // Empty vector rejected + DummyRecord d1 = new DummyRecord("id1", 4, 120d, new float[]{}); + SKException ex = assertThrows(SKException.class, + () -> collection.upsertBatchAsync(Arrays.asList(d1), null).block()); + System.out.println(ex.getMessage()); + assertTrue(ex.getCause().getMessage().contains("ORA-51803")); + + // Vector dimension mismatch + DummyRecord d2 = new DummyRecord("id1", 4, 120d, new float[]{1.1f,2.2f,3.3f,4.4f,5.5f}); + SKException ex2 = assertThrows(SKException.class, + () -> collection.upsertBatchAsync(Arrays.asList(d2), null).block()); + assertTrue(ex2.getCause().getMessage().contains("ORA-51803")); + + collection.deleteCollectionAsync().block(); + } + + @Test + void testNull() { + VectorStoreRecordCollection collection = + createCollection("test_null", DummyRecord.class, null); + + DummyRecord d1 = new DummyRecord("id1", 4, null, floatVec(1)); + collection.upsertBatchAsync(Arrays.asList(d1), null).block(); + + VectorSearchFilter filter = VectorSearchFilter.builder() + .equalTo("price",null)// + .build(); + + VectorSearchResults results = collection.searchAsync( + null, + VectorSearchOptions.builder() + .withVectorSearchFilter(filter) + .build() + ).block(); + + assertEquals(1, results.getTotalCount()); + assertEquals("id1", results.getResults().get(0).getRecord().getId()); + + collection.deleteCollectionAsync().block(); + } + + @Test + void testSkipAndTop() { + + } + + private VectorStoreRecordCollection createCollection( + String collectionName, + Class recordClass, + OracleVectorStoreQueryProvider.StringTypeMapping stringTypeMapping) { + + OracleVectorStoreQueryProvider.Builder builder = + OracleVectorStoreQueryProvider.builder() + .withDataSource(DATA_SOURCE); + + if (stringTypeMapping != null) { + builder.withStringTypeMapping(stringTypeMapping); + } + OracleVectorStoreQueryProvider queryProvider = builder.build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withDataSource(DATA_SOURCE) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + VectorStoreRecordCollection collection = + vectorStore.getCollection(collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(recordClass) + .build()).createCollectionAsync().block(); + + return collection; + } + + private List vec(float x) { + return Arrays.asList(x, x+1, x+2, x+3, x+4, x+5, x+6, x+7); + } + + private float[] floatVec(float x) { + return new float[] { x, x+1, x+2, x+3, x+4, x+5, x+6, x+7 }; + } + + private static class DummyRecordForVecString { + @VectorStoreRecordKey + private final String id; + + @VectorStoreRecordData(isFilterable = false) + private final String description; + + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.IVFFLAT) + private final String vec; + + public DummyRecordForVecString() { + this(null, null, null); + } + public DummyRecordForVecString(String id, String description, String vec) { + this.id = id; + this.description = description; + this.vec = vec; + } + + public String getId() { + return id; + } + public String getDescription() { + return description; + } + public String getVec() { + return vec; + } + } + + private static class DummyRecordForVecCollection{ + @VectorStoreRecordKey + private String id; + + @VectorStoreRecordData(isFilterable = false) + private String description; + + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.IVFFLAT) + private Collection vec; + + public DummyRecordForVecCollection() { + this(null, null, null); + } + public DummyRecordForVecCollection(String id, String description, Collection vec) { + this.id = id; + this.description = description; + this.vec = vec; + } + + public String getId() { + return id; + } + public String getDescription() { + return description; + } + public Collection getVec() { + return vec; + } + } + + private static class DummyRecordForCLOB { + @VectorStoreRecordKey + private String id; + + @VectorStoreRecordData(isFilterable = false) + private String description; + + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.IVFFLAT) + private List vec; + + private DummyRecordForCLOB() { + this(null, null, null); + } + private DummyRecordForCLOB(String id, String description, List vec) { + this.id = id; + this.description = description; + this.vec = vec; + } + + public String getId() { + return id; + } + public String getDescription() { + return description; + } + public List getVec() { + return vec; + } + } + + private static class DummyRecordForMultipleFilter { + @VectorStoreRecordKey + private String id; + + @VectorStoreRecordData(isFilterable = true) + private int stars; + + @VectorStoreRecordData(isFilterable = true) + private double price; + + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.IVFFLAT) + private float[] vec; + + public DummyRecordForMultipleFilter() { + this(null, 0, 0d, null); + } + + public DummyRecordForMultipleFilter(String id, int stars, double price, float[] vec) { + this.id = id; + this.stars = stars; + this.price = price; + this.vec = vec; + } + + public String getId() { + return id; + } + public int getStars() { + return stars; + } + public double getPrice() { + return price; + } + public float[] getVec() { + return vec; + } + } + + private static class DummyRecord { + @VectorStoreRecordKey + private String id; + + @VectorStoreRecordData(isFilterable = true) + private int stars; + + @VectorStoreRecordData(isFilterable = true) + private Double price; + + @VectorStoreRecordVector(dimensions = 8, distanceFunction = DistanceFunction.COSINE_DISTANCE, indexKind = IndexKind.IVFFLAT) + private float[] vec; + + public DummyRecord() { + this(null, 0, 0d, null); + } + + public DummyRecord(String id, int stars, Double price, float[] vec) { + this.id = id; + this.stars = stars; + this.price = price; + this.vec = vec; + } + + public String getId() { + return id; + } + public int getStars() { + return stars; + } + public Double getPrice() { + return price; + } + public float[] getVec() { + return vec; + } + } +} From 756e1adc6b7b0a6e2e6b5cba1e323d0428899bbf Mon Sep 17 00:00:00 2001 From: Kaiyuan Li Date: Thu, 26 Jun 2025 23:08:11 -0700 Subject: [PATCH 203/277] resolve conflict of git rebase --- .../OracleVectorStoreQueryProvider.java | 21 +++++++++++++++++++ .../oracle/OracleVectorStoreRecordMapper.java | 14 +++++++++++++ 2 files changed, 35 insertions(+) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index 9c347a5e..151a1c1e 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -45,6 +45,7 @@ import java.sql.SQLException; import java.time.OffsetDateTime; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -328,6 +329,25 @@ private void setUpsertStatementValues(PreparedStatement upsertStatement, Object // Some field types require special treatment to convert the java type to the // DB type if (field instanceof VectorStoreRecordVectorField) { + + // Convert the vector field to a string + if (field.getFieldType().equals(String.class)) { + String json = (valueNode == null || valueNode.isNull()) + ? null + : valueNode.asText(); + double[] values = (json == null) + ? null + : objectMapper.readValue(json, double[].class); + + int dim = ((VectorStoreRecordVectorField) field).getDimensions(); + if (values != null && values.length != dim) { + throw new SKException("Vector dimension mismatch: expected " + dim); + } + + upsertStatement.setObject(i + 1, values, OracleTypes.VECTOR_FLOAT32); + continue; + } + // If the vector field is not set as a string convert to an array of doubles // and set the value if (!field.getFieldType().equals(String.class)) { @@ -336,6 +356,7 @@ private void setUpsertStatementValues(PreparedStatement upsertStatement, Object : StreamSupport.stream(( (ArrayNode)valueNode).spliterator(), false) .mapToDouble(d -> d.asDouble()).toArray(); + upsertStatement.setObject(i + 1, values, OracleTypes.VECTOR_FLOAT32); continue; } diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index fd4f9dbf..2c3b3db3 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -6,6 +6,7 @@ */ package com.microsoft.semantickernel.data.jdbc.oracle; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -210,6 +211,17 @@ public OracleVectorStoreRecordMapper build() { } if (options != null && options.isIncludeVectors()) { for (VectorStoreRecordVectorField field : vectorStoreRecordDefinition.getVectorFields()) { + + // String vector + if (field.getFieldType().equals(String.class)) { + float[] arr = resultSet.getObject(field.getEffectiveStorageName(), float[].class); + String str = (arr == null) + ? null + : objectMapper.writeValueAsString(arr); + objectNode.put(field.getEffectiveStorageName(), str); + continue; + } + Object value = resultSet.getObject(field.getEffectiveStorageName(), float[].class); JsonNode genericNode = objectMapper.valueToTree(value); objectNode.set(field.getEffectiveStorageName(), genericNode); @@ -227,6 +239,8 @@ public OracleVectorStoreRecordMapper build() { throw new SKException( "Failure to serialize object, by default the JDBC connector uses Jackson, ensure your model object can be serialized by Jackson, i.e the class is visible, has getters, constructor, annotations etc.", e); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); } }); } From 578c733d48b4f17a96b695d8d76c4c81aef344c0 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Fri, 27 Jun 2025 10:34:32 +0200 Subject: [PATCH 204/277] Changes after Michael's code review --- .../OracleVectorStoreQueryProvider.java | 68 +++++++++---------- .../oracle/OracleVectorStoreRecordMapper.java | 5 +- ...OracleVectorStoreRecordCollectionTest.java | 2 +- 3 files changed, 33 insertions(+), 42 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index 151a1c1e..30c52ce3 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -6,7 +6,9 @@ */ package com.microsoft.semantickernel.data.jdbc.oracle; +import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -33,10 +35,13 @@ import oracle.jdbc.OracleStatement; import oracle.jdbc.OracleTypes; import oracle.sql.TIMESTAMPTZ; +import oracle.jdbc.provider.oson.OsonFactory; import javax.annotation.Nonnull; import javax.annotation.concurrent.GuardedBy; import javax.sql.DataSource; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.math.BigDecimal; import java.sql.Connection; import java.sql.PreparedStatement; @@ -48,12 +53,10 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.UUID; import java.util.concurrent.locks.ReentrantLock; import java.util.logging.Logger; import java.util.stream.Collectors; -import java.util.stream.StreamSupport; /** * JDBC Vector Store for the Oracle Database @@ -329,42 +332,33 @@ private void setUpsertStatementValues(PreparedStatement upsertStatement, Object // Some field types require special treatment to convert the java type to the // DB type if (field instanceof VectorStoreRecordVectorField) { - - // Convert the vector field to a string - if (field.getFieldType().equals(String.class)) { - String json = (valueNode == null || valueNode.isNull()) - ? null - : valueNode.asText(); - double[] values = (json == null) - ? null - : objectMapper.readValue(json, double[].class); - - int dim = ((VectorStoreRecordVectorField) field).getDimensions(); - if (values != null && values.length != dim) { - throw new SKException("Vector dimension mismatch: expected " + dim); - } - - upsertStatement.setObject(i + 1, values, OracleTypes.VECTOR_FLOAT32); - continue; - } - - // If the vector field is not set as a string convert to an array of doubles + // If the vector field is not set as a string convert to an array of floats // and set the value if (!field.getFieldType().equals(String.class)) { - double[] values = (valueNode == null || valueNode.isNull()) - ? null - : StreamSupport.stream(( - (ArrayNode)valueNode).spliterator(), false) - .mapToDouble(d -> d.asDouble()).toArray(); - - upsertStatement.setObject(i + 1, values, OracleTypes.VECTOR_FLOAT32); + if (valueNode != null && !valueNode.isNull() && valueNode.isArray()) { + final float[] values = new float[valueNode.size()]; + for (int j = 0; j < ((ArrayNode)valueNode).size(); j++) { + values[j] = ((ArrayNode)valueNode).get(j).floatValue(); + } + upsertStatement.setObject(i + 1, values, OracleTypes.VECTOR_FLOAT32); + } else { + upsertStatement.setNull(i + 1, OracleTypes.VECTOR_FLOAT32); + } continue; } } else if (field instanceof VectorStoreRecordDataField) { - // Lists are stored as JSON objects, write the list as a JSON string representation - // of the list. + // Lists are stored as JSON objects, write the list using the JDBC OSON + // extensions. if (field.getFieldType().equals(List.class)) { - upsertStatement.setObject(i + 1, objectMapper.writeValueAsString(valueNode)); + JsonFactory osonFactory = new OsonFactory(); + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + try (JsonGenerator osonGen = osonFactory.createGenerator(out)) { + objectMapper.writeValue(osonGen, valueNode); + } + upsertStatement.setBytes(i + 1, out.toByteArray()); + } catch (IOException ioEx) { + throw new SKException("Failed to convert list to JSON value", ioEx); + } continue; } // Convert UUID to string before setting the value. @@ -372,14 +366,14 @@ private void setUpsertStatementValues(PreparedStatement upsertStatement, Object upsertStatement.setObject(i + 1, valueNode.isNull() ? null : valueNode.asText()); continue; } - // Convert OffsetDateTime to TIMESTAMPTZ before setting the value. + // Convert value node (its representations depends on Jackson JSON features) + // to OffsetDateTime before setting the value. if (field.getFieldType().equals(OffsetDateTime.class)) { if (valueNode == null || valueNode.isNull()) { upsertStatement.setNull(i + 1, OracleTypes.TIMESTAMPTZ); } else { OffsetDateTime offsetDateTime = (OffsetDateTime) objectMapper.convertValue(valueNode, field.getFieldType()); - ((OraclePreparedStatement) upsertStatement).setTIMESTAMPTZ(i + 1, - TIMESTAMPTZ.of(offsetDateTime)); + upsertStatement.setObject(i + 1, offsetDateTime); } continue; } @@ -388,8 +382,8 @@ private void setUpsertStatementValues(PreparedStatement upsertStatement, Object // For all other field type use setObject with the field value upsertStatement.setObject(i + 1, objectMapper.convertValue(valueNode,field.getFieldType())); - } catch (SQLException | JsonProcessingException e) { - throw new RuntimeException(e); + } catch (SQLException e) { + throw new SKException(e); } } } diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index 2c3b3db3..2575261c 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -18,9 +18,7 @@ import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import oracle.jdbc.OracleResultSet; import oracle.jdbc.provider.oson.OsonModule; -import oracle.sql.TIMESTAMPTZ; import java.sql.ResultSet; import java.sql.SQLException; import java.util.Map; @@ -183,8 +181,7 @@ public OracleVectorStoreRecordMapper build() { value = resultSet.getBoolean(field.getEffectiveStorageName()); break; case OracleDataTypesMapping.OFFSET_DATE_TIME: - TIMESTAMPTZ timestamptz = ((OracleResultSet)resultSet).getTIMESTAMPTZ(field.getEffectiveStorageName()); - value = timestamptz != null ? timestamptz.offsetDateTimeValue() : null; + value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); break; case OracleDataTypesMapping.BYTE_ARRAY: value = resultSet.getBytes(field.getEffectiveStorageName()); diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 1d66a8e3..a2bd7af8 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -290,7 +290,7 @@ public void searchWithTagFilter() { .searchAsync(SEARCH_EMBEDDINGS, options).block().getResults(); assertNotNull(results); assertEquals(1, results.size()); - // The first hotel should be the most similar + // The second hotel contains the tag we are searching for assertEquals(hotels.get(1).getId(), results.get(0).getRecord().getId()); } From dd6be5e6adfa90e9e0e6791a5b4d1f5b4698e211 Mon Sep 17 00:00:00 2001 From: Kaiyuan Li Date: Sat, 28 Jun 2025 00:09:21 -0700 Subject: [PATCH 205/277] add more test to extended test --- .../oracle/OracleVectorStoreExtendedTest.java | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java index 7fe66229..834c52c1 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java @@ -3,6 +3,7 @@ import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreOptions; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider.StringTypeMapping; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollection; @@ -11,17 +12,23 @@ import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions; import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import com.microsoft.semantickernel.exceptions.SKException; + import org.junit.jupiter.api.Test; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertIterableEquals; @@ -184,7 +191,7 @@ void testVectorDimensionMismatch() { } @Test - void testNull() { + void testNullFieldValue() { VectorStoreRecordCollection collection = createCollection("test_null", DummyRecord.class, null); @@ -210,7 +217,59 @@ void testNull() { @Test void testSkipAndTop() { + VectorStoreRecordCollection collection = + createCollection( + "test_skip_and_top", + DummyRecord.class, + null); + + List l1 = new ArrayList<>(); + for (int i = 1; i <= 10; i++) { + l1.add(new DummyRecord("id" + i, i, (double) i, floatVec(i))); + } + collection.upsertBatchAsync(l1, null).block(); + + VectorSearchResults results = collection.searchAsync( + Collections.nCopies(8,0f), + VectorSearchOptions.builder() + .withIncludeVectors(true) + .withSkip(5) + .withTop(3) + .build() + ).block(); + + assertEquals(3, results.getResults().size()); + List ids = results.getResults().stream().map(r -> r.getRecord().getId()).collect( + Collectors.toList()); + assertEquals(Arrays.asList("id6","id7","id8"), ids); + + collection.deleteCollectionAsync().block(); + } + + // corner case for OracleVectorStoreRecordMapper + @Test + void testMapRecordToStorageModel_throws() { + VectorStoreRecordKeyField keyField = VectorStoreRecordKeyField.builder() + .withName("id") + .withStorageName("id") + .withFieldType(String.class) + .build(); + + VectorStoreRecordDefinition definition = + VectorStoreRecordDefinition.fromFields( + Arrays.asList(keyField) + ); + + OracleVectorStoreRecordMapper mapper = + OracleVectorStoreRecordMapper. builder() + .withRecordClass(DummyRecord.class) + .withVectorStoreRecordDefinition(definition) + .build(); + UnsupportedOperationException ex = assertThrows( + UnsupportedOperationException.class, + () -> mapper.mapRecordToStorageModel(new DummyRecord())); + assertEquals("Not implemented", ex.getMessage()); } private VectorStoreRecordCollection createCollection( From f5c9cfab57a94dbcb754e22bbbaae2260ddec09f Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Tue, 1 Jul 2025 21:55:24 +0200 Subject: [PATCH 206/277] Set search vector as array of float[] insteand of List as string --- .../data/jdbc/oracle/OracleVectorStoreQueryProvider.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index 30c52ce3..aeda5177 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -450,8 +450,11 @@ public VectorSearchResults search(String collectionName, List VectorSearchResults search(String collectionName, List(mapper.mapStorageModelToRecord(rs, getRecordOptions), score)); } } - } catch (SQLException | JsonProcessingException e) { + } catch (SQLException e) { throw new SKException("Search failed", e); } From 2a8d37240271e2a1c50ca00e5bc6dee4bce2f407 Mon Sep 17 00:00:00 2001 From: psilberk Date: Tue, 1 Jul 2025 16:04:28 -0700 Subject: [PATCH 207/277] Clean up and remove deps from pom --- data/semantickernel-data-oracle/pom.xml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml index b383e16d..f777f72f 100644 --- a/data/semantickernel-data-oracle/pom.xml +++ b/data/semantickernel-data-oracle/pom.xml @@ -43,17 +43,6 @@ jackson-core compile - - com.fasterxml.jackson.datatype - jackson-datatype-jsr310 - 2.18.0 - - com.oracle.database.jdbc ojdbc11 @@ -64,7 +53,6 @@ ojdbc-provider-jackson-oson 1.0.4 - org.junit.jupiter junit-jupiter From c793b82a2284c4caef00f847bff47e261c7b5385 Mon Sep 17 00:00:00 2001 From: psilberk Date: Tue, 1 Jul 2025 22:29:17 -0700 Subject: [PATCH 208/277] Remove unused imports --- .../oracle/OracleVectorStoreFieldHelper.java | 2 -- .../OracleVectorStoreQueryProvider.java | 23 ++++++++----------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java index c177131f..7fe4895b 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java @@ -8,10 +8,8 @@ import com.microsoft.semantickernel.data.jdbc.oracle.OracleVectorStoreQueryProvider.StringTypeMapping; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; -import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; -import oracle.jdbc.OracleTypes; import java.math.BigDecimal; import java.time.OffsetDateTime; import java.util.Collection; diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index aeda5177..9de32ef3 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -7,7 +7,6 @@ package com.microsoft.semantickernel.data.jdbc.oracle; import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.MapperFeature; @@ -16,7 +15,7 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; import com.microsoft.semantickernel.data.filter.EqualToFilterClause; -import com.microsoft.semantickernel.data.jdbc.*; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreQueryProvider; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; @@ -31,32 +30,30 @@ import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import oracle.jdbc.OraclePreparedStatement; -import oracle.jdbc.OracleStatement; -import oracle.jdbc.OracleTypes; -import oracle.sql.TIMESTAMPTZ; -import oracle.jdbc.provider.oson.OsonFactory; - -import javax.annotation.Nonnull; -import javax.annotation.concurrent.GuardedBy; -import javax.sql.DataSource; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.math.BigDecimal; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.sql.Statement; import java.sql.SQLException; +import java.sql.Statement; import java.time.OffsetDateTime; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.UUID; import java.util.concurrent.locks.ReentrantLock; import java.util.logging.Logger; import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.concurrent.GuardedBy; +import javax.sql.DataSource; +import oracle.jdbc.OraclePreparedStatement; +import oracle.jdbc.OracleStatement; +import oracle.jdbc.OracleTypes; +import oracle.jdbc.provider.oson.OsonFactory; +import oracle.sql.TIMESTAMPTZ; /** * JDBC Vector Store for the Oracle Database From 3b299ee5c1b1f26272b69fa541c4d948330f09b1 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 7 Jul 2025 18:37:54 +0200 Subject: [PATCH 209/277] Copyright and license --- .../jdbc/oracle/OracleDataTypesMapping.java | 25 ++++++++++++++-- .../oracle/OracleVectorStoreFieldHelper.java | 25 ++++++++++++++-- .../OracleVectorStoreQueryProvider.java | 25 ++++++++++++++-- .../oracle/OracleVectorStoreRecordMapper.java | 25 ++++++++++++++-- .../jdbc/oracle/ClassWithAllBoxedTypes.java | 25 ++++++++++++++-- .../oracle/ClassWithAllPrimitiveTypes.java | 25 ++++++++++++++-- .../jdbc/oracle/ClassWithAnnotatedTypes.java | 25 ++++++++++++++-- .../data/jdbc/oracle/Hotel.java | 29 ++++++++++++++++--- ...CommonVectorStoreRecordCollectionTest.java | 25 ++++++++++++++-- .../OracleVectorStoreAnnotatedTypeTest.java | 25 ++++++++++++++-- .../OracleVectorStoreDataTypeSearchTest.java | 25 ++++++++++++++-- .../oracle/OracleVectorStoreDataTypeTest.java | 25 ++++++++++++++-- .../oracle/OracleVectorStoreExtendedTest.java | 25 ++++++++++++++++ ...OracleVectorStoreRecordCollectionTest.java | 25 ++++++++++++++-- 14 files changed, 314 insertions(+), 40 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java index cfca9ecc..efde970c 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java index 7fe4895b..369a036a 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index 9de32ef3..239be7a0 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index 2575261c..c8ca4b32 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllBoxedTypes.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllBoxedTypes.java index 307c974d..14b96c14 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllBoxedTypes.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllBoxedTypes.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllPrimitiveTypes.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllPrimitiveTypes.java index 0ca52d7b..6a8d76d0 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllPrimitiveTypes.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAllPrimitiveTypes.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java index 5964cb3c..75190deb 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/ClassWithAnnotatedTypes.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java index dc736ba3..19d31071 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java @@ -1,7 +1,28 @@ -// Copyright (c) Microsoft. All rights reserved. - -// Should we change the copyright from the file we copied? Or should we -// just change the test to use a class that we create? +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleCommonVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleCommonVectorStoreRecordCollectionTest.java index 7549188b..ce260c77 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleCommonVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleCommonVectorStoreRecordCollectionTest.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java index 42036826..e0332950 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreAnnotatedTypeTest.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java index f81176b3..bac0ee4c 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeSearchTest.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java index cc8d2462..eeb6a2e7 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreDataTypeTest.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java index 834c52c1..50c0a608 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java @@ -1,3 +1,28 @@ +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index a2bd7af8..41bb0cee 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -1,8 +1,27 @@ /* - ** Semantic Kernel Oracle connector version 1.0. + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) ** - ** Copyright (c) 2025 Oracle and/or its affiliates. - ** Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. */ package com.microsoft.semantickernel.data.jdbc.oracle; From 632e38867f3b010b9c34bac1d7b04c75926b4bb3 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 7 Jul 2025 18:43:46 +0200 Subject: [PATCH 210/277] Samples --- .../data/jdbc/oracle/Hotel.java | 26 +------------------ ...OracleVectorStoreRecordCollectionTest.java | 25 ------------------ .../data/vectorstores/oracle/Book.java | 26 ++++++++++++++++++- .../data/vectorstores/oracle/Main.java | 26 ++++++++++++++++++- 4 files changed, 51 insertions(+), 52 deletions(-) diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java index 19d31071..0f93ff7f 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java @@ -1,28 +1,4 @@ -/* - ** Oracle Database Vector Store Connector for Semantic Kernel (Java) - ** - ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. - ** - ** The MIT License (MIT) - ** - ** Permission is hereby granted, free of charge, to any person obtaining a copy - ** of this software and associated documentation files (the "Software"), to - ** deal in the Software without restriction, including without limitation the - ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - ** sell copies of the Software, and to permit persons to whom the Software is - ** furnished to do so, subject to the following conditions: - ** - ** The above copyright notice and this permission notice shall be included in - ** all copies or substantial portions of the Software. - ** - ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - ** IN THE SOFTWARE. - */ + package com.microsoft.semantickernel.data.jdbc.oracle; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 41bb0cee..a6a197ec 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -1,28 +1,3 @@ -/* - ** Oracle Database Vector Store Connector for Semantic Kernel (Java) - ** - ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. - ** - ** The MIT License (MIT) - ** - ** Permission is hereby granted, free of charge, to any person obtaining a copy - ** of this software and associated documentation files (the "Software"), to - ** deal in the Software without restriction, including without limitation the - ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - ** sell copies of the Software, and to permit persons to whom the Software is - ** furnished to do so, subject to the following conditions: - ** - ** The above copyright notice and this permission notice shall be included in - ** all copies or substantial portions of the Software. - ** - ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - ** IN THE SOFTWARE. - */ package com.microsoft.semantickernel.data.jdbc.oracle; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java index 87104cf8..8de0b5aa 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java @@ -1,4 +1,28 @@ -// Copyright (c) Microsoft. All rights reserved. +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ package com.microsoft.semantickernel.samples.documentationexamples.data.vectorstores.oracle; import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordData; diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java index 570dfc60..c03f3eb3 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Main.java @@ -1,4 +1,28 @@ -// Copyright (c) Microsoft. All rights reserved. +/* + ** Oracle Database Vector Store Connector for Semantic Kernel (Java) + ** + ** Copyright (c) 2025 Oracle and/or its affiliates. All rights reserved. + ** + ** The MIT License (MIT) + ** + ** Permission is hereby granted, free of charge, to any person obtaining a copy + ** of this software and associated documentation files (the "Software"), to + ** deal in the Software without restriction, including without limitation the + ** rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + ** sell copies of the Software, and to permit persons to whom the Software is + ** furnished to do so, subject to the following conditions: + ** + ** The above copyright notice and this permission notice shall be included in + ** all copies or substantial portions of the Software. + ** + ** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + ** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + ** IN THE SOFTWARE. + */ package com.microsoft.semantickernel.samples.documentationexamples.data.vectorstores.oracle; import com.microsoft.semantickernel.data.jdbc.JDBCVectorStore; From 68f7914a46ce0d8a4a018f305a636f0e4ed04962 Mon Sep 17 00:00:00 2001 From: psilberk Date: Wed, 9 Jul 2025 14:34:30 -0700 Subject: [PATCH 211/277] Addressing review comments --- pom.xml | 1 - .../semantickernel-syntax-examples/pom.xml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 3fb4158d..b52f7484 100644 --- a/pom.xml +++ b/pom.xml @@ -71,7 +71,6 @@ semantickernel-bom semantickernel-api semantickernel-experimental - samples aiservices/openai aiservices/google aiservices/huggingface diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index d5471730..2392ee65 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -139,7 +139,7 @@ com.microsoft.semantic-kernel semantickernel-data-jdbc - 1.4.4-SNAPSHOT + ${project.version} com.microsoft.semantic-kernel From e6690176273f34fe23d360d65d2a6f1376940615 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Fri, 25 Jul 2025 17:24:19 +0200 Subject: [PATCH 212/277] Addressed some of the review comments --- .vscode/settings.json | 3 +- data/semantickernel-data-jdbc/pom.xml | 5 +- .../jdbc/JDBCVectorStoreQueryProvider.java | 2 +- .../oracle/OracleVectorStoreFieldHelper.java | 72 ++++--- .../OracleVectorStoreQueryProvider.java | 45 ++++- .../oracle/OracleVectorStoreRecordMapper.java | 183 +++++++++--------- .../semantickernel-syntax-examples/pom.xml | 2 +- 7 files changed, 185 insertions(+), 127 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 16667e41..6f3bd17c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -21,5 +21,6 @@ }, }, "java.debug.settings.onBuildFailureProceed": true, - "java.compile.nullAnalysis.mode": "disabled" + "java.compile.nullAnalysis.mode": "disabled", + "java.configuration.updateBuildConfiguration": "interactive" } diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index e9b18f75..592cbbe5 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-SNAPSHOT + 1.4.4-RC2-SNAPSHOT ../../pom.xml @@ -15,9 +15,8 @@ com.microsoft.semantic-kernel - semantickernel-api + semantickernel-api-data - org.slf4j slf4j-api diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java index 9381ef06..f19d0c22 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -106,7 +106,7 @@ public JDBCVectorStoreQueryProvider( @SuppressFBWarnings("EI_EXPOSE_REP2") @Nonnull DataSource dataSource, @Nonnull String collectionsTable, @Nonnull String prefixForCollectionTables, - @Nonnull HashMap, String> supportedKeyTypes, + @Nonnull Map, String> supportedKeyTypes, @Nonnull Map, String> supportedDataTypes, @Nonnull Map, String> supportedVectorTypes) { this.dataSource = dataSource; diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java index 369a036a..d6592215 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java @@ -29,9 +29,11 @@ import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.vectorstorage.definition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.exceptions.SKException; import java.math.BigDecimal; import java.time.OffsetDateTime; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -45,6 +47,10 @@ */ class OracleVectorStoreFieldHelper { + /** + * Object naming regular expression + */ + private static final String OBJECT_NAMING_REGEXP = "[a-zA-Z_][a-zA-Z0-9_]{1,128}"; /** * The logger */ @@ -109,8 +115,9 @@ class OracleVectorStoreFieldHelper { * * @return the mapping between the supported Java key types and the Oracle database type. */ - public static HashMap, String> getSupportedKeyTypes() { - return supportedKeyTypes; + static Map, String> getSupportedKeyTypes() { + + return Collections.unmodifiableMap(supportedKeyTypes); } /** @@ -118,14 +125,14 @@ public static HashMap, String> getSupportedKeyTypes() { * * @return the mapping between the supported Java data types and the Oracle database type. */ - public static Map, String> getSupportedDataTypes( + static Map, String> getSupportedDataTypes( StringTypeMapping stringTypeMapping, int defaultVarCharLength) { String stringType = stringTypeMapping.equals(StringTypeMapping.USE_VARCHAR) ? String.format(OracleDataTypesMapping.STRING_VARCHAR, defaultVarCharLength) : OracleDataTypesMapping.STRING_CLOB; supportedDataTypes.put(String.class, stringType); LOGGER.finest("Mapping String columns to " + stringType); - return supportedDataTypes; + return Collections.unmodifiableMap(supportedDataTypes); } /** @@ -133,8 +140,9 @@ public static Map, String> getSupportedDataTypes( * * @return the mapping between the supported Java data types and the Oracle database type. */ - public static Map, String> getSupportedVectorTypes() { - return supportedVectorTypes; + static Map, String> getSupportedVectorTypes() { + + return Collections.unmodifiableMap(supportedVectorTypes); } /** @@ -143,20 +151,23 @@ public static Map, String> getSupportedVectorTypes() { * @return the CREATE VECTOR INDEX statement to create the index according to the vector * field definition. */ - public static String getCreateVectorIndexStatement(VectorStoreRecordVectorField field, String collectionTableName) { + static String getCreateVectorIndexStatement(VectorStoreRecordVectorField field, String collectionTableName) { switch (field.getIndexKind()) { case IVFFLAT: return "CREATE VECTOR INDEX IF NOT EXISTS " - + getIndexName(field.getEffectiveStorageName()) + + validateObjectNaming(getIndexName(field.getEffectiveStorageName())) + " ON " - + collectionTableName + "( " + field.getEffectiveStorageName() + " ) " + + validateObjectNaming(collectionTableName) + + "( " + validateObjectNaming(field.getEffectiveStorageName()) + " ) " + " ORGANIZATION NEIGHBOR PARTITIONS " + " WITH DISTANCE COSINE " + "PARAMETERS ( TYPE IVF )"; case HNSW: - return "CREATE VECTOR INDEX IF NOT EXISTS " + getIndexName(field.getEffectiveStorageName()) + return "CREATE VECTOR INDEX IF NOT EXISTS " + + validateObjectNaming(getIndexName(field.getEffectiveStorageName())) + " ON " - + collectionTableName + "( " + field.getEffectiveStorageName() + " ) " + + validateObjectNaming(collectionTableName) + + "( " + validateObjectNaming(field.getEffectiveStorageName()) + " ) " + "ORGANIZATION INMEMORY GRAPH " + "WITH DISTANCE COSINE " + "PARAMETERS (TYPE HNSW)"; @@ -173,20 +184,20 @@ public static String getCreateVectorIndexStatement(VectorStoreRecordVectorField * * @return the CREATE INDEX statement to create the index according to the field definition. */ - public static String createIndexForDataField(String collectionTableName, VectorStoreRecordDataField dataField, Map, String> supportedDataTypes) { + static String createIndexForDataField(String collectionTableName, VectorStoreRecordDataField dataField, Map, String> supportedDataTypes) { if (supportedDataTypes.get(dataField.getFieldType()) == "JSON") { String dataFieldIndex = "CREATE MULTIVALUE INDEX IF NOT EXISTS %s ON %s t (t.%s.%s)"; return String.format(dataFieldIndex, - collectionTableName + "_" + dataField.getEffectiveStorageName(), - collectionTableName, - dataField.getEffectiveStorageName(), + validateObjectNaming(collectionTableName + "_" + dataField.getEffectiveStorageName()), + validateObjectNaming(collectionTableName), + validateObjectNaming(dataField.getEffectiveStorageName()), getFunctionForType(supportedDataTypes.get(dataField.getFieldSubType()))); } else { String dataFieldIndex = "CREATE INDEX IF NOT EXISTS %s ON %s (%s ASC)"; return String.format(dataFieldIndex, - collectionTableName + "_" + dataField.getEffectiveStorageName(), - collectionTableName, - dataField.getEffectiveStorageName() + validateObjectNaming(collectionTableName + "_" + dataField.getEffectiveStorageName()), + validateObjectNaming(collectionTableName), + validateObjectNaming(dataField.getEffectiveStorageName()) ); } } @@ -196,9 +207,9 @@ public static String createIndexForDataField(String collectionTableName, VectorS * @param fields list of vector record fields. * @return comma separated list of columns and types for CREATE TABLE statement. */ - public static String getVectorColumnNamesAndTypes(List fields) { + static String getVectorColumnNamesAndTypes(List fields) { List columns = fields.stream() - .map(field -> field.getEffectiveStorageName() + " " + + .map(field -> validateObjectNaming(field.getEffectiveStorageName()) + " " + OracleVectorStoreFieldHelper.getTypeForVectorField(field) ).collect(Collectors.toList()); @@ -210,8 +221,8 @@ public static String getVectorColumnNamesAndTypes(List VectorSearchResults search(String collectionName, List mapper) { + + if (vector != null && recordDefinition.getVectorFields().isEmpty()) { + throw new SKException("Record definition must contain at least one vector field" + + " to perform a vector search"); + } + // Gets the search vector field and its distance function. If not vector field was provided, // use the first one - VectorStoreRecordVectorField vectorField = options.getVectorFieldName() == null - ? recordDefinition.getVectorFields().get(0) - : (VectorStoreRecordVectorField) recordDefinition - .getField(options.getVectorFieldName()); - DistanceFunction distanceFunction = vectorField == null ? null : vectorField.getDistanceFunction(); - if (options.getVectorFieldName() != null && vectorField == null) { - throw new SKException(""); + VectorStoreRecordVectorField vectorField = null; + if (vector != null) { + vectorField = getVectorFieldByName(recordDefinition, options.getVectorFieldName()); } + + // get list of fields that should be returned by the query List fields = (options.isIncludeVectors()) ? recordDefinition.getAllFields() @@ -449,7 +453,9 @@ public VectorSearchResults search(String collectionName, List VectorSearchResults search(String collectionName, List VectorSearchResults search(String collectionName, List(records); } + private VectorStoreRecordVectorField getVectorFieldByName( + VectorStoreRecordDefinition recordDefinition, + String name) { + VectorStoreRecordField vectorField; + if (name != null) { + vectorField = recordDefinition.getField(name); + if (vectorField == null) { + throw new SKException("Vector field not found in record definition"); + } + if (!(vectorField instanceof VectorStoreRecordVectorField)) { + throw new SKException("Invalid type"); + } + } else { + if (recordDefinition.getVectorFields().isEmpty()) { + throw new SKException("Record definition should contain at least one vector field"); + } + vectorField = recordDefinition.getVectorFields().get(0); + } + return (VectorStoreRecordVectorField)vectorField; + } + /** * Sets the parameter value * @param statement the statement diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index c8ca4b32..bed0b5ed 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -161,104 +161,109 @@ public OracleVectorStoreRecordMapper build() { return new OracleVectorStoreRecordMapper<>( (resultSet, options) -> { - try { - objectMapper.registerModule(new OsonModule()); - // Create an ObjectNode to hold the values - ObjectNode objectNode = objectMapper.createObjectNode(); + return MapResultSetToRecord(resultSet, options); + }); + } - // Read non vector fields - for (VectorStoreRecordField field : vectorStoreRecordDefinition.getNonVectorFields()) { - Class fieldType = field.getFieldType(); + private Record MapResultSetToRecord(ResultSet resultSet, GetRecordOptions options) { + try { + objectMapper.registerModule(new OsonModule()); + // Create an ObjectNode to hold the values + ObjectNode objectNode = objectMapper.createObjectNode(); - Object value; - switch (supportedDataTypesMapping.get(fieldType)) { - case OracleDataTypesMapping.STRING_CLOB: - value = resultSet.getString(field.getEffectiveStorageName()); - break; - case OracleDataTypesMapping.BYTE: - value = resultSet.getByte(field.getEffectiveStorageName()); - break; - case OracleDataTypesMapping.SHORT: - value = resultSet.getShort(field.getEffectiveStorageName()); - break; - case OracleDataTypesMapping.INTEGER: - value = resultSet.getInt(field.getEffectiveStorageName()); - break; - case OracleDataTypesMapping.LONG: - value = resultSet.getLong(field.getEffectiveStorageName()); - break; - case OracleDataTypesMapping.FLOAT: - value = resultSet.getFloat(field.getEffectiveStorageName()); - break; - case OracleDataTypesMapping.DOUBLE: - value = resultSet.getDouble(field.getEffectiveStorageName()); - break; - case OracleDataTypesMapping.DECIMAL: - value = resultSet.getBigDecimal(field.getEffectiveStorageName()); - break; - case OracleDataTypesMapping.BOOLEAN: - value = resultSet.getBoolean(field.getEffectiveStorageName()); - break; - case OracleDataTypesMapping.OFFSET_DATE_TIME: - value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); - break; - case OracleDataTypesMapping.BYTE_ARRAY: - value = resultSet.getBytes(field.getEffectiveStorageName()); - break; - case OracleDataTypesMapping.UUID: - String uuidValue = resultSet.getString(field.getEffectiveStorageName()); - value = uuidValue == null ? null : UUID.fromString(uuidValue); - break; - case OracleDataTypesMapping.JSON: - value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); - break; - default: - value = resultSet.getString(field.getEffectiveStorageName()); - } - // Result set getter method sometimes returns a default value when NULL, - // set value to null in that case. - if (resultSet.wasNull()) { - value = null; - } + // Read non vector fields + for (VectorStoreRecordField field : vectorStoreRecordDefinition.getNonVectorFields()) { + Class fieldType = field.getFieldType(); - JsonNode genericNode = objectMapper.valueToTree(value); + Object value; + switch (supportedDataTypesMapping.get(fieldType)) { + case OracleDataTypesMapping.STRING_CLOB: + value = resultSet.getString(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.BYTE: + value = resultSet.getByte(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.SHORT: + value = resultSet.getShort(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.INTEGER: + value = resultSet.getInt(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.LONG: + value = resultSet.getLong(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.FLOAT: + value = resultSet.getFloat(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.DOUBLE: + value = resultSet.getDouble(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.DECIMAL: + value = resultSet.getBigDecimal(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.BOOLEAN: + value = resultSet.getBoolean(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.OFFSET_DATE_TIME: + value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); + break; + case OracleDataTypesMapping.BYTE_ARRAY: + value = resultSet.getBytes(field.getEffectiveStorageName()); + break; + case OracleDataTypesMapping.UUID: + String uuidValue = resultSet.getString(field.getEffectiveStorageName()); + value = uuidValue == null ? null : UUID.fromString(uuidValue); + break; + case OracleDataTypesMapping.JSON: + value = resultSet.getObject(field.getEffectiveStorageName(), fieldType); + break; + default: + value = resultSet.getString(field.getEffectiveStorageName()); + } + // Result set getter method sometimes returns a default value when NULL, + // set value to null in that case. + if (resultSet.wasNull()) { + value = null; + } - objectNode.set(field.getEffectiveStorageName(), genericNode); - } - if (options != null && options.isIncludeVectors()) { - for (VectorStoreRecordVectorField field : vectorStoreRecordDefinition.getVectorFields()) { + JsonNode genericNode = objectMapper.valueToTree(value); - // String vector - if (field.getFieldType().equals(String.class)) { - float[] arr = resultSet.getObject(field.getEffectiveStorageName(), float[].class); - String str = (arr == null) - ? null - : objectMapper.writeValueAsString(arr); - objectNode.put(field.getEffectiveStorageName(), str); - continue; - } + objectNode.set(field.getEffectiveStorageName(), genericNode); + } + if (options != null && options.isIncludeVectors()) { + for (VectorStoreRecordVectorField field : vectorStoreRecordDefinition.getVectorFields()) { - Object value = resultSet.getObject(field.getEffectiveStorageName(), float[].class); - JsonNode genericNode = objectMapper.valueToTree(value); - objectNode.set(field.getEffectiveStorageName(), genericNode); - } - } else { - for (VectorStoreRecordVectorField field : vectorStoreRecordDefinition.getVectorFields()) { - JsonNode genericNode = objectMapper.valueToTree(null); - objectNode.set(field.getEffectiveStorageName(), genericNode); - } + // String vector + if (field.getFieldType().equals(String.class)) { + float[] arr = resultSet.getObject(field.getEffectiveStorageName(), float[].class); + String str = (arr == null) + ? null + : objectMapper.writeValueAsString(arr); + objectNode.put(field.getEffectiveStorageName(), str); + continue; } - // Deserialize the object node to the record class - return objectMapper.convertValue(objectNode, recordClass); - } catch (SQLException e) { - throw new SKException( - "Failure to serialize object, by default the JDBC connector uses Jackson, ensure your model object can be serialized by Jackson, i.e the class is visible, has getters, constructor, annotations etc.", - e); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); + Object value = resultSet.getObject(field.getEffectiveStorageName(), float[].class); + JsonNode genericNode = objectMapper.valueToTree(value); + objectNode.set(field.getEffectiveStorageName(), genericNode); } - }); + } else { + for (VectorStoreRecordVectorField field : vectorStoreRecordDefinition.getVectorFields()) { + JsonNode genericNode = objectMapper.valueToTree(null); + objectNode.set(field.getEffectiveStorageName(), genericNode); + } + } + + // Deserialize the object node to the record class + return objectMapper.convertValue(objectNode, recordClass); + } catch (SQLException e) { + throw new SKException( + "Failure to serialize object, by default the JDBC connector uses Jackson, ensure your model object can be serialized by Jackson, i.e the class is visible, has getters, constructor, annotations etc.", + e); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } } } + } diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 2392ee65..cb735f9c 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -144,7 +144,7 @@ com.microsoft.semantic-kernel semantickernel-learn-resources - 1.4.4-SNAPSHOT + 1.4.4-RC2-SNAPSHOT compile From 4f919c33502bd5dd3f52ea2307c539e4c592cbbc Mon Sep 17 00:00:00 2001 From: psilberk Date: Wed, 6 Aug 2025 21:15:34 -0700 Subject: [PATCH 213/277] Addressed DoubleBraceInitialization warning/error --- .../oracle/OracleVectorStoreFieldHelper.java | 71 +++++++++---------- 1 file changed, 34 insertions(+), 37 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java index d6592215..bf4dd295 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreFieldHelper.java @@ -59,51 +59,48 @@ class OracleVectorStoreFieldHelper { /** * Maps supported key java classes to Oracle database types */ - private static final HashMap, String> supportedKeyTypes = new HashMap() { - { - put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, 255)); - } - }; + private static final HashMap, String> supportedKeyTypes = new HashMap(); + static { + supportedKeyTypes.put(String.class, String.format(OracleDataTypesMapping.STRING_VARCHAR, 255)); + } /** * Maps supported vector java classes to Oracle database types */ - private static final Map, String> supportedVectorTypes = new HashMap() { - { - put(String.class, OracleDataTypesMapping.VECTOR_FLOAT); - put(List.class, OracleDataTypesMapping.VECTOR_FLOAT); - put(Collection.class, OracleDataTypesMapping.VECTOR_FLOAT); - put(float[].class, OracleDataTypesMapping.VECTOR_FLOAT); - put(Float[].class, OracleDataTypesMapping.VECTOR_FLOAT); - } - }; + private static final Map, String> supportedVectorTypes = new HashMap(); + static { + supportedVectorTypes.put(String.class, OracleDataTypesMapping.VECTOR_FLOAT); + supportedVectorTypes.put(List.class, OracleDataTypesMapping.VECTOR_FLOAT); + supportedVectorTypes.put(Collection.class, OracleDataTypesMapping.VECTOR_FLOAT); + supportedVectorTypes.put(float[].class, OracleDataTypesMapping.VECTOR_FLOAT); + supportedVectorTypes.put(Float[].class, OracleDataTypesMapping.VECTOR_FLOAT); + } /** * Maps supported data java classes to Oracle database types */ - private static final HashMap, String> supportedDataTypes = new HashMap() { - { - put(byte.class, OracleDataTypesMapping.BYTE); - put(Byte.class, OracleDataTypesMapping.BYTE); - put(short.class, OracleDataTypesMapping.SHORT); - put(Short.class, OracleDataTypesMapping.SHORT); - put(int.class, OracleDataTypesMapping.INTEGER); - put(Integer.class, OracleDataTypesMapping.INTEGER); - put(long.class, OracleDataTypesMapping.LONG); - put(Long.class, OracleDataTypesMapping.LONG); - put(Float.class, OracleDataTypesMapping.FLOAT); - put(float.class, OracleDataTypesMapping.FLOAT); - put(Double.class, OracleDataTypesMapping.DOUBLE); - put(double.class, OracleDataTypesMapping.DOUBLE); - put(BigDecimal.class, OracleDataTypesMapping.DECIMAL); - put(Boolean.class, OracleDataTypesMapping.BOOLEAN); - put(boolean.class, OracleDataTypesMapping.BOOLEAN); - put(OffsetDateTime.class, OracleDataTypesMapping.OFFSET_DATE_TIME); - put(UUID.class, OracleDataTypesMapping.UUID); - put(byte[].class, OracleDataTypesMapping.BYTE_ARRAY); - put(List.class, OracleDataTypesMapping.JSON); - } - }; + private static final HashMap, String> supportedDataTypes = new HashMap(); + static { + supportedDataTypes.put(byte.class, OracleDataTypesMapping.BYTE); + supportedDataTypes.put(Byte.class, OracleDataTypesMapping.BYTE); + supportedDataTypes.put(short.class, OracleDataTypesMapping.SHORT); + supportedDataTypes.put(Short.class, OracleDataTypesMapping.SHORT); + supportedDataTypes.put(int.class, OracleDataTypesMapping.INTEGER); + supportedDataTypes.put(Integer.class, OracleDataTypesMapping.INTEGER); + supportedDataTypes.put(long.class, OracleDataTypesMapping.LONG); + supportedDataTypes.put(Long.class, OracleDataTypesMapping.LONG); + supportedDataTypes.put(Float.class, OracleDataTypesMapping.FLOAT); + supportedDataTypes.put(float.class, OracleDataTypesMapping.FLOAT); + supportedDataTypes.put(Double.class, OracleDataTypesMapping.DOUBLE); + supportedDataTypes.put(double.class, OracleDataTypesMapping.DOUBLE); + supportedDataTypes.put(BigDecimal.class, OracleDataTypesMapping.DECIMAL); + supportedDataTypes.put(Boolean.class, OracleDataTypesMapping.BOOLEAN); + supportedDataTypes.put(boolean.class, OracleDataTypesMapping.BOOLEAN); + supportedDataTypes.put(OffsetDateTime.class, OracleDataTypesMapping.OFFSET_DATE_TIME); + supportedDataTypes.put(UUID.class, OracleDataTypesMapping.UUID); + supportedDataTypes.put(byte[].class, OracleDataTypesMapping.BYTE_ARRAY); + supportedDataTypes.put(List.class, OracleDataTypesMapping.JSON); + } /** * Suffix added to the effective column name to generate the index name for a vector column. From e55bda371f345023050d7c9376a7684c1f0c33f4 Mon Sep 17 00:00:00 2001 From: psilberk Date: Thu, 7 Aug 2025 16:55:50 -0700 Subject: [PATCH 214/277] Fixed typos in comments --- .../data/jdbc/oracle/OracleVectorStoreQueryProvider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index b42678a2..e52364c4 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -494,7 +494,7 @@ public VectorSearchResults search(String collectionName, List * @throws SQLException if an error occurs while defining the column type */ private void defineDataColumnType(int columnIndex, OracleStatement statement, Class fieldType) throws SQLException { - // swich between supported classes and define the column type on the statement + // switch between supported classes and define the column type on the statement switch (supportedDataTypes.get(fieldType)) { case OracleDataTypesMapping.STRING_CLOB: statement.defineColumnType(columnIndex, OracleTypes.CLOB, Integer.MAX_VALUE); From 1c556af0eb7c7c621d25d5e2c950832ae4421065 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 11 Aug 2025 18:49:45 +0200 Subject: [PATCH 215/277] Disable Android compatibility check --- data/semantickernel-data-oracle/pom.xml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml index f777f72f..6fd20ba1 100644 --- a/data/semantickernel-data-oracle/pom.xml +++ b/data/semantickernel-data-oracle/pom.xml @@ -79,4 +79,24 @@ test + + + + org.codehaus.mojo + animal-sniffer-maven-plugin + + + android + test + + check + + + + + true + + + + \ No newline at end of file From 3b49161b1beceb0406ca81e1b6c6e7ba50385550 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Tue, 12 Aug 2025 09:47:02 +0200 Subject: [PATCH 216/277] Fixed errors returned by CI on JDK8 --- .../data/jdbc/oracle/OracleVectorStoreQueryProvider.java | 7 +++---- .../data/jdbc/oracle/OracleVectorStoreRecordMapper.java | 9 ++------- .../data/jdbc/oracle/OracleVectorStoreExtendedTest.java | 1 - .../oracle/OracleVectorStoreRecordCollectionTest.java | 3 --- 4 files changed, 5 insertions(+), 15 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java index e52364c4..e3fa157b 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreQueryProvider.java @@ -245,6 +245,7 @@ public void createCollection(String collectionName, * @param options the options */ @Override + @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, @@ -422,6 +423,7 @@ private void setUpsertStatementValues(PreparedStatement upsertStatement, Object * @param the record type */ @Override + @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") public VectorSearchResults search(String collectionName, List vector, VectorSearchOptions options, VectorStoreRecordDefinition recordDefinition, VectorStoreRecordMapper mapper) { @@ -558,8 +560,6 @@ private void setSearchParameter(PreparedStatement statement, int index, Class // Use JSON string to set lists if (List.class.equals(type)) { statement.setObject(index, objectMapper.writeValueAsString(value)); - System.out.println( - "Set values: " + objectMapper.writeValueAsString(value)); return; } // convert UUID to string @@ -575,7 +575,6 @@ private void setSearchParameter(PreparedStatement statement, int index, Class OffsetDateTime offsetDateTime = (OffsetDateTime) value; ((OraclePreparedStatement) statement).setTIMESTAMPTZ(index, TIMESTAMPTZ.of(offsetDateTime)); - System.out.println("Set values: " + offsetDateTime); } return; } @@ -587,7 +586,6 @@ private void setSearchParameter(PreparedStatement statement, int index, Class BigDecimal bigDecimal = (BigDecimal) value; ((OraclePreparedStatement) statement).setBigDecimal(index, bigDecimal); - System.out.println("Set values: " + bigDecimal); } return; } @@ -825,6 +823,7 @@ public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { * @param objectMapper the object mapper * @return the builder */ + @SuppressFBWarnings("EI_EXPOSE_REP2") public Builder withObjectMapper( ObjectMapper objectMapper) { this.objectMapper = objectMapper; diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index bed0b5ed..3b62f07d 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -41,6 +41,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.Map; +import java.util.HashMap; import java.util.UUID; import java.util.function.BiFunction; @@ -91,7 +92,6 @@ public static class Builder private VectorStoreRecordDefinition vectorStoreRecordDefinition; private Map, String> supportedDataTypesMapping; private ObjectMapper objectMapper = new ObjectMapper(); - private Map, String> annotatedTypeMapping; /** * Sets the record class. @@ -137,12 +137,7 @@ public Builder withObjectMapper(ObjectMapper objectMapper) { */ public Builder withSupportedDataTypesMapping( Map, String> supportedDataTypesMapping) { - this.supportedDataTypesMapping = supportedDataTypesMapping; - return this; - } - - public Builder withAnnotatedTypeMapping(Map, String> annotatedTypeMapping) { - this.annotatedTypeMapping = annotatedTypeMapping; + this.supportedDataTypesMapping = new HashMap<>(supportedDataTypesMapping); return this; } diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java index 50c0a608..af2e28fd 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreExtendedTest.java @@ -203,7 +203,6 @@ void testVectorDimensionMismatch() { DummyRecord d1 = new DummyRecord("id1", 4, 120d, new float[]{}); SKException ex = assertThrows(SKException.class, () -> collection.upsertBatchAsync(Arrays.asList(d1), null).block()); - System.out.println(ex.getMessage()); assertTrue(ex.getCause().getMessage().contains("ORA-51803")); // Vector dimension mismatch diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index a6a197ec..238a5ef2 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -215,9 +215,6 @@ public void exactSearch(DistanceFunction distanceFunction, List expected assertNotNull(results); assertEquals(3, results.size()); // The third hotel should be the most similar - System.out.println(results.get(0).getScore()); - System.out.println(results.get(1).getScore()); - System.out.println(results.get(2).getScore()); assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); assertEquals(expectedDistance.get(0).doubleValue(), results.get(0).getScore(), 0.0001d); assertEquals(hotels.get(0).getId(), results.get(1).getRecord().getId()); From b3dcd99504b7421872fc999b2998dc6b9978cb03 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 11:17:21 +0200 Subject: [PATCH 217/277] Change timestamp precision --- .../semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java index efde970c..1d5db527 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleDataTypesMapping.java @@ -77,7 +77,7 @@ public class OracleDataTypesMapping { /** * Oracle database type used to map offset date time */ - public static final String OFFSET_DATE_TIME = "TIMESTAMP(7) WITH TIME ZONE"; + public static final String OFFSET_DATE_TIME = "TIMESTAMP(9) WITH TIME ZONE"; /** * Oracle database type used to map UUID */ From d77dee23a426339d2425c72489fa635a9ddfdfd3 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 11:25:17 +0200 Subject: [PATCH 218/277] Changed distance precision in tests --- .../oracle/OracleVectorStoreRecordCollectionTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java index 238a5ef2..850ae923 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordCollectionTest.java @@ -216,11 +216,11 @@ public void exactSearch(DistanceFunction distanceFunction, List expected assertEquals(3, results.size()); // The third hotel should be the most similar assertEquals(hotels.get(2).getId(), results.get(0).getRecord().getId()); - assertEquals(expectedDistance.get(0).doubleValue(), results.get(0).getScore(), 0.0001d); + assertEquals(expectedDistance.get(0).doubleValue(), results.get(0).getScore(), 0.0002d); assertEquals(hotels.get(0).getId(), results.get(1).getRecord().getId()); - assertEquals(expectedDistance.get(1).doubleValue(), results.get(1).getScore(), 0.0001d); + assertEquals(expectedDistance.get(1).doubleValue(), results.get(1).getScore(), 0.0002d); assertEquals(hotels.get(3).getId(), results.get(2).getRecord().getId()); - assertEquals(expectedDistance.get(2).doubleValue(), results.get(2).getScore(), 0.0001d); + assertEquals(expectedDistance.get(2).doubleValue(), results.get(2).getScore(), 0.0002d); options = VectorSearchOptions.builder() .withVectorFieldName(distanceFunction.getValue()) @@ -258,7 +258,7 @@ public void searchWithFilter(DistanceFunction distanceFunction, double expectedD assertEquals(3, results.size()); // The first hotel should be the most similar assertEquals(hotels.get(0).getId(), results.get(0).getRecord().getId()); - assertEquals(results.get(0).getScore(), expectedDistance, 0.0001d); + assertEquals(results.get(0).getScore(), expectedDistance, 0.0002d); } From 0d51d3f8438553e57baeeae1e4554ea4da30389e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Aug 2025 10:40:31 +0000 Subject: [PATCH 219/277] Bump actions/checkout from 4 to 5 Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/java-build.yml | 2 +- .github/workflows/java-integration-tests.yml | 2 +- .github/workflows/java-publish-package.yml | 2 +- .github/workflows/markdown-link-check.yml | 2 +- .github/workflows/typos.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index caf56980..80fe95e9 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -30,7 +30,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/java-build.yml b/.github/workflows/java-build.yml index f9da894f..c03eee4b 100644 --- a/.github/workflows/java-build.yml +++ b/.github/workflows/java-build.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 # Need to use JDK 11 to build for JDK 8 - name: Set JDK diff --git a/.github/workflows/java-integration-tests.yml b/.github/workflows/java-integration-tests.yml index a65ee5cc..974ddc39 100644 --- a/.github/workflows/java-integration-tests.yml +++ b/.github/workflows/java-integration-tests.yml @@ -22,7 +22,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 # Need to use JDK 11 to build for JDK 8 - name: Set JDK diff --git a/.github/workflows/java-publish-package.yml b/.github/workflows/java-publish-package.yml index 25afec46..8b24024c 100644 --- a/.github/workflows/java-publish-package.yml +++ b/.github/workflows/java-publish-package.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 # Sets up the specified JDK version from the matrix - uses: actions/setup-java@v4 diff --git a/.github/workflows/markdown-link-check.yml b/.github/workflows/markdown-link-check.yml index c0993625..4145df27 100644 --- a/.github/workflows/markdown-link-check.yml +++ b/.github/workflows/markdown-link-check.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest # check out the latest version of the code steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 # Checks the status of hyperlinks in .md files in verbose mode - name: Check links diff --git a/.github/workflows/typos.yaml b/.github/workflows/typos.yaml index d7a937ba..6452831b 100644 --- a/.github/workflows/typos.yaml +++ b/.github/workflows/typos.yaml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Use custom config file uses: crate-ci/typos@master From 912db48aeba662c1c10b7fad46d86b856469d883 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 14:16:33 +0200 Subject: [PATCH 220/277] Lower case on MapResultSetToRecord --- .../data/jdbc/oracle/OracleVectorStoreRecordMapper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java index 3b62f07d..c75eb663 100644 --- a/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java +++ b/data/semantickernel-data-oracle/src/main/java/com/microsoft/semantickernel/data/jdbc/oracle/OracleVectorStoreRecordMapper.java @@ -156,11 +156,11 @@ public OracleVectorStoreRecordMapper build() { return new OracleVectorStoreRecordMapper<>( (resultSet, options) -> { - return MapResultSetToRecord(resultSet, options); + return mapResultSetToRecord(resultSet, options); }); } - private Record MapResultSetToRecord(ResultSet resultSet, GetRecordOptions options) { + private Record mapResultSetToRecord(ResultSet resultSet, GetRecordOptions options) { try { objectMapper.registerModule(new OsonModule()); // Create an ObjectNode to hold the values From cfde37f9e1a3b42e4116eef8286330db3deb00a2 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 15:01:08 +0200 Subject: [PATCH 221/277] Update samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml Co-authored-by: John Oliver <1615532+johnoliver@users.noreply.github.com> --- .../semantickernel-syntax-examples/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index cb735f9c..cd4a367c 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -150,7 +150,7 @@ com.microsoft.semantic-kernel semantickernel-data-postgres - 1.4.4-RC2-SNAPSHOT + ${project.version} compile From 24490cca39fb6bfba1194f6e7d92cd6164385dc4 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 15:01:55 +0200 Subject: [PATCH 222/277] Update samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml Co-authored-by: John Oliver <1615532+johnoliver@users.noreply.github.com> --- .../semantickernel-syntax-examples/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index cd4a367c..d4e80015 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -156,7 +156,7 @@ com.microsoft.semantic-kernel semantickernel-data-oracle - 1.4.4-RC2-SNAPSHOT + ${project.version} compile From b854f9b8e4bc5886799c7a02b0cfe17f3932b710 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 15:02:06 +0200 Subject: [PATCH 223/277] Update samples/semantickernel-learn-resources/pom.xml Co-authored-by: John Oliver <1615532+johnoliver@users.noreply.github.com> --- samples/semantickernel-learn-resources/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 8d3cef89..d0903e4c 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -97,7 +97,7 @@ com.microsoft.semantic-kernel semantickernel-data-postgres - 1.4.4-RC2-SNAPSHOT + ${project.version} compile From ba45bb34c31488663da2de3f5d2e5ba968777e45 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 15:02:41 +0200 Subject: [PATCH 224/277] Update samples/semantickernel-learn-resources/pom.xml Co-authored-by: John Oliver <1615532+johnoliver@users.noreply.github.com> --- samples/semantickernel-learn-resources/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index d0903e4c..46cc092e 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -49,7 +49,7 @@ com.microsoft.semantic-kernel semantickernel-data-postgres - 1.4.4-RC2-SNAPSHOT + ${project.version} org.apache.logging.log4j From 735587b1a596771f500169fa4cc1bb7e7cd67cad Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 15:03:02 +0200 Subject: [PATCH 225/277] Update samples/semantickernel-learn-resources/pom.xml Co-authored-by: John Oliver <1615532+johnoliver@users.noreply.github.com> --- samples/semantickernel-learn-resources/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 46cc092e..73cdac1d 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -44,7 +44,7 @@ com.microsoft.semantic-kernel semantickernel-data-oracle - 1.4.4-RC2-SNAPSHOT + ${project.version} com.microsoft.semantic-kernel From f0c9615c5fddab9d308ee396bf5e7d14c9c18f7b Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 15:03:42 +0200 Subject: [PATCH 226/277] Update samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml Co-authored-by: John Oliver <1615532+johnoliver@users.noreply.github.com> --- .../semantickernel-syntax-examples/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index d4e80015..177a2712 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -144,7 +144,7 @@ com.microsoft.semantic-kernel semantickernel-learn-resources - 1.4.4-RC2-SNAPSHOT + ${project.version} compile From 4d61f13e27114dc4f632d89e03471ea544ec0869 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 15:53:37 +0200 Subject: [PATCH 227/277] Fix: Run Java Integration Tests and Samples --- api-test/integration-tests/pom.xml | 15 +++++++++++++++ semantickernel-bom/pom.xml | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 433e14a8..94e577f1 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -61,6 +61,21 @@ semantickernel-data-jdbc test + + com.microsoft.semantic-kernel + semantickernel-data-mysql + test + + + com.microsoft.semantic-kernel + semantickernel-data-hsqldb + test + + + com.microsoft.semantic-kernel + semantickernel-data-sqlite + test + com.microsoft.semantic-kernel semantickernel-data-redis diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index ad6d7180..9f56c952 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -106,6 +106,21 @@ semantickernel-data-jdbc ${project.version} + + com.microsoft.semantic-kernel + semantickernel-data-mysql + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-data-sqlite + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-data-hsqldb + ${project.version} + com.microsoft.semantic-kernel semantickernel-data-redis From f78b2ab12cdf07a8bc62f6bf3967cd4ae6f024b2 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 16:47:20 +0200 Subject: [PATCH 228/277] Added oracle store to BOM * integration tests * Added oracle module to BOM --- semantickernel-bom/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 9f56c952..940093a0 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -121,6 +121,11 @@ semantickernel-data-hsqldb ${project.version} + + com.microsoft.semantic-kernel + semantickernel-data-oracle + ${project.version} + com.microsoft.semantic-kernel semantickernel-data-redis From ced785b77309a297b2755dc17bed34ef06521fb7 Mon Sep 17 00:00:00 2001 From: Fernanda Meheust Date: Mon, 18 Aug 2025 17:07:28 +0200 Subject: [PATCH 229/277] Disable spotless plugin (#15) * integration tests * Added oracle module to BOM * Disable spotless for oracle store and result of spotless apply --- .../jdbc/JDBCVectorStoreQueryProvider.java | 16 ++++++++-------- .../data/jdbc/SQLVectorStoreQueryProvider.java | 4 ++-- data/semantickernel-data-oracle/pom.xml | 7 +++++++ .../PostgreSQLVectorStoreQueryProvider.java | 15 ++++++++------- .../memory/VectorStoreWithOracle.java | 18 +++++++++--------- .../data/vectorstores/oracle/Book.java | 5 +++-- .../VectorStoreRecordDefinition.java | 12 ++++++++++-- 7 files changed, 47 insertions(+), 30 deletions(-) diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java index f19d0c22..2fcc2d5b 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -696,18 +696,19 @@ public String getEqualToFilter(EqualToFilterClause filterClause) { @Override public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { String fieldName = JDBCVectorStoreQueryProvider - .validateSQLidentifier(filterClause.getFieldName()); + .validateSQLidentifier(filterClause.getFieldName()); return String.format("%s LIKE ?", fieldName); } - + @Override - public VectorStoreRecordMapper getVectorStoreRecordMapper(Class recordClass, - VectorStoreRecordDefinition recordDefinition) { + public VectorStoreRecordMapper getVectorStoreRecordMapper( + Class recordClass, + VectorStoreRecordDefinition recordDefinition) { return JDBCVectorStoreRecordMapper.builder() - .withRecordClass(recordClass) - .withVectorStoreRecordDefinition(recordDefinition) - .build(); + .withRecordClass(recordClass) + .withVectorStoreRecordDefinition(recordDefinition) + .build(); } /** @@ -765,5 +766,4 @@ public JDBCVectorStoreQueryProvider build() { } } - } diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java index cadfdaf2..30a535fb 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/SQLVectorStoreQueryProvider.java @@ -160,8 +160,8 @@ VectorSearchResults search(String collectionName, * @return the record mapper that maps JDBC result sets to the given record. */ VectorStoreRecordMapper getVectorStoreRecordMapper( - final Class recordClass, - final VectorStoreRecordDefinition recordDefinition); + final Class recordClass, + final VectorStoreRecordDefinition recordDefinition); /** * The builder for the JDBC vector store query provider. diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml index 6fd20ba1..6514ff0b 100644 --- a/data/semantickernel-data-oracle/pom.xml +++ b/data/semantickernel-data-oracle/pom.xml @@ -97,6 +97,13 @@ true + + com.diffplug.spotless + spotless-maven-plugin + + true + + \ No newline at end of file diff --git a/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java b/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java index bd8dbba7..1f3273eb 100644 --- a/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -442,18 +442,19 @@ public List getFilterParameters(VectorSearchFilter filter) { @Override public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { String fieldName = JDBCVectorStoreQueryProvider - .validateSQLidentifier(filterClause.getFieldName()); + .validateSQLidentifier(filterClause.getFieldName()); return String.format("%s @> ?::jsonb", fieldName); } - + @Override - public VectorStoreRecordMapper getVectorStoreRecordMapper(Class recordClass, - VectorStoreRecordDefinition recordDefinition) { + public VectorStoreRecordMapper getVectorStoreRecordMapper( + Class recordClass, + VectorStoreRecordDefinition recordDefinition) { return PostgreSQLVectorStoreRecordMapper.builder() - .withRecordClass(recordClass) - .withVectorStoreRecordDefinition(recordDefinition) - .build(); + .withRecordClass(recordClass) + .withVectorStoreRecordDefinition(recordDefinition) + .build(); } /** diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java index 3d037c70..e3213105 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/VectorStoreWithOracle.java @@ -39,20 +39,20 @@ public static void main(String[] args) throws SQLException { .build(); // Get a collection from the vector store - VectorStoreRecordCollection collection = - vectorStore.getCollection("skhotels", - JDBCVectorStoreRecordCollectionOptions.builder() - .withRecordClass(Hotel.class) - .build()); + VectorStoreRecordCollection collection = vectorStore.getCollection( + "skhotels", + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .build()); // Create the collection if it doesn't exist yet. collection.createCollectionAsync().block(); collection.upsertAsync(new Hotel("1", - "HotelOne", - "Desc for HotelOne", - Collections.emptyList(), Collections.emptyList()), - null) + "HotelOne", + "Desc for HotelOne", + Collections.emptyList(), Collections.emptyList()), + null) .block(); } diff --git a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java index 8de0b5aa..77b034bf 100644 --- a/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java +++ b/samples/semantickernel-learn-resources/src/main/java/com/microsoft/semantickernel/samples/documentationexamples/data/vectorstores/oracle/Book.java @@ -32,7 +32,8 @@ public class Book { - public Book() {} + public Book() { + } public Book(String isbn, String title, String author, int pages, List tags, String summary, List summaryEmbedding) { @@ -60,7 +61,7 @@ public Book(String isbn, String title, String author, int pages, @VectorStoreRecordData(isFilterable = true) private List tags; - @VectorStoreRecordData( isFilterable = true, isFullTextSearchable = true ) + @VectorStoreRecordData(isFilterable = true, isFullTextSearchable = true) private String summary; @VectorStoreRecordVector(dimensions = 2) diff --git a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java index e769bb6f..abb06111 100644 --- a/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java +++ b/semantickernel-api-data/src/main/java/com/microsoft/semantickernel/data/vectorstorage/definition/VectorStoreRecordDefinition.java @@ -194,7 +194,11 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) dataFields.add(VectorStoreRecordDataField.builder() .withName(field.getName()) .withStorageName(storageName) - .withFieldType(field.getType(), List.class.equals(field.getType()) ? (Class)((ParameterizedType) field.getGenericType()).getActualTypeArguments()[0] : null) + .withFieldType(field.getType(), + List.class.equals(field.getType()) + ? (Class) ((ParameterizedType) field.getGenericType()) + .getActualTypeArguments()[0] + : null) .isFilterable(dataAttribute.isFilterable()) .build()); } @@ -210,7 +214,11 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) vectorFields.add(VectorStoreRecordVectorField.builder() .withName(field.getName()) .withStorageName(storageName) - .withFieldType(field.getType(), List.class.equals(field.getType()) ? (Class)((ParameterizedType) field.getGenericType()).getActualTypeArguments()[0] : null) + .withFieldType(field.getType(), + List.class.equals(field.getType()) + ? (Class) ((ParameterizedType) field.getGenericType()) + .getActualTypeArguments()[0] + : null) .withDimensions(vectorAttribute.dimensions()) .withIndexKind(vectorAttribute.indexKind()) .withDistanceFunction(vectorAttribute.distanceFunction()) From d31f911c1e2e4b59c61298f45b7f619444e5de34 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Aug 2025 12:47:42 +0000 Subject: [PATCH 230/277] Bump actions/setup-java from 4 to 5 Bumps [actions/setup-java](https://github.com/actions/setup-java) from 4 to 5. - [Release notes](https://github.com/actions/setup-java/releases) - [Commits](https://github.com/actions/setup-java/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-java dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/java-build.yml | 2 +- .github/workflows/java-integration-tests.yml | 2 +- .github/workflows/java-publish-package.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 80fe95e9..ba586a25 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -51,7 +51,7 @@ jobs: uses: github/codeql-action/autobuild@v2 - name: Setup JDK - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 if: ${{ matrix.language == 'java' }} with: java-version: 17 diff --git a/.github/workflows/java-build.yml b/.github/workflows/java-build.yml index c03eee4b..73edf46d 100644 --- a/.github/workflows/java-build.yml +++ b/.github/workflows/java-build.yml @@ -41,7 +41,7 @@ jobs: fi # Sets up the specified JDK version from the matrix - - uses: actions/setup-java@v4 + - uses: actions/setup-java@v5 with: java-version: ${{ steps.set-jdk.outputs.JDK_VERSION }} distribution: microsoft diff --git a/.github/workflows/java-integration-tests.yml b/.github/workflows/java-integration-tests.yml index 974ddc39..f8d25c33 100644 --- a/.github/workflows/java-integration-tests.yml +++ b/.github/workflows/java-integration-tests.yml @@ -36,7 +36,7 @@ jobs: echo "JDK_VERSION=$version" >> $GITHUB_OUTPUT fi - - uses: actions/setup-java@v4 + - uses: actions/setup-java@v5 with: java-version: ${{ steps.set-jdk.outputs.JDK_VERSION }} distribution: microsoft diff --git a/.github/workflows/java-publish-package.yml b/.github/workflows/java-publish-package.yml index 8b24024c..6da0a6c5 100644 --- a/.github/workflows/java-publish-package.yml +++ b/.github/workflows/java-publish-package.yml @@ -20,7 +20,7 @@ jobs: uses: actions/checkout@v5 # Sets up the specified JDK version from the matrix - - uses: actions/setup-java@v4 + - uses: actions/setup-java@v5 with: java-version: 11 distribution: microsoft From 9c69a2723fb88ad6d2094833a38b5bb0c313a03e Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Wed, 27 Aug 2025 15:49:34 +0100 Subject: [PATCH 231/277] Fix packageing to include utils in the API --- agents/semantickernel-agents-core/pom.xml | 15 +++++- aiservices/google/pom.xml | 16 +++++++ aiservices/huggingface/pom.xml | 8 ++++ aiservices/openai/pom.xml | 24 ++++++++++ .../semantickernel-data-azureaisearch/pom.xml | 8 ++++ data/semantickernel-data-hsqldb/pom.xml | 23 +++++++-- data/semantickernel-data-jdbc/pom.xml | 11 +++++ data/semantickernel-data-mysql/pom.xml | 23 +++++++-- data/semantickernel-data-oracle/pom.xml | 15 ++++++ data/semantickernel-data-postgres/pom.xml | 13 +++++ data/semantickernel-data-redis/pom.xml | 12 +++++ data/semantickernel-data-sqlite/pom.xml | 13 +++++ pom.xml | 13 +++++ .../booking-agent-m365/pom.xml | 4 ++ .../pom.xml | 4 ++ semantickernel-api-ai-services/pom.xml | 39 +++++++++++++++ .../semantickernel/services/AIService.java | 0 semantickernel-api-builders/pom.xml | 6 +-- semantickernel-api-data/pom.xml | 9 +++- semantickernel-api-exceptions/pom.xml | 1 + .../pom.xml | 6 ++- semantickernel-api/pom.xml | 48 ++++++++++++++++++- semantickernel-api/src/assembly/custom.xml | 22 +++++++++ semantickernel-bom/pom.xml | 8 +++- semantickernel-experimental/pom.xml | 5 ++ 25 files changed, 326 insertions(+), 20 deletions(-) create mode 100644 semantickernel-api-ai-services/pom.xml rename {semantickernel-api-textembedding-services => semantickernel-api-ai-services}/src/main/java/com/microsoft/semantickernel/services/AIService.java (100%) create mode 100644 semantickernel-api/src/assembly/custom.xml diff --git a/agents/semantickernel-agents-core/pom.xml b/agents/semantickernel-agents-core/pom.xml index 9d40d75d..cafee140 100644 --- a/agents/semantickernel-agents-core/pom.xml +++ b/agents/semantickernel-agents-core/pom.xml @@ -1,5 +1,6 @@ - + 4.0.0 com.microsoft.semantic-kernel @@ -18,6 +19,18 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + \ No newline at end of file diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index a205a60b..390c3bc1 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -34,6 +34,22 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-localization + com.fasterxml.jackson.core diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index aaed6aad..4ce69b95 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -31,6 +31,14 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + com.azure azure-core diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index 4a93b679..04b31ca3 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -19,6 +19,30 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-data + provided + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + provided + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + provided + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + com.microsoft.semantic-kernel + semantickernel-api-textembedding-services + + com.azure azure-ai-openai diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml index 74dc84bc..038cc038 100644 --- a/data/semantickernel-data-azureaisearch/pom.xml +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -21,6 +21,14 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + com.azure diff --git a/data/semantickernel-data-hsqldb/pom.xml b/data/semantickernel-data-hsqldb/pom.xml index f755f0f1..42245185 100644 --- a/data/semantickernel-data-hsqldb/pom.xml +++ b/data/semantickernel-data-hsqldb/pom.xml @@ -4,10 +4,10 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - com.microsoft.semantic-kernel - semantickernel-parent - 1.4.4-RC2-SNAPSHOT - ../../pom.xml + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../../pom.xml com.microsoft.semantic-kernel @@ -15,7 +15,7 @@ Semantic Kernel HLSQLDB connector Provides a HLSQLDB connector for the Semantic Kernel - + com.microsoft.semantic-kernel semantickernel-api @@ -24,6 +24,19 @@ com.microsoft.semantic-kernel semantickernel-data-jdbc + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + com.fasterxml.jackson.core jackson-databind diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 592cbbe5..f1e62c6a 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -16,6 +16,17 @@ com.microsoft.semantic-kernel semantickernel-api-data + provided + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + provided + + + com.microsoft.semantic-kernel + semantickernel-api-builders + provided org.slf4j diff --git a/data/semantickernel-data-mysql/pom.xml b/data/semantickernel-data-mysql/pom.xml index aefd1cd9..899f1f2a 100644 --- a/data/semantickernel-data-mysql/pom.xml +++ b/data/semantickernel-data-mysql/pom.xml @@ -4,10 +4,10 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - com.microsoft.semantic-kernel - semantickernel-parent - 1.4.4-RC2-SNAPSHOT - ../../pom.xml + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../../pom.xml com.microsoft.semantic-kernel @@ -15,7 +15,7 @@ Semantic Kernel MySQL connector Provides a MySQL connector for the Semantic Kernel - + com.microsoft.semantic-kernel semantickernel-api @@ -24,6 +24,19 @@ com.microsoft.semantic-kernel semantickernel-data-jdbc + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + com.fasterxml.jackson.core jackson-databind diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml index 6514ff0b..aa953870 100644 --- a/data/semantickernel-data-oracle/pom.xml +++ b/data/semantickernel-data-oracle/pom.xml @@ -29,6 +29,21 @@ + + com.microsoft.semantic-kernel + semantickernel-api-data + provided + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + provided + + + com.microsoft.semantic-kernel + semantickernel-api-builders + provided + com.microsoft.semantic-kernel semantickernel-data-jdbc diff --git a/data/semantickernel-data-postgres/pom.xml b/data/semantickernel-data-postgres/pom.xml index 73591658..2d01ed81 100644 --- a/data/semantickernel-data-postgres/pom.xml +++ b/data/semantickernel-data-postgres/pom.xml @@ -23,6 +23,19 @@ com.microsoft.semantic-kernel semantickernel-data-jdbc + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + com.fasterxml.jackson.core jackson-databind diff --git a/data/semantickernel-data-redis/pom.xml b/data/semantickernel-data-redis/pom.xml index a8e582eb..4af2dc6a 100644 --- a/data/semantickernel-data-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -17,6 +17,18 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-builders + org.slf4j diff --git a/data/semantickernel-data-sqlite/pom.xml b/data/semantickernel-data-sqlite/pom.xml index c8a747a9..64b4a6db 100644 --- a/data/semantickernel-data-sqlite/pom.xml +++ b/data/semantickernel-data-sqlite/pom.xml @@ -24,6 +24,19 @@ com.microsoft.semantic-kernel semantickernel-data-jdbc + + com.microsoft.semantic-kernel + semantickernel-api-data + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + com.fasterxml.jackson.core jackson-databind diff --git a/pom.xml b/pom.xml index b52f7484..c50eda63 100644 --- a/pom.xml +++ b/pom.xml @@ -88,6 +88,7 @@ semantickernel-api-builders semantickernel-api-textembedding-services semantickernel-api-localization + semantickernel-api-ai-services @@ -149,26 +150,31 @@ com.microsoft.semantic-kernel semantickernel-api-builders ${project.version} + provided com.microsoft.semantic-kernel semantickernel-api-data ${project.version} + provided com.microsoft.semantic-kernel semantickernel-api-exceptions ${project.version} + provided com.microsoft.semantic-kernel semantickernel-api-localization ${project.version} + provided com.microsoft.semantic-kernel semantickernel-api-textembedding-services ${project.version} + provided com.microsoft.semantic-kernel.extensions @@ -180,6 +186,13 @@ semantickernel-actionplanner-extension ${project.version} + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + ${project.version} + provided + + com.github.spotbugs diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index f6f8d2a8..d9fd6eee 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -43,6 +43,10 @@ com.microsoft.semantic-kernel semantickernel-aiservices-openai + + com.microsoft.semantic-kernel + semantickernel-api-builders + com.azure diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml index 10fb9a30..be88deb6 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml @@ -29,6 +29,10 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + org.apache.logging.log4j diff --git a/semantickernel-api-ai-services/pom.xml b/semantickernel-api-ai-services/pom.xml new file mode 100644 index 00000000..9b4d76f6 --- /dev/null +++ b/semantickernel-api-ai-services/pom.xml @@ -0,0 +1,39 @@ + + + 4.0.0 + + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT + ../pom.xml + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + Semantic Kernel AI Services API + Defines the public interface for the Semantic Kernel Services + + + + com.google.code.findbugs + jsr305 + provided + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + 1 + + + + + + \ No newline at end of file diff --git a/semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/AIService.java b/semantickernel-api-ai-services/src/main/java/com/microsoft/semantickernel/services/AIService.java similarity index 100% rename from semantickernel-api-textembedding-services/src/main/java/com/microsoft/semantickernel/services/AIService.java rename to semantickernel-api-ai-services/src/main/java/com/microsoft/semantickernel/services/AIService.java diff --git a/semantickernel-api-builders/pom.xml b/semantickernel-api-builders/pom.xml index f48a2297..5bc0e11b 100644 --- a/semantickernel-api-builders/pom.xml +++ b/semantickernel-api-builders/pom.xml @@ -4,9 +4,9 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - com.microsoft.semantic-kernel - semantickernel-parent - 1.4.4-RC2-SNAPSHOT + com.microsoft.semantic-kernel + semantickernel-parent + 1.4.4-RC2-SNAPSHOT com.microsoft.semantic-kernel diff --git a/semantickernel-api-data/pom.xml b/semantickernel-api-data/pom.xml index 895e70e1..822b7c57 100644 --- a/semantickernel-api-data/pom.xml +++ b/semantickernel-api-data/pom.xml @@ -19,14 +19,22 @@ com.microsoft.semantic-kernel semantickernel-api-exceptions + provided com.microsoft.semantic-kernel semantickernel-api-builders + provided com.microsoft.semantic-kernel semantickernel-api-textembedding-services + provided + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + provided com.fasterxml.jackson.core @@ -41,7 +49,6 @@ io.projectreactor reactor-core - 3.4.38 diff --git a/semantickernel-api-exceptions/pom.xml b/semantickernel-api-exceptions/pom.xml index 71d1794b..b459db13 100644 --- a/semantickernel-api-exceptions/pom.xml +++ b/semantickernel-api-exceptions/pom.xml @@ -19,6 +19,7 @@ com.microsoft.semantic-kernel semantickernel-api-localization + provided com.google.code.findbugs diff --git a/semantickernel-api-textembedding-services/pom.xml b/semantickernel-api-textembedding-services/pom.xml index dbc5fa6a..48a524ed 100644 --- a/semantickernel-api-textembedding-services/pom.xml +++ b/semantickernel-api-textembedding-services/pom.xml @@ -19,7 +19,6 @@ io.projectreactor reactor-core - 3.4.38 com.google.code.findbugs @@ -29,7 +28,10 @@ com.github.spotbugs spotbugs-annotations - ${spotbugs.version} + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index d34e5a17..0911e705 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -1,6 +1,7 @@ - + 4.0.0 @@ -15,7 +16,7 @@ Semantic Kernel API Defines the public interface for the Semantic Kernel - + com.microsoft.semantic-kernel semantickernel-api-data @@ -35,6 +36,10 @@ com.microsoft.semantic-kernel semantickernel-api-textembedding-services + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + io.opentelemetry.instrumentation opentelemetry-reactor-3.1 @@ -123,6 +128,45 @@ 1 + + org.apache.maven.plugins + maven-dependency-plugin + + + unpack-dependencies + process-resources + + unpack-dependencies + + + + semantickernel-api-exceptions,semantickernel-api-builders,semantickernel-api-localization,semantickernel-api-textembedding-services,semantickernel-api-ai-services,semantickernel-api-data + + ${project.build.directory}/lib + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.6.0 + + + src/assembly/custom.xml + + false + + + + make-assembly + package + + single + + + + diff --git a/semantickernel-api/src/assembly/custom.xml b/semantickernel-api/src/assembly/custom.xml new file mode 100644 index 00000000..48becfff --- /dev/null +++ b/semantickernel-api/src/assembly/custom.xml @@ -0,0 +1,22 @@ + + with-exceptions + + jar + + false + + + ${project.build.outputDirectory} + / + + + ${project.build.directory}/lib + / + + **/** + + + + \ No newline at end of file diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 940093a0..990eb433 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -1,6 +1,7 @@ - + 4.0.0 com.microsoft.semantic-kernel @@ -252,6 +253,11 @@ 4.36.0 true + + io.projectreactor + reactor-core + 3.7.8 + diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 98ce0918..538c9498 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -88,6 +88,11 @@ com.microsoft.semantic-kernel semantickernel-api + + com.microsoft.semantic-kernel + semantickernel-api-builders + provided + From 6e5ed4663ade5946fcdf2a641cba113f4307b2a2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Aug 2025 22:16:24 +0000 Subject: [PATCH 232/277] Bump org.postgresql:postgresql in /data/semantickernel-data-postgres Bumps [org.postgresql:postgresql](https://github.com/pgjdbc/pgjdbc) from 42.7.4 to 42.7.7. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.7.4...REL42.7.7) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-version: 42.7.7 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- data/semantickernel-data-postgres/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/semantickernel-data-postgres/pom.xml b/data/semantickernel-data-postgres/pom.xml index 2d01ed81..56e65685 100644 --- a/data/semantickernel-data-postgres/pom.xml +++ b/data/semantickernel-data-postgres/pom.xml @@ -53,7 +53,7 @@ org.postgresql postgresql - 42.7.4 + 42.7.7 \ No newline at end of file From faa90f71c08d899cf52f47ca4c8af315acd665f4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Sep 2025 07:32:30 +0000 Subject: [PATCH 233/277] Bump actions/stale from 5 to 10 Bumps [actions/stale](https://github.com/actions/stale) from 5 to 10. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v5...v10) --- updated-dependencies: - dependency-name: actions/stale dependency-version: '10' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/close-inactive-issues.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/close-inactive-issues.yml b/.github/workflows/close-inactive-issues.yml index 10244c76..4e6ebcce 100644 --- a/.github/workflows/close-inactive-issues.yml +++ b/.github/workflows/close-inactive-issues.yml @@ -10,7 +10,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@v5 + - uses: actions/stale@v10 with: days-before-issue-stale: 90 days-before-issue-close: 14 From e79d6b59a37cc951c8c522b3f00ef3c333243485 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Sep 2025 07:32:42 +0000 Subject: [PATCH 234/277] Bump actions/github-script from 7 to 8 Bumps [actions/github-script](https://github.com/actions/github-script) from 7 to 8. - [Release notes](https://github.com/actions/github-script/releases) - [Commits](https://github.com/actions/github-script/compare/v7...v8) --- updated-dependencies: - dependency-name: actions/github-script dependency-version: '8' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/label-issues.yml | 2 +- .github/workflows/label-title-prefix.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/label-issues.yml b/.github/workflows/label-issues.yml index 42806188..cf901ade 100644 --- a/.github/workflows/label-issues.yml +++ b/.github/workflows/label-issues.yml @@ -13,7 +13,7 @@ jobs: permissions: issues: write steps: - - uses: actions/github-script@v7 + - uses: actions/github-script@v8 with: github-token: ${{ secrets.GH_ACTIONS_PR_WRITE }} script: | diff --git a/.github/workflows/label-title-prefix.yml b/.github/workflows/label-title-prefix.yml index f443c980..81369dee 100644 --- a/.github/workflows/label-title-prefix.yml +++ b/.github/workflows/label-title-prefix.yml @@ -15,7 +15,7 @@ jobs: pull-requests: write steps: - - uses: actions/github-script@v7 + - uses: actions/github-script@v8 name: "Issue/PR: update title" with: github-token: ${{ secrets.GITHUB_TOKEN }} From 6fa980292e470be8c29108438aed3f38698e4959 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Tue, 16 Sep 2025 14:48:07 +0100 Subject: [PATCH 235/277] Fix build dependencies --- api-test/pom.xml | 26 ++++++++++- pom.xml | 45 ++++++++++++++++++- .../semantickernel-syntax-examples/pom.xml | 27 +++++++++++ .../booking-agent-m365/pom.xml | 13 ------ samples/semantickernel-demos/pom.xml | 23 ++++++++++ .../semantickernel-spring-starter/pom.xml | 8 ---- .../sk-presidio-sample/pom.xml | 8 ---- .../semantickernel-learn-resources/pom.xml | 21 +++++++-- samples/semantickernel-sample-plugins/pom.xml | 25 ++++++++++- semantickernel-api/pom.xml | 1 - semantickernel-bom/pom.xml | 7 ++- 11 files changed, 167 insertions(+), 37 deletions(-) diff --git a/api-test/pom.xml b/api-test/pom.xml index c347e60a..6ad2288d 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -1,6 +1,7 @@ - + 4.0.0 @@ -34,6 +35,28 @@ + + + com.microsoft.semantic-kernel + semantickernel-api + + + com.microsoft.semantic-kernel + semantickernel-aiservices-openai + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + @@ -57,6 +80,7 @@ com.diffplug.spotless spotless-maven-plugin + ${maven.spotless-plugin.version} true diff --git a/pom.xml b/pom.xml index c50eda63..379aaa47 100644 --- a/pom.xml +++ b/pom.xml @@ -43,7 +43,7 @@ 3.5.0 2.4.0 - 3.21.2 + 3.27.0 3.8.0 0.16.1 3.1.1 @@ -212,6 +212,44 @@ 5.14.2 test + + + + com.microsoft.semantic-kernel + semantickernel-api-builders + provided + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + provided + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-api-localization + provided + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-api-textembedding-services + provided + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + provided + ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-api-data + provided + ${project.version} + @@ -348,6 +386,11 @@ pmd-jsp ${pmd.version} + + org.apache.maven.plugins + maven-pmd-plugin + 3.27.0 + diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 177a2712..81e477c9 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -110,6 +110,33 @@ semantickernel-text-splitter-plugin ${project.version} + + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + com.microsoft.semantic-kernel + semantickernel-api-localization + + + com.microsoft.semantic-kernel + semantickernel-api-textembedding-services + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + + + com.microsoft.semantic-kernel + semantickernel-api-data + + org.apache.pdfbox pdfbox diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index d9fd6eee..3f5a9a3f 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -35,19 +35,6 @@ runtime - - com.microsoft.semantic-kernel - semantickernel-api - - - com.microsoft.semantic-kernel - semantickernel-aiservices-openai - - - com.microsoft.semantic-kernel - semantickernel-api-builders - - com.azure azure-identity diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index 3ad7a9c2..af125132 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -12,6 +12,29 @@ pom semantic-kernel-demos + + + com.microsoft.semantic-kernel + semantickernel-api + + + com.microsoft.semantic-kernel + semantickernel-aiservices-openai + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + + booking-agent-m365 semantickernel-spring-starter diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index 0029654f..18ee76ae 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -23,14 +23,6 @@ - - com.microsoft.semantic-kernel - semantickernel-api - - - com.microsoft.semantic-kernel - semantickernel-aiservices-openai - org.slf4j diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index e23396db..4427cf20 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -23,14 +23,6 @@ - - com.microsoft.semantic-kernel - semantickernel-api - - - com.microsoft.semantic-kernel - semantickernel-aiservices-openai - com.microsoft.semantic-kernel semantickernel-presidio-plugin diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 73cdac1d..d03e3727 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -44,12 +44,14 @@ com.microsoft.semantic-kernel semantickernel-data-oracle - ${project.version} com.microsoft.semantic-kernel semantickernel-data-postgres - ${project.version} + + + com.microsoft.semantic-kernel + semantickernel-api-data org.apache.logging.log4j @@ -88,6 +90,10 @@ com.microsoft.semantic-kernel semantickernel-experimental + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + com.mysql mysql-connector-j @@ -97,9 +103,18 @@ com.microsoft.semantic-kernel semantickernel-data-postgres - ${project.version} compile + + com.microsoft.semantic-kernel + semantickernel-api-data + compile + + + com.microsoft.semantic-kernel + semantickernel-api-builders + provided + diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index 4aaaf412..ee30ae29 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -1,5 +1,6 @@ - + 4.0.0 com.microsoft.semantic-kernel @@ -12,6 +13,28 @@ pom semantickernel-sample-plugins + + + com.microsoft.semantic-kernel + semantickernel-api + + + com.microsoft.semantic-kernel + semantickernel-aiservices-openai + + + com.microsoft.semantic-kernel + semantickernel-api-ai-services + + + com.microsoft.semantic-kernel + semantickernel-api-builders + + + com.microsoft.semantic-kernel + semantickernel-api-exceptions + + semantickernel-openapi-plugin semantickernel-presidio-plugin diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index 0911e705..2463ec77 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -134,7 +134,6 @@ unpack-dependencies - process-resources unpack-dependencies diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 990eb433..6b343840 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -132,12 +132,16 @@ semantickernel-data-redis ${project.version} - com.microsoft.semantic-kernel semantickernel-agents-core ${project.version} + + com.microsoft.semantic-kernel + semantickernel-data-postgres + ${project.version} + com.azure @@ -258,6 +262,7 @@ reactor-core 3.7.8 + From 39ff6b4aa097f1113ef831d3f7c75134f61a490b Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Tue, 16 Sep 2025 14:55:18 +0100 Subject: [PATCH 236/277] fix from review --- samples/semantickernel-learn-resources/pom.xml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index d03e3727..4b41f13a 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -45,14 +45,6 @@ com.microsoft.semantic-kernel semantickernel-data-oracle - - com.microsoft.semantic-kernel - semantickernel-data-postgres - - - com.microsoft.semantic-kernel - semantickernel-api-data - org.apache.logging.log4j log4j-api From 5601fa5ab4a4c9a0ea09edfb175034d84a21838a Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Tue, 16 Sep 2025 15:03:14 +0100 Subject: [PATCH 237/277] Remove dead links --- COMMUNITY.md | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/COMMUNITY.md b/COMMUNITY.md index be98d425..7afdb02c 100644 --- a/COMMUNITY.md +++ b/COMMUNITY.md @@ -7,22 +7,6 @@ Below are some ways that you can get involved in the SK Community. File issues, submit PRs, and provide feedback and ideas to what you'd like to see from the Semantic Kernel. We do our best to respond to each submission. -## Public Community Office Hours - -We regularly have Community Office Hours that are open to the **public** to join. - -Add Semantic Kernel events to your calendar - we're running two community calls to cater different timezones for Q&A Office Hours: -* Americas timezone: download the [calendar.ics](https://aka.ms/sk-community-calendar) file. -* Asia Pacific timezone: download the [calendar-APAC.ics](https://aka.ms/sk-community-calendar-apac) file. - -Add Semantic Kernel Development Office Hours for Python and Java to your calendar to help with development: -* Java Development Office Hours: [Java Development Office Hours](https://aka.ms/sk-java-dev-sync) -* Python Development Office Hours: [Python Development Office Hours](https://aka.ms/sk-python-dev-sync) - -If you have any questions or if you would like to showcase your project(s), please email what you'd like us to cover here: skofficehours[at]microsoft.com. - -If you are unable to make it live, all meetings will be recorded and posted online. - ## Join the conversation on Discord We have a growing and active channel on Discord where you can get help, engage in lively discussion, From 2d9ec903d5a56927099709af1cc56d3428011120 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Tue, 16 Sep 2025 15:13:26 +0100 Subject: [PATCH 238/277] Update pom.xml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 379aaa47..0ac1986c 100644 --- a/pom.xml +++ b/pom.xml @@ -389,7 +389,7 @@ org.apache.maven.plugins maven-pmd-plugin - 3.27.0 + ${maven.pmd-plugin.version} From 110e7f52927e19fac3f1be2cadf2ae4b3356e948 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Tue, 16 Sep 2025 17:01:12 +0100 Subject: [PATCH 239/277] Update dependencies for CG --- pom.xml | 2 +- semantickernel-bom/pom.xml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 0ac1986c..6319235d 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ Parent pom for the Semantic Kernel Project - 1.0.0-beta.14 + 1.0.0-beta.16 10.18.2 0.10.21 false diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 6b343840..f88dd038 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -146,7 +146,7 @@ com.azure azure-ai-openai - 1.0.0-beta.14 + 1.0.0-beta.16 com.azure @@ -236,7 +236,7 @@ org.apache.commons commons-text - 1.12.0 + 1.14.0 From ff7202dc90db7e38f6c2a19533ef5b9440a2617c Mon Sep 17 00:00:00 2001 From: GitHub Date: Thu, 18 Sep 2025 11:54:06 +0100 Subject: [PATCH 240/277] [maven-release-plugin] prepare release java-1.4.4-RC2 --- agents/semantickernel-agents-core/pom.xml | 5 ++--- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 5 ++--- data/semantickernel-data-azureaisearch/pom.xml | 2 +- data/semantickernel-data-hsqldb/pom.xml | 6 ++---- data/semantickernel-data-jdbc/pom.xml | 2 +- data/semantickernel-data-mysql/pom.xml | 6 ++---- data/semantickernel-data-oracle/pom.xml | 2 +- data/semantickernel-data-postgres/pom.xml | 6 ++---- data/semantickernel-data-redis/pom.xml | 2 +- data/semantickernel-data-sqlite/pom.xml | 6 ++---- pom.xml | 4 ++-- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 5 ++--- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- .../semantickernel-text-splitter-plugin/pom.xml | 2 +- semantickernel-api-ai-services/pom.xml | 6 ++---- semantickernel-api-builders/pom.xml | 6 ++---- semantickernel-api-data/pom.xml | 6 ++---- semantickernel-api-exceptions/pom.xml | 6 ++---- semantickernel-api-localization/pom.xml | 6 ++---- semantickernel-api-textembedding-services/pom.xml | 6 ++---- semantickernel-api/pom.xml | 5 ++--- semantickernel-bom/pom.xml | 7 +++---- semantickernel-experimental/pom.xml | 2 +- 36 files changed, 53 insertions(+), 78 deletions(-) diff --git a/agents/semantickernel-agents-core/pom.xml b/agents/semantickernel-agents-core/pom.xml index cafee140..ef6cdf0d 100644 --- a/agents/semantickernel-agents-core/pom.xml +++ b/agents/semantickernel-agents-core/pom.xml @@ -1,11 +1,10 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../../pom.xml diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index 390c3bc1..9d6025be 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index 4ce69b95..07031ddd 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index 04b31ca3..f8597091 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 94e577f1..c6878c36 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index 6ad2288d..a56a45b9 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -1,13 +1,12 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../pom.xml diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml index 038cc038..d12b152d 100644 --- a/data/semantickernel-data-azureaisearch/pom.xml +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../../pom.xml diff --git a/data/semantickernel-data-hsqldb/pom.xml b/data/semantickernel-data-hsqldb/pom.xml index 42245185..46a2c1ad 100644 --- a/data/semantickernel-data-hsqldb/pom.xml +++ b/data/semantickernel-data-hsqldb/pom.xml @@ -1,12 +1,10 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../../pom.xml diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index f1e62c6a..00b96faf 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../../pom.xml diff --git a/data/semantickernel-data-mysql/pom.xml b/data/semantickernel-data-mysql/pom.xml index 899f1f2a..89ab03ed 100644 --- a/data/semantickernel-data-mysql/pom.xml +++ b/data/semantickernel-data-mysql/pom.xml @@ -1,12 +1,10 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../../pom.xml diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml index aa953870..c42aa9fe 100644 --- a/data/semantickernel-data-oracle/pom.xml +++ b/data/semantickernel-data-oracle/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../../pom.xml diff --git a/data/semantickernel-data-postgres/pom.xml b/data/semantickernel-data-postgres/pom.xml index 56e65685..31157dd3 100644 --- a/data/semantickernel-data-postgres/pom.xml +++ b/data/semantickernel-data-postgres/pom.xml @@ -1,12 +1,10 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../../pom.xml diff --git a/data/semantickernel-data-redis/pom.xml b/data/semantickernel-data-redis/pom.xml index 4af2dc6a..47b7ed82 100644 --- a/data/semantickernel-data-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../../pom.xml diff --git a/data/semantickernel-data-sqlite/pom.xml b/data/semantickernel-data-sqlite/pom.xml index 64b4a6db..74b199bf 100644 --- a/data/semantickernel-data-sqlite/pom.xml +++ b/data/semantickernel-data-sqlite/pom.xml @@ -1,12 +1,10 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../../pom.xml diff --git a/pom.xml b/pom.xml index 6319235d..8a075a7b 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 pom https://www.github.com/microsoft/semantic-kernel @@ -930,6 +930,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - HEAD + java-1.4.4-RC2 diff --git a/samples/pom.xml b/samples/pom.xml index 30246ab2..df485ba2 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index 1018e6c6..d9e075c0 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 81e477c9..ae224749 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index 3f5a9a3f..06f06a7f 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index af125132..9a94bcdf 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index 18ee76ae..86584673 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index 4427cf20..de3bcaba 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 4b41f13a..0c1a63e8 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index ee30ae29..a6a22e2b 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -1,11 +1,10 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index 0c88458e..9b6cbeb7 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index 37c49ebc..8cc8dc80 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml index be88deb6..d7ed94c4 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../pom.xml diff --git a/semantickernel-api-ai-services/pom.xml b/semantickernel-api-ai-services/pom.xml index 9b4d76f6..5a7d6756 100644 --- a/semantickernel-api-ai-services/pom.xml +++ b/semantickernel-api-ai-services/pom.xml @@ -1,12 +1,10 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../pom.xml diff --git a/semantickernel-api-builders/pom.xml b/semantickernel-api-builders/pom.xml index 5bc0e11b..4ae0382f 100644 --- a/semantickernel-api-builders/pom.xml +++ b/semantickernel-api-builders/pom.xml @@ -1,12 +1,10 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 com.microsoft.semantic-kernel diff --git a/semantickernel-api-data/pom.xml b/semantickernel-api-data/pom.xml index 822b7c57..a17e2108 100644 --- a/semantickernel-api-data/pom.xml +++ b/semantickernel-api-data/pom.xml @@ -1,12 +1,10 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../pom.xml diff --git a/semantickernel-api-exceptions/pom.xml b/semantickernel-api-exceptions/pom.xml index b459db13..7fe68e6d 100644 --- a/semantickernel-api-exceptions/pom.xml +++ b/semantickernel-api-exceptions/pom.xml @@ -1,12 +1,10 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../pom.xml diff --git a/semantickernel-api-localization/pom.xml b/semantickernel-api-localization/pom.xml index 84be7960..471a2bbb 100644 --- a/semantickernel-api-localization/pom.xml +++ b/semantickernel-api-localization/pom.xml @@ -1,12 +1,10 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../pom.xml diff --git a/semantickernel-api-textembedding-services/pom.xml b/semantickernel-api-textembedding-services/pom.xml index 48a524ed..eac391f7 100644 --- a/semantickernel-api-textembedding-services/pom.xml +++ b/semantickernel-api-textembedding-services/pom.xml @@ -1,12 +1,10 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index 2463ec77..eee86b5b 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -1,13 +1,12 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index f88dd038..e4ffe7fe 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -1,12 +1,11 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-bom - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 pom Semantic Kernel Java BOM @@ -327,6 +326,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - HEAD + java-1.4.4-RC2 diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 538c9498..f7734a9c 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2-SNAPSHOT + 1.4.4-RC2 semantickernel-experimental From 488e36958cb717b610d3a675fe8c816d107e967a Mon Sep 17 00:00:00 2001 From: GitHub Date: Thu, 18 Sep 2025 11:54:06 +0100 Subject: [PATCH 241/277] [maven-release-plugin] prepare for next development iteration --- agents/semantickernel-agents-core/pom.xml | 2 +- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- data/semantickernel-data-azureaisearch/pom.xml | 2 +- data/semantickernel-data-hsqldb/pom.xml | 2 +- data/semantickernel-data-jdbc/pom.xml | 2 +- data/semantickernel-data-mysql/pom.xml | 2 +- data/semantickernel-data-oracle/pom.xml | 2 +- data/semantickernel-data-postgres/pom.xml | 2 +- data/semantickernel-data-redis/pom.xml | 2 +- data/semantickernel-data-sqlite/pom.xml | 2 +- pom.xml | 4 ++-- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- .../semantickernel-text-splitter-plugin/pom.xml | 2 +- semantickernel-api-ai-services/pom.xml | 2 +- semantickernel-api-builders/pom.xml | 2 +- semantickernel-api-data/pom.xml | 2 +- semantickernel-api-exceptions/pom.xml | 2 +- semantickernel-api-localization/pom.xml | 2 +- semantickernel-api-textembedding-services/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-bom/pom.xml | 4 ++-- semantickernel-experimental/pom.xml | 2 +- 36 files changed, 38 insertions(+), 38 deletions(-) diff --git a/agents/semantickernel-agents-core/pom.xml b/agents/semantickernel-agents-core/pom.xml index ef6cdf0d..1270d00d 100644 --- a/agents/semantickernel-agents-core/pom.xml +++ b/agents/semantickernel-agents-core/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../../pom.xml diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index 9d6025be..145ee549 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index 07031ddd..152aba54 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index f8597091..992629a4 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index c6878c36..862cc518 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index a56a45b9..587dfe5b 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../pom.xml diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml index d12b152d..e9db7daa 100644 --- a/data/semantickernel-data-azureaisearch/pom.xml +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-hsqldb/pom.xml b/data/semantickernel-data-hsqldb/pom.xml index 46a2c1ad..1cd33179 100644 --- a/data/semantickernel-data-hsqldb/pom.xml +++ b/data/semantickernel-data-hsqldb/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 00b96faf..077d4da6 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-mysql/pom.xml b/data/semantickernel-data-mysql/pom.xml index 89ab03ed..3d6d40e6 100644 --- a/data/semantickernel-data-mysql/pom.xml +++ b/data/semantickernel-data-mysql/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml index c42aa9fe..9971679b 100644 --- a/data/semantickernel-data-oracle/pom.xml +++ b/data/semantickernel-data-oracle/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-postgres/pom.xml b/data/semantickernel-data-postgres/pom.xml index 31157dd3..8f22241d 100644 --- a/data/semantickernel-data-postgres/pom.xml +++ b/data/semantickernel-data-postgres/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-redis/pom.xml b/data/semantickernel-data-redis/pom.xml index 47b7ed82..de2f60ee 100644 --- a/data/semantickernel-data-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-sqlite/pom.xml b/data/semantickernel-data-sqlite/pom.xml index 74b199bf..fc4d8018 100644 --- a/data/semantickernel-data-sqlite/pom.xml +++ b/data/semantickernel-data-sqlite/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../../pom.xml diff --git a/pom.xml b/pom.xml index 8a075a7b..8399f1c7 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT pom https://www.github.com/microsoft/semantic-kernel @@ -930,6 +930,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.4.4-RC2 + HEAD diff --git a/samples/pom.xml b/samples/pom.xml index df485ba2..d2bea638 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index d9e075c0..0b51d125 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index ae224749..bd5b298c 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index 06f06a7f..60bc7ffb 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index 9a94bcdf..c19477a7 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index 86584673..d7755a11 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index de3bcaba..079b5af0 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 0c1a63e8..8d5130b2 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index a6a22e2b..0abacea7 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index 9b6cbeb7..f0b57642 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index 8cc8dc80..10326e6e 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml index d7ed94c4..1bf80430 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../pom.xml diff --git a/semantickernel-api-ai-services/pom.xml b/semantickernel-api-ai-services/pom.xml index 5a7d6756..6187b56b 100644 --- a/semantickernel-api-ai-services/pom.xml +++ b/semantickernel-api-ai-services/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../pom.xml diff --git a/semantickernel-api-builders/pom.xml b/semantickernel-api-builders/pom.xml index 4ae0382f..be51b46f 100644 --- a/semantickernel-api-builders/pom.xml +++ b/semantickernel-api-builders/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT com.microsoft.semantic-kernel diff --git a/semantickernel-api-data/pom.xml b/semantickernel-api-data/pom.xml index a17e2108..13bd1993 100644 --- a/semantickernel-api-data/pom.xml +++ b/semantickernel-api-data/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../pom.xml diff --git a/semantickernel-api-exceptions/pom.xml b/semantickernel-api-exceptions/pom.xml index 7fe68e6d..b07b001c 100644 --- a/semantickernel-api-exceptions/pom.xml +++ b/semantickernel-api-exceptions/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../pom.xml diff --git a/semantickernel-api-localization/pom.xml b/semantickernel-api-localization/pom.xml index 471a2bbb..f1c3c246 100644 --- a/semantickernel-api-localization/pom.xml +++ b/semantickernel-api-localization/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../pom.xml diff --git a/semantickernel-api-textembedding-services/pom.xml b/semantickernel-api-textembedding-services/pom.xml index eac391f7..eb839135 100644 --- a/semantickernel-api-textembedding-services/pom.xml +++ b/semantickernel-api-textembedding-services/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index eee86b5b..ace57557 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index e4ffe7fe..372e04c1 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT pom Semantic Kernel Java BOM @@ -326,6 +326,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.4.4-RC2 + HEAD diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index f7734a9c..5cdbd534 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC2 + 1.4.4-RC3-SNAPSHOT semantickernel-experimental From 811f6fe793121ae1ef1789c475ceb610afe927bc Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Thu, 18 Sep 2025 12:06:27 +0100 Subject: [PATCH 242/277] Release notes --- CHANGELOG.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0a3d203..0267d08f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# 1.4.4-RC2 + +- Upgrade many dependencies to recent versions +- Migrate from "Tool calls" to "Function calls" terminology, deprecated OpenAIFunctionToolCall +- Refactored Data storage area to be more extensible and added Oracle Database support (Thank you to the contributors + from Oracle for the contribution) + # 1.4.4-RC1 - Add Agent framework abstractions. @@ -25,7 +32,8 @@ # 1.4.0 - Upgrade to azure-ai-openai 1.0.0-beta.12 -- Add vector stores with vector search support for Azure AI Search, Redis, JDBC with Postgres, MySQL, SQLite and HSQLDB. Moving these features out of the experimental stage. +- Add vector stores with vector search support for Azure AI Search, Redis, JDBC with Postgres, MySQL, SQLite and HSQLDB. + Moving these features out of the experimental stage. # 1.3.0 From b40a8ccd7b58180d7165793a8edc503120bf8d23 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Oct 2025 07:28:19 +0000 Subject: [PATCH 243/277] Bump github/codeql-action from 2 to 4 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2 to 4. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v2...v4) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index ba586a25..9a2bc55e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -34,7 +34,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -48,7 +48,7 @@ jobs: # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild if: ${{ matrix.language != 'java' }} - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v4 - name: Setup JDK uses: actions/setup-java@v5 @@ -73,6 +73,6 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v4 with: category: "/language:${{matrix.language}}" From 141b8c401881c15b46d469777ef1b25ab21adb1a Mon Sep 17 00:00:00 2001 From: Martijn Verburg Date: Mon, 13 Oct 2025 19:38:52 +0100 Subject: [PATCH 244/277] Fix typo to pass GH Actions --- .../syntaxexamples/functions/Example09_FunctionTypes.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example09_FunctionTypes.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example09_FunctionTypes.java index 4c14ef54..92aab554 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example09_FunctionTypes.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example09_FunctionTypes.java @@ -698,7 +698,7 @@ public Mono> WithListReturn3() { * public async Task * TaskInjectingCancellationTokenWithStringResult(CancellationToken cancellationToken) * { - * var result = $"Cancellation resquested: {cancellationToken.IsCancellationRequested}"; + * var result = $"Cancellation requested: {cancellationToken.IsCancellationRequested}"; * Console.WriteLine( * $"Running {nameof(this.TaskInjectingCultureInfoOrIFormatProviderWithStringResult)} -> Injected Cancellation Token -> result: {result}" * ); From 773c37883e20caa3f42b37965371d9f32d73b0dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 07:28:27 +0000 Subject: [PATCH 245/277] Bump actions/upload-artifact from 4 to 5 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4 to 5. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/java-build.yml | 2 +- .github/workflows/java-publish-package.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/java-build.yml b/.github/workflows/java-build.yml index 73edf46d..08e039a0 100644 --- a/.github/workflows/java-build.yml +++ b/.github/workflows/java-build.yml @@ -56,7 +56,7 @@ jobs: run: ./mvnw -B -Pbug-check -Pcompile-jdk${{ matrix.java-versions }} test --file pom.xml # Uploads test artifacts for each JDK version - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v5 if: always() with: name: test_output_sk_jdk${{ matrix.java-versions }}u diff --git a/.github/workflows/java-publish-package.yml b/.github/workflows/java-publish-package.yml index 6da0a6c5..f2631c25 100644 --- a/.github/workflows/java-publish-package.yml +++ b/.github/workflows/java-publish-package.yml @@ -30,7 +30,7 @@ jobs: run: ./mvnw -B -DskipTests -Pcompile-jdk8 -P-compile-jdk17 clean deploy --file pom.xml -DaltDeploymentRepository=local::file:///tmp/target/staging-deploy - name: Upload Artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: Artifacts path: /tmp/target/staging-deploy From 3904ce365f2a68a0e8ffa9dbb64e9d006d68cbcc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 07:10:26 +0000 Subject: [PATCH 246/277] Bump actions/checkout from 5 to 6 Bumps [actions/checkout](https://github.com/actions/checkout) from 5 to 6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/java-build.yml | 2 +- .github/workflows/java-integration-tests.yml | 2 +- .github/workflows/java-publish-package.yml | 2 +- .github/workflows/markdown-link-check.yml | 2 +- .github/workflows/typos.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 9a2bc55e..dee4d1c4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -30,7 +30,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/java-build.yml b/.github/workflows/java-build.yml index 08e039a0..ae14c035 100644 --- a/.github/workflows/java-build.yml +++ b/.github/workflows/java-build.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 # Need to use JDK 11 to build for JDK 8 - name: Set JDK diff --git a/.github/workflows/java-integration-tests.yml b/.github/workflows/java-integration-tests.yml index f8d25c33..17edd75e 100644 --- a/.github/workflows/java-integration-tests.yml +++ b/.github/workflows/java-integration-tests.yml @@ -22,7 +22,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 # Need to use JDK 11 to build for JDK 8 - name: Set JDK diff --git a/.github/workflows/java-publish-package.yml b/.github/workflows/java-publish-package.yml index f2631c25..93a136f4 100644 --- a/.github/workflows/java-publish-package.yml +++ b/.github/workflows/java-publish-package.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 # Sets up the specified JDK version from the matrix - uses: actions/setup-java@v5 diff --git a/.github/workflows/markdown-link-check.yml b/.github/workflows/markdown-link-check.yml index 4145df27..bc1a1fa7 100644 --- a/.github/workflows/markdown-link-check.yml +++ b/.github/workflows/markdown-link-check.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest # check out the latest version of the code steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 # Checks the status of hyperlinks in .md files in verbose mode - name: Check links diff --git a/.github/workflows/typos.yaml b/.github/workflows/typos.yaml index 6452831b..532f4d5d 100644 --- a/.github/workflows/typos.yaml +++ b/.github/workflows/typos.yaml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Use custom config file uses: crate-ci/typos@master From 07f20c327f98a27495116545b280009e4a0143ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 07:09:34 +0000 Subject: [PATCH 247/277] Bump actions/upload-artifact from 5 to 6 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 5 to 6. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/java-build.yml | 2 +- .github/workflows/java-publish-package.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/java-build.yml b/.github/workflows/java-build.yml index ae14c035..32b71f75 100644 --- a/.github/workflows/java-build.yml +++ b/.github/workflows/java-build.yml @@ -56,7 +56,7 @@ jobs: run: ./mvnw -B -Pbug-check -Pcompile-jdk${{ matrix.java-versions }} test --file pom.xml # Uploads test artifacts for each JDK version - - uses: actions/upload-artifact@v5 + - uses: actions/upload-artifact@v6 if: always() with: name: test_output_sk_jdk${{ matrix.java-versions }}u diff --git a/.github/workflows/java-publish-package.yml b/.github/workflows/java-publish-package.yml index 93a136f4..8cc4e00c 100644 --- a/.github/workflows/java-publish-package.yml +++ b/.github/workflows/java-publish-package.yml @@ -30,7 +30,7 @@ jobs: run: ./mvnw -B -DskipTests -Pcompile-jdk8 -P-compile-jdk17 clean deploy --file pom.xml -DaltDeploymentRepository=local::file:///tmp/target/staging-deploy - name: Upload Artifacts - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@v6 with: name: Artifacts path: /tmp/target/staging-deploy From 90457050cb93494675aac827f1e535cd86307f4f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 19 Dec 2025 23:50:26 +0000 Subject: [PATCH 248/277] Bump org.apache.logging.log4j:log4j-core from 2.24.1 to 2.25.3 Bumps org.apache.logging.log4j:log4j-core from 2.24.1 to 2.25.3. --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-core dependency-version: 2.25.3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8399f1c7..b096c4a4 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 1.17.0 1.6.0 5.11.3 - 2.24.1 + 2.25.3 3.1.0 2.12.1 3.5.0 From 62684bd2736f1ea1e9df4153e56cd86ec28437c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jan 2026 21:53:13 +0000 Subject: [PATCH 249/277] Bump org.assertj:assertj-core Bumps [org.assertj:assertj-core](https://github.com/assertj/assertj) from 3.26.3 to 3.27.7. - [Release notes](https://github.com/assertj/assertj/releases) - [Commits](https://github.com/assertj/assertj/compare/assertj-build-3.26.3...assertj-build-3.27.7) --- updated-dependencies: - dependency-name: org.assertj:assertj-core dependency-version: 3.27.7 dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- .../semantickernel-demos/semantickernel-spring-starter/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index d7755a11..8db11dcd 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -37,7 +37,7 @@ org.assertj assertj-core - 3.26.3 + 3.27.7 test From c222fb37580f0e483e560e898ef7795ea4813834 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Tue, 3 Feb 2026 16:31:06 +0000 Subject: [PATCH 250/277] Bump testcontainers version --- data/semantickernel-data-oracle/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml index 9971679b..e2c93443 100644 --- a/data/semantickernel-data-oracle/pom.xml +++ b/data/semantickernel-data-oracle/pom.xml @@ -13,7 +13,7 @@ Provides a Oracle connector for the Semantic Kernel - 1.20.4 + 1.21.4 From 65fbb3b8f3b731d7c356077ac29ef6ab4e46c69c Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Tue, 3 Feb 2026 16:37:08 +0000 Subject: [PATCH 251/277] Fix typo --- .../plugin/Example13_ConversationSummaryPlugin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugin/Example13_ConversationSummaryPlugin.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugin/Example13_ConversationSummaryPlugin.java index 63412b60..47f0e5a4 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugin/Example13_ConversationSummaryPlugin.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/plugin/Example13_ConversationSummaryPlugin.java @@ -107,7 +107,7 @@ public class Example13_ConversationSummaryPlugin { Jane: Darn, it's just repeating stuff now. John: I think we're done. Jane: We're not though! We need like 1500 more characters. - John: Oh Cananda, our home and native land. + John: Oh Canada, our home and native land. Jane: True patriot love in all thy sons command. John: With glowing hearts we see thee rise. Jane: The True North strong and free. From 1db294c09b82179fa74648c50cd22dc595ee2dee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 03:19:04 +0000 Subject: [PATCH 252/277] Bump com.fasterxml.jackson.core:jackson-core in /semantickernel-bom Bumps [com.fasterxml.jackson.core:jackson-core](https://github.com/FasterXML/jackson-core) from 2.18.0 to 2.18.6. - [Commits](https://github.com/FasterXML/jackson-core/compare/jackson-core-2.18.0...jackson-core-2.18.6) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.core:jackson-core dependency-version: 2.18.6 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- semantickernel-bom/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 372e04c1..303e08b7 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -13,7 +13,7 @@ https://www.github.com/microsoft/semantic-kernel - 2.18.0 + 2.18.6 From 58fbd5082732f0c99a21e053e4e2aec624fec475 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 03:33:41 +0000 Subject: [PATCH 253/277] Bump net.sourceforge.pmd:pmd-core from 7.10.0 to 7.22.0 Bumps [net.sourceforge.pmd:pmd-core](https://github.com/pmd/pmd) from 7.10.0 to 7.22.0. - [Release notes](https://github.com/pmd/pmd/releases) - [Commits](https://github.com/pmd/pmd/compare/pmd_releases/7.10.0...pmd_releases/7.22.0) --- updated-dependencies: - dependency-name: net.sourceforge.pmd:pmd-core dependency-version: 7.22.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b096c4a4..9a52016c 100644 --- a/pom.xml +++ b/pom.xml @@ -60,7 +60,7 @@ 5.14.2 0.9.1 - 7.10.0 + 7.22.0 UTF-8 microsoft/semantic-kernel git@github.com:${project.github.repository}.git From 21b15c11fd8c969235c75babcfefc5883afdfaa8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 07:54:17 +0000 Subject: [PATCH 254/277] Bump actions/upload-artifact from 6 to 7 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 6 to 7. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/java-build.yml | 2 +- .github/workflows/java-publish-package.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/java-build.yml b/.github/workflows/java-build.yml index 32b71f75..d63a0d44 100644 --- a/.github/workflows/java-build.yml +++ b/.github/workflows/java-build.yml @@ -56,7 +56,7 @@ jobs: run: ./mvnw -B -Pbug-check -Pcompile-jdk${{ matrix.java-versions }} test --file pom.xml # Uploads test artifacts for each JDK version - - uses: actions/upload-artifact@v6 + - uses: actions/upload-artifact@v7 if: always() with: name: test_output_sk_jdk${{ matrix.java-versions }}u diff --git a/.github/workflows/java-publish-package.yml b/.github/workflows/java-publish-package.yml index 8cc4e00c..1f73e688 100644 --- a/.github/workflows/java-publish-package.yml +++ b/.github/workflows/java-publish-package.yml @@ -30,7 +30,7 @@ jobs: run: ./mvnw -B -DskipTests -Pcompile-jdk8 -P-compile-jdk17 clean deploy --file pom.xml -DaltDeploymentRepository=local::file:///tmp/target/staging-deploy - name: Upload Artifacts - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v7 with: name: Artifacts path: /tmp/target/staging-deploy From 33e1a5d334fb6862dd457913f095a2fe5f232d9a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Apr 2026 23:47:20 +0000 Subject: [PATCH 255/277] Bump org.apache.logging.log4j:log4j-core from 2.25.3 to 2.25.4 Bumps org.apache.logging.log4j:log4j-core from 2.25.3 to 2.25.4. --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-core dependency-version: 2.25.4 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9a52016c..0a6838be 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 1.17.0 1.6.0 5.11.3 - 2.25.3 + 2.25.4 3.1.0 2.12.1 3.5.0 From 7177b90264138d4b730fd328c993662a997262cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 08:01:09 +0000 Subject: [PATCH 256/277] Bump actions/github-script from 8 to 9 Bumps [actions/github-script](https://github.com/actions/github-script) from 8 to 9. - [Release notes](https://github.com/actions/github-script/releases) - [Commits](https://github.com/actions/github-script/compare/v8...v9) --- updated-dependencies: - dependency-name: actions/github-script dependency-version: '9' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/label-issues.yml | 2 +- .github/workflows/label-title-prefix.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/label-issues.yml b/.github/workflows/label-issues.yml index cf901ade..d5a48785 100644 --- a/.github/workflows/label-issues.yml +++ b/.github/workflows/label-issues.yml @@ -13,7 +13,7 @@ jobs: permissions: issues: write steps: - - uses: actions/github-script@v8 + - uses: actions/github-script@v9 with: github-token: ${{ secrets.GH_ACTIONS_PR_WRITE }} script: | diff --git a/.github/workflows/label-title-prefix.yml b/.github/workflows/label-title-prefix.yml index 81369dee..e84551dd 100644 --- a/.github/workflows/label-title-prefix.yml +++ b/.github/workflows/label-title-prefix.yml @@ -15,7 +15,7 @@ jobs: pull-requests: write steps: - - uses: actions/github-script@v8 + - uses: actions/github-script@v9 name: "Issue/PR: update title" with: github-token: ${{ secrets.GITHUB_TOKEN }} From c3230235a2dc5b134694c4a83fbc24f86d3c7277 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Thu, 16 Apr 2026 12:28:37 +0000 Subject: [PATCH 257/277] Fix storage filter issues Update testcontainers and fix testing Remove unsused test --- api-test/integration-tests/pom.xml | 9 +- .../tests/data/redis/Hotel.java | 4 +- ...ashSetVectorStoreRecordCollectionTest.java | 138 +++++++++--------- ...isJsonVectorStoreRecordCollectionTest.java | 124 ++++++++-------- .../semantickernel-data-azureaisearch/pom.xml | 20 ++- ...rchVectorStoreCollectionSearchMapping.java | 19 ++- data/semantickernel-data-redis/pom.xml | 7 + ...disVectorStoreCollectionSearchMapping.java | 19 ++- 8 files changed, 197 insertions(+), 143 deletions(-) diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 862cc518..256e4cb7 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -122,7 +122,7 @@ com.redis testcontainers-redis - 2.2.2 + 2.2.4 test @@ -150,6 +150,11 @@ 2.7.3 test + + com.microsoft.semantic-kernel + semantickernel-api-data + test + @@ -157,7 +162,7 @@ org.testcontainers testcontainers-bom - 1.18.3 + 1.21.4 pom import diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/Hotel.java index 416e06b6..46083425 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/Hotel.java @@ -18,11 +18,11 @@ public class Hotel { @VectorStoreRecordData(isFilterable = true) private final String name; - @VectorStoreRecordData + @VectorStoreRecordData(isFilterable = true) private final int code; @JsonProperty("summary") - @VectorStoreRecordData() + @VectorStoreRecordData(isFilterable = true) private final String description; @JsonProperty("summaryEmbedding1") diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisHashSetVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisHashSetVectorStoreRecordCollectionTest.java index c5b6a186..6bfebb8d 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisHashSetVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisHashSetVectorStoreRecordCollectionTest.java @@ -53,56 +53,56 @@ public enum RecordCollectionOptions { @BeforeAll static void setup() { optionsMap.put(RecordCollectionOptions.DEFAULT, RedisHashSetVectorStoreRecordCollectionOptions.builder() - .withRecordClass(Hotel.class) - .build()); + .withRecordClass(Hotel.class) + .build()); List fields = new ArrayList<>(); fields.add(VectorStoreRecordKeyField.builder() - .withName("id") - .withFieldType(String.class) - .build()); + .withName("id") + .withFieldType(String.class) + .build()); fields.add(VectorStoreRecordDataField.builder() - .withName("name") - .withFieldType(String.class) - .build()); + .withName("name") + .withFieldType(String.class) + .build()); fields.add(VectorStoreRecordDataField.builder() - .withName("code") - .withFieldType(Integer.class) - .build()); + .withName("code") + .withFieldType(Integer.class) + .build()); fields.add(VectorStoreRecordDataField.builder() - .withName("description") - .withStorageName("summary") - .withFieldType(String.class) - .build()); + .withName("description") + .withStorageName("summary") + .withFieldType(String.class) + .build()); fields.add(VectorStoreRecordVectorField.builder() - .withName("euclidean") - .withStorageName("summaryEmbedding1") - .withFieldType(List.class) - .withDimensions(8) - .build()); + .withName("euclidean") + .withStorageName("summaryEmbedding1") + .withFieldType(List.class) + .withDimensions(8) + .build()); fields.add(VectorStoreRecordVectorField.builder() - .withName("cosineDistance") - .withStorageName("summaryEmbedding2") - .withFieldType(List.class) - .withDimensions(8) - .build()); + .withName("cosineDistance") + .withStorageName("summaryEmbedding2") + .withFieldType(List.class) + .withDimensions(8) + .build()); fields.add(VectorStoreRecordVectorField.builder() - .withName("dotProduct") - .withStorageName("summaryEmbedding3") - .withFieldType(List.class) - .withDimensions(8) - .build()); + .withName("dotProduct") + .withStorageName("summaryEmbedding3") + .withFieldType(List.class) + .withDimensions(8) + .build()); fields.add(VectorStoreRecordDataField.builder() - .withName("rating") - .withFieldType(Double.class) - .isFilterable(true) - .build()); + .withName("rating") + .withFieldType(Double.class) + .isFilterable(true) + .build()); VectorStoreRecordDefinition recordDefinition = VectorStoreRecordDefinition.fromFields(fields); optionsMap.put(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, RedisHashSetVectorStoreRecordCollectionOptions.builder() - .withRecordClass(Hotel.class) - .withRecordDefinition(recordDefinition) - .build()); + .withRecordClass(Hotel.class) + .withRecordDefinition(recordDefinition) + .build()); // Search configuration List hotels = getHotels(); @@ -127,20 +127,20 @@ static void setup() { private static RedisHashSetVectorStoreRecordCollection createCollection(@Nonnull RedisHashSetVectorStoreRecordCollectionOptions options, @Nonnull String collectionName) { return new RedisHashSetVectorStoreRecordCollection<>(new JedisPooled(redisContainer.getRedisURI()), collectionName, RedisHashSetVectorStoreRecordCollectionOptions.builder() - .withRecordClass(options.getRecordClass()) - .withVectorStoreRecordMapper(options.getVectorStoreRecordMapper()) - .withRecordDefinition(options.getRecordDefinition()) - .withPrefixCollectionName(options.isPrefixCollectionName()) - .build()); + .withRecordClass(options.getRecordClass()) + .withVectorStoreRecordMapper(options.getVectorStoreRecordMapper()) + .withRecordDefinition(options.getRecordDefinition()) + .withPrefixCollectionName(options.isPrefixCollectionName()) + .build()); } private static List getHotels() { return Arrays.asList( - new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f),null, null, 4.0), - new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f),null, null, 4.0), - new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f),null, null, 5.0), - new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f),null, null, 4.0), - new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f),null, null, 4.0) + new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f),null, null, 4.0), + new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f),null, null, 4.0), + new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f),null, null, 5.0), + new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f),null, null, 4.0), + new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f),null, null, 4.0) ); } @@ -362,12 +362,12 @@ public void getBatchAsyncWithNoVectors(RecordCollectionOptions options) { private static Stream provideSearchParameters() { return Stream.of( - Arguments.of(RecordCollectionOptions.DEFAULT, "euclidean"), - Arguments.of(RecordCollectionOptions.DEFAULT, "cosineDistance"), - Arguments.of(RecordCollectionOptions.DEFAULT, "dotProduct"), - Arguments.of(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, "euclidean"), - Arguments.of(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, "cosineDistance"), - Arguments.of(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, "dotProduct") + Arguments.of(RecordCollectionOptions.DEFAULT, "euclidean"), + Arguments.of(RecordCollectionOptions.DEFAULT, "cosineDistance"), + Arguments.of(RecordCollectionOptions.DEFAULT, "dotProduct"), + Arguments.of(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, "euclidean"), + Arguments.of(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, "cosineDistance"), + Arguments.of(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, "dotProduct") ); } @@ -383,8 +383,8 @@ public void search(RecordCollectionOptions options, String embeddingName) { recordCollection.upsertBatchAsync(hotels, null).block(); VectorSearchOptions searchOptions = VectorSearchOptions.builder() - .withVectorFieldName(embeddingName) - .build(); + .withVectorFieldName(embeddingName) + .build(); // Embeddings similar to the third hotel List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block().getResults(); @@ -407,9 +407,9 @@ public void searchWithVectors(RecordCollectionOptions options, String embeddingN recordCollection.upsertBatchAsync(hotels, null).block(); VectorSearchOptions searchOptions = VectorSearchOptions.builder() - .withVectorFieldName(embeddingName) - .withIncludeVectors(true) - .build(); + .withVectorFieldName(embeddingName) + .withIncludeVectors(true) + .build(); // Embeddings similar to the third hotel List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block().getResults(); @@ -430,10 +430,10 @@ public void searchWithOffSet(RecordCollectionOptions options, String embeddingNa recordCollection.upsertBatchAsync(hotels, null).block(); VectorSearchOptions searchOptions = VectorSearchOptions.builder() - .withVectorFieldName(embeddingName) - .withSkip(1) - .withTop(4) - .build(); + .withVectorFieldName(embeddingName) + .withSkip(1) + .withTop(4) + .build(); // Embeddings similar to the third hotel List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block().getResults(); @@ -453,16 +453,16 @@ public void searchWithFilterEqualToFilter(RecordCollectionOptions recordCollecti recordCollection.upsertBatchAsync(hotels, null).block(); VectorSearchOptions options = VectorSearchOptions.builder() - .withVectorFieldName(embeddingName) - .withTop(3) - .withVectorSearchFilter( - VectorSearchFilter.builder() - .equalTo("rating", 4.0).build()) - .build(); + .withVectorFieldName(embeddingName) + .withTop(3) + .withVectorSearchFilter( + VectorSearchFilter.builder() + .equalTo("rating", 4.0).build()) + .build(); // Embeddings similar to the third hotel, but as the filter is set to 4.0, the third hotel should not be returned List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, - options).block().getResults(); + options).block().getResults(); assertNotNull(results); assertEquals(3, results.size()); // The first hotel should be the most similar diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisJsonVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisJsonVectorStoreRecordCollectionTest.java index 1b4c30e8..1190e437 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisJsonVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/data/redis/RedisJsonVectorStoreRecordCollectionTest.java @@ -53,56 +53,56 @@ public enum RecordCollectionOptions { @BeforeAll static void setup() { optionsMap.put(RecordCollectionOptions.DEFAULT, RedisJsonVectorStoreRecordCollectionOptions.builder() - .withRecordClass(Hotel.class) - .build()); + .withRecordClass(Hotel.class) + .build()); List fields = new ArrayList<>(); fields.add(VectorStoreRecordKeyField.builder() - .withName("id") - .withFieldType(String.class) - .build()); + .withName("id") + .withFieldType(String.class) + .build()); fields.add(VectorStoreRecordDataField.builder() - .withName("name") - .withFieldType(String.class) - .build()); + .withName("name") + .withFieldType(String.class) + .build()); fields.add(VectorStoreRecordDataField.builder() - .withName("code") - .withFieldType(Integer.class) - .build()); + .withName("code") + .withFieldType(Integer.class) + .build()); fields.add(VectorStoreRecordDataField.builder() - .withName("description") - .withStorageName("summary") - .withFieldType(String.class) - .build()); + .withName("description") + .withStorageName("summary") + .withFieldType(String.class) + .build()); fields.add(VectorStoreRecordVectorField.builder() - .withName("euclidean") - .withStorageName("summaryEmbedding1") - .withFieldType(List.class) - .withDimensions(8) - .build()); + .withName("euclidean") + .withStorageName("summaryEmbedding1") + .withFieldType(List.class) + .withDimensions(8) + .build()); fields.add(VectorStoreRecordVectorField.builder() - .withName("cosineDistance") - .withStorageName("summaryEmbedding2") - .withFieldType(List.class) - .withDimensions(8) - .build()); + .withName("cosineDistance") + .withStorageName("summaryEmbedding2") + .withFieldType(List.class) + .withDimensions(8) + .build()); fields.add(VectorStoreRecordVectorField.builder() - .withName("dotProduct") - .withStorageName("summaryEmbedding3") - .withFieldType(List.class) - .withDimensions(8) - .build()); + .withName("dotProduct") + .withStorageName("summaryEmbedding3") + .withFieldType(List.class) + .withDimensions(8) + .build()); fields.add(VectorStoreRecordDataField.builder() - .withName("rating") - .withFieldType(Double.class) - .isFilterable(true) - .build()); + .withName("rating") + .withFieldType(Double.class) + .isFilterable(true) + .build()); VectorStoreRecordDefinition recordDefinition = VectorStoreRecordDefinition.fromFields(fields); optionsMap.put(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, RedisJsonVectorStoreRecordCollectionOptions.builder() - .withRecordClass(Hotel.class) - .withRecordDefinition(recordDefinition) - .build()); + .withRecordClass(Hotel.class) + .withRecordDefinition(recordDefinition) + .build()); // Search configuration List hotels = getHotels(); @@ -127,20 +127,20 @@ static void setup() { private static RedisJsonVectorStoreRecordCollection createCollection(@Nonnull RedisJsonVectorStoreRecordCollectionOptions options, @Nonnull String collectionName) { return new RedisJsonVectorStoreRecordCollection<>(new JedisPooled(redisContainer.getRedisURI()), collectionName, RedisJsonVectorStoreRecordCollectionOptions.builder() - .withRecordClass(options.getRecordClass()) - .withVectorStoreRecordMapper(options.getVectorStoreRecordMapper()) - .withRecordDefinition(options.getRecordDefinition()) - .withPrefixCollectionName(options.isPrefixCollectionName()) - .build()); + .withRecordClass(options.getRecordClass()) + .withVectorStoreRecordMapper(options.getVectorStoreRecordMapper()) + .withRecordDefinition(options.getRecordDefinition()) + .withPrefixCollectionName(options.isPrefixCollectionName()) + .build()); } private static List getHotels() { return Arrays.asList( - new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f),null, null, 4.0), - new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f),null, null, 4.0), - new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f),null, null, 5.0), - new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f),null, null, 4.0), - new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f),null, null, 4.0) + new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(0.5f, 3.2f, 7.1f, -4.0f, 2.8f, 10.0f, -1.3f, 5.5f),null, null, 4.0), + new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(-2.0f, 8.1f, 0.9f, 5.4f, -3.3f, 2.2f, 9.9f, -4.5f),null, null, 4.0), + new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(4.5f, -6.2f, 3.1f, 7.7f, -0.8f, 1.1f, -2.2f, 8.3f),null, null, 5.0), + new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(7.0f, 1.2f, -5.3f, 2.5f, 6.6f, -7.8f, 3.9f, -0.1f),null, null, 4.0), + new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(-3.5f, 4.4f, -1.2f, 9.9f, 5.7f, -6.1f, 7.8f, -2.0f),null, null, 4.0) ); } @@ -362,12 +362,12 @@ public void getBatchAsyncWithNoVectors(RecordCollectionOptions options) { private static Stream provideSearchParameters() { return Stream.of( - Arguments.of(RecordCollectionOptions.DEFAULT, "euclidean"), - Arguments.of(RecordCollectionOptions.DEFAULT, "cosineDistance"), - Arguments.of(RecordCollectionOptions.DEFAULT, "dotProduct"), - Arguments.of(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, "euclidean"), - Arguments.of(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, "cosineDistance"), - Arguments.of(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, "dotProduct") + Arguments.of(RecordCollectionOptions.DEFAULT, "euclidean"), + Arguments.of(RecordCollectionOptions.DEFAULT, "cosineDistance"), + Arguments.of(RecordCollectionOptions.DEFAULT, "dotProduct"), + Arguments.of(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, "euclidean"), + Arguments.of(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, "cosineDistance"), + Arguments.of(RecordCollectionOptions.WITH_CUSTOM_DEFINITION, "dotProduct") ); } @@ -383,8 +383,8 @@ public void search(RecordCollectionOptions options, String embeddingName) { recordCollection.upsertBatchAsync(hotels, null).block(); VectorSearchOptions searchOptions = VectorSearchOptions.builder() - .withVectorFieldName(embeddingName) - .build(); + .withVectorFieldName(embeddingName) + .build(); // Embeddings similar to the third hotel List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block().getResults(); @@ -407,9 +407,9 @@ public void searchWithVectors(RecordCollectionOptions options, String embeddingN recordCollection.upsertBatchAsync(hotels, null).block(); VectorSearchOptions searchOptions = VectorSearchOptions.builder() - .withVectorFieldName(embeddingName) - .withIncludeVectors(true) - .build(); + .withVectorFieldName(embeddingName) + .withIncludeVectors(true) + .build(); // Embeddings similar to the third hotel List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block().getResults(); @@ -430,10 +430,10 @@ public void searchWithOffSet(RecordCollectionOptions options, String embeddingNa recordCollection.upsertBatchAsync(hotels, null).block(); VectorSearchOptions searchOptions = VectorSearchOptions.builder() - .withVectorFieldName(embeddingName) - .withSkip(1) - .withTop(4) - .build(); + .withVectorFieldName(embeddingName) + .withSkip(1) + .withTop(4) + .build(); // Embeddings similar to the third hotel List> results = recordCollection.searchAsync(SEARCH_EMBEDDINGS, searchOptions).block().getResults(); diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml index e9db7daa..c0f0a8c6 100644 --- a/data/semantickernel-data-azureaisearch/pom.xml +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -1,5 +1,6 @@ - + 4.0.0 com.microsoft.semantic-kernel @@ -40,6 +41,23 @@ + + + + org.junit.jupiter + junit-jupiter + test + + + org.mockito + mockito-core + test + + + com.microsoft.semantic-kernel + semantickernel-api-builders + test + \ No newline at end of file diff --git a/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java b/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java index c31af301..5d61f981 100644 --- a/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java +++ b/data/semantickernel-data-azureaisearch/src/main/java/com/microsoft/semantickernel/data/azureaisearch/AzureAISearchVectorStoreCollectionSearchMapping.java @@ -57,11 +57,11 @@ public String getFilter(VectorSearchFilter vectorSearchFilter, @Override public String getEqualToFilter(EqualToFilterClause filterClause) { - String fieldName = filterClause.getFieldName(); + String fieldName = validateFieldName(filterClause.getFieldName()); Object value = filterClause.getValue(); if (value instanceof String) { - return String.format("%s eq '%s'", fieldName, value); + return String.format("%s eq '%s'", fieldName, escapeSingleQuotes((String) value)); } else if (value instanceof Boolean) { return String.format("%s eq %s", fieldName, value.toString().toLowerCase()); @@ -86,7 +86,18 @@ public String getEqualToFilter(EqualToFilterClause filterClause) { @Override public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { - return String.format("%s/any(t: t eq '%s')", filterClause.getFieldName(), - filterClause.getValue()); + return String.format("%s/any(t: t eq '%s')", validateFieldName(filterClause.getFieldName()), + escapeSingleQuotes(filterClause.getValue().toString())); + } + + private String validateFieldName(String fieldName) { + if (fieldName.matches("[a-zA-Z_][a-zA-Z0-9_]*")) { + return fieldName; + } + throw new SKException("Invalid field name: " + fieldName); + } + + private String escapeSingleQuotes(String value) { + return value.replaceAll("'", "''"); } } diff --git a/data/semantickernel-data-redis/pom.xml b/data/semantickernel-data-redis/pom.xml index de2f60ee..885084d3 100644 --- a/data/semantickernel-data-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -70,6 +70,13 @@ redis.clients jedis + + + + org.junit.jupiter + junit-jupiter + test + \ No newline at end of file diff --git a/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisVectorStoreCollectionSearchMapping.java b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisVectorStoreCollectionSearchMapping.java index f029f678..7f7d31bc 100644 --- a/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisVectorStoreCollectionSearchMapping.java +++ b/data/semantickernel-data-redis/src/main/java/com/microsoft/semantickernel/data/redis/RedisVectorStoreCollectionSearchMapping.java @@ -154,12 +154,12 @@ public String getFilter(VectorSearchFilter filter, */ @Override public String getEqualToFilter(EqualToFilterClause filterClause) { - String fieldName = filterClause.getFieldName(); + String fieldName = validateFieldName(filterClause.getFieldName()); Object value = filterClause.getValue(); String formattedValue; if (value instanceof String) { - formattedValue = String.format("\"%s\"", value); + formattedValue = String.format("\"%s\"", escapeRedisString((String) value)); } else if (value instanceof Number) { formattedValue = String.format("[%s %s]", value, value); } else { @@ -178,6 +178,19 @@ public String getEqualToFilter(EqualToFilterClause filterClause) { */ @Override public String getAnyTagEqualToFilter(AnyTagEqualToFilterClause filterClause) { - return String.format("@%s:\"%s\"", filterClause.getFieldName(), filterClause.getValue()); + return String.format("@%s:\"%s\"", + validateFieldName(filterClause.getFieldName()), + escapeRedisString(filterClause.getValue().toString())); + } + + private String validateFieldName(String fieldName) { + if (fieldName.matches("[a-zA-Z_][a-zA-Z0-9_]*")) { + return fieldName; + } + throw new SKException("Invalid field name: " + fieldName); + } + + private String escapeRedisString(String searchString) { + return searchString.replaceAll("([,.<>{}\\[\\]\"':;!@#$%^&*()\\-+=~|\\\\/?\\s])", "\\\\$1"); } } From a6809be8be3e001e4845251332e36bf24b4419a7 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Wed, 22 Apr 2026 16:25:03 +0000 Subject: [PATCH 258/277] Fix prompt parsing issues --- .../chatcompletion/ChatXMLPromptParser.java | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/chatcompletion/ChatXMLPromptParser.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/chatcompletion/ChatXMLPromptParser.java index 8cc1cc97..975f5e53 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/chatcompletion/ChatXMLPromptParser.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/implementation/chatcompletion/ChatXMLPromptParser.java @@ -18,6 +18,7 @@ import java.util.Locale; import java.util.Map; import javax.annotation.Nullable; +import javax.xml.XMLConstants; import javax.xml.namespace.QName; import javax.xml.stream.XMLEventReader; import javax.xml.stream.XMLInputFactory; @@ -32,6 +33,29 @@ public class ChatXMLPromptParser { private static final Logger LOGGER = LoggerFactory.getLogger(ChatXMLPromptParser.class); + private static XMLInputFactory createXMLInputFactory() { + XMLInputFactory factory = XMLInputFactory.newInstance(); + + trySetProperty(factory, XMLConstants.FEATURE_SECURE_PROCESSING, true); + trySetProperty(factory, XMLConstants.ACCESS_EXTERNAL_DTD, ""); + trySetProperty(factory, XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); + + factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + factory.setProperty(XMLInputFactory.SUPPORT_DTD, false); + factory.setProperty(XMLInputFactory.IS_REPLACING_ENTITY_REFERENCES, false); + + return factory; + } + + private static void trySetProperty(XMLInputFactory factory, String property, Object value) { + try { + factory.setProperty(property, value); + } catch (IllegalArgumentException e) { + // Property not supported by this XMLInputFactory implementation + LOGGER.trace("XMLInputFactory property '{}' not supported", property); + } + } + public static ChatPromptParseVisitor parse( String rawPrompt, ChatPromptParseVisitor chatPromptParseVisitor) { @@ -64,7 +88,7 @@ private static ChatPromptParseVisitor getChatRequestMessages(String promp // In this way, we can avoid parsing the whole prompt twice and easily extend the parsing logic. try (InputStream is = new ByteArrayInputStream(prompt.getBytes(StandardCharsets.UTF_8))) { - XMLInputFactory factory = XMLInputFactory.newInstance(); + XMLInputFactory factory = createXMLInputFactory(); XMLEventReader reader = factory.createXMLEventReader(is); while (reader.hasNext()) { XMLEvent event = reader.nextEvent(); @@ -109,7 +133,7 @@ private static ChatPromptParseVisitor getFunctionDefinitions(String promp // try (InputStream is = new ByteArrayInputStream(prompt.getBytes(StandardCharsets.UTF_8))) { - XMLInputFactory factory = XMLInputFactory.newInstance(); + XMLInputFactory factory = createXMLInputFactory(); XMLEventReader reader = factory.createXMLEventReader(is); FunctionDefinition functionDefinition = null; Map parameters = new HashMap<>(); From 55f5d714e528c9098a6fdc55af9586a754a86cb1 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Thu, 16 Apr 2026 17:21:06 +0000 Subject: [PATCH 259/277] Fix classloading --- .../aiservices/openai/chatcompletion/OpenAIFunction.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java index cf126d09..a5c5c662 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java @@ -229,14 +229,9 @@ private static String getJavaTypeToOpenAiFunctionType(String javaType) { } private static String getObjectSchema(String type, String description) { - String schema = "{ \"type\" : \"object\" }"; - try { - Class clazz = Class.forName(type); - schema = ResponseSchemaGenerator.jacksonGenerator().generateSchema(clazz); - - } catch (ClassNotFoundException | SKException ignored) { + Class clazz = KernelPluginFactory.getTypeForName(type); + String schema = ResponseSchemaGenerator.jacksonGenerator().generateSchema(clazz); - } Map properties = BinaryData.fromString(schema).toObject(Map.class); if (StringUtils.isNotBlank(description)) { properties.put("description", description); From 3128efc1976b6b5ac99210772f81b995818c2d54 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Thu, 16 Apr 2026 17:23:52 +0000 Subject: [PATCH 260/277] Support max_completion_tokens and bump maven versions --- aiservices/google/pom.xml | 2 +- .../HuggingFacePromptExecutionSettings.java | 9 ++- .../chatcompletion/OpenAIChatCompletion.java | 72 ++++++++++--------- api-test/integration-tests/pom.xml | 10 +-- .../tests/ImportingMultiplePluginsTest.java | 1 - data/semantickernel-data-jdbc/pom.xml | 6 +- data/semantickernel-data-oracle/pom.xml | 4 +- .../data/jdbc/oracle/Hotel.java | 13 +--- data/semantickernel-data-postgres/pom.xml | 2 +- data/semantickernel-data-sqlite/pom.xml | 2 +- pom.xml | 33 ++++++--- .../semantickernel-syntax-examples/pom.xml | 8 +-- .../Example62_CustomAIServiceSelector.java | 2 +- .../Example_ChatWithResponseFormat.java | 1 + ...xample_ChatWithResponseFormatToolCall.java | 1 + .../Example59_OpenAIFunctionCalling.java | 2 +- .../booking-agent-m365/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 14 ++-- .../semantickernel-learn-resources/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- .../semantickernel/hooks/KernelHook.java | 1 + .../PromptExecutionSettings.java | 63 ++++++++++++++-- semantickernel-bom/pom.xml | 30 ++++---- semantickernel-experimental/pom.xml | 4 +- 25 files changed, 178 insertions(+), 110 deletions(-) diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index 145ee549..f56bcab7 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -17,7 +17,7 @@ com.google.cloud libraries-bom - 26.49.0 + 26.80.0 pom import diff --git a/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFacePromptExecutionSettings.java b/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFacePromptExecutionSettings.java index adcabe7c..4ee23f26 100644 --- a/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFacePromptExecutionSettings.java +++ b/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFacePromptExecutionSettings.java @@ -47,7 +47,8 @@ public HuggingFacePromptExecutionSettings(PromptExecutionSettings copy) { copy.getUser(), copy.getStopSequences(), copy.getTokenSelectionBiases(), - copy.getResponseFormat() == null ? null : copy.getResponseFormat()); + copy.getResponseFormat() == null ? null : copy.getResponseFormat(), + copy.getMaxCompletionTokens() == null ? null : copy.getMaxCompletionTokens().toString()); this.topK = null; this.repetitionPenalty = null; this.maxTime = null; @@ -101,10 +102,11 @@ public HuggingFacePromptExecutionSettings( @Nullable Boolean details, @Nullable Boolean logProbs, @Nullable Integer topLogProbs, - @Nullable Long seed) { + @Nullable Long seed, + @Nullable Boolean maxCompletionTokens) { super( serviceId, modelId, temperature, topP, presencePenalty, frequencyPenalty, maxTokens, - resultsPerPrompt, bestOf, user, stopSequences, tokenSelectionBiases, responseFormat); + resultsPerPrompt, bestOf, user, stopSequences, tokenSelectionBiases, responseFormat, Boolean.toString(maxCompletionTokens)); this.topK = topK; this.repetitionPenalty = repetitionPenalty; @@ -151,6 +153,7 @@ public static HuggingFacePromptExecutionSettings fromExecutionSettings( null, null, null, + null, null); } diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java index 8256bb00..8f0ebaf2 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java @@ -65,8 +65,8 @@ import com.microsoft.semantickernel.orchestration.ToolCallBehavior; import com.microsoft.semantickernel.orchestration.responseformat.JsonResponseSchema; import com.microsoft.semantickernel.orchestration.responseformat.JsonSchemaResponseFormat; -import com.microsoft.semantickernel.semanticfunctions.KernelFunction; import com.microsoft.semantickernel.semanticfunctions.KernelArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelFunction; import com.microsoft.semantickernel.services.chatcompletion.AuthorRole; import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; import com.microsoft.semantickernel.services.chatcompletion.ChatHistory; @@ -149,7 +149,7 @@ public Mono>> getChatMessageContentsAsync( if (invocationContext != null && invocationContext - .returnMode() == InvocationReturnMode.LAST_MESSAGE_ONLY) { + .returnMode() == InvocationReturnMode.LAST_MESSAGE_ONLY) { chatHistoryResult = new ChatHistory( Collections.singletonList( CollectionUtil.getLastOrNull(chatHistoryResult.getMessages()))); @@ -183,7 +183,7 @@ public Mono>> getChatMessageContentsAsync( if (invocationContext != null && invocationContext - .returnMode() == InvocationReturnMode.LAST_MESSAGE_ONLY) { + .returnMode() == InvocationReturnMode.LAST_MESSAGE_ONLY) { result = new ChatHistory( Collections.singletonList( CollectionUtil.getLastOrNull(result.getMessages()))); @@ -443,31 +443,31 @@ private Mono internalChatMessageContentsAsync( .getOptions(); return Mono.deferContextual(contextView -> { - ChatCompletionSpan span = ChatCompletionSpan.startChatCompletionSpan( - SemanticKernelTelemetry.getTelemetry(invocationContext), - contextView, - getModelId(), - SemanticKernelTelemetry.OPEN_AI_PROVIDER, - options.getMaxTokens(), - options.getTemperature(), - options.getTopP()); - - return getClient() - .getChatCompletionsWithResponse(getDeploymentName(), options, - OpenAIRequestSettings.getRequestOptions()) - .contextWrite(span.getReactorContextModifier()) - .flatMap(completionsResult -> { - if (completionsResult.getStatusCode() >= 400) { - return Mono.error(new AIException(ErrorCodes.SERVICE_ERROR, - "Request failed: " + completionsResult.getStatusCode())); - } + ChatCompletionSpan span = ChatCompletionSpan.startChatCompletionSpan( + SemanticKernelTelemetry.getTelemetry(invocationContext), + contextView, + getModelId(), + SemanticKernelTelemetry.OPEN_AI_PROVIDER, + options.getMaxTokens(), + options.getTemperature(), + options.getTopP()); + + return getClient() + .getChatCompletionsWithResponse(getDeploymentName(), options, + OpenAIRequestSettings.getRequestOptions()) + .contextWrite(span.getReactorContextModifier()) + .flatMap(completionsResult -> { + if (completionsResult.getStatusCode() >= 400) { + return Mono.error(new AIException(ErrorCodes.SERVICE_ERROR, + "Request failed: " + completionsResult.getStatusCode())); + } - return Mono.just(completionsResult.getValue()); - }) - .doOnError(span::endSpanWithError) - .doOnSuccess(span::endSpanWithUsage) - .doOnTerminate(span::close); - }) + return Mono.just(completionsResult.getValue()); + }) + .doOnError(span::endSpanWithError) + .doOnSuccess(span::endSpanWithUsage) + .doOnTerminate(span::close); + }) .flatMap(completions -> { List responseMessages = completions .getChoices() @@ -920,7 +920,8 @@ private static ChatCompletionsOptions getCompletionsOptions( } Map logit = null; - if (promptExecutionSettings.getTokenSelectionBiases() != null) { + if (promptExecutionSettings.getTokenSelectionBiases() != null + && !promptExecutionSettings.getTokenSelectionBiases().isEmpty()) { logit = promptExecutionSettings .getTokenSelectionBiases() .entrySet() @@ -937,12 +938,13 @@ private static ChatCompletionsOptions getCompletionsOptions( .setFrequencyPenalty(promptExecutionSettings.getFrequencyPenalty()) .setPresencePenalty(promptExecutionSettings.getPresencePenalty()) .setMaxTokens(promptExecutionSettings.getMaxTokens()) + .setMaxCompletionTokens(promptExecutionSettings.getMaxCompletionTokens()) .setN(promptExecutionSettings.getResultsPerPrompt()) // Azure OpenAI WithData API does not allow to send empty array of stop sequences // Gives back "Validation error at #/stop/str: Input should be a valid string\nValidation error at #/stop/list[str]: List should have at least 1 item after validation, not 0" .setStop(promptExecutionSettings.getStopSequences() == null || promptExecutionSettings.getStopSequences().isEmpty() ? null - : promptExecutionSettings.getStopSequences()) + : promptExecutionSettings.getStopSequences()) .setUser(promptExecutionSettings.getUser()) .setLogitBias(logit); @@ -1147,7 +1149,7 @@ private static OpenAIToolCallConfig getToolCallBehaviorConfig( toolChoice, toolCallBehavior.isAutoInvokeAllowed() && requestIndex < Math.min(MAXIMUM_INFLIGHT_AUTO_INVOKES, - toolCallBehavior.getMaximumAutoInvokeAttempts()), + toolCallBehavior.getMaximumAutoInvokeAttempts()), null); } @@ -1262,11 +1264,11 @@ private static ChatRequestAssistantMessage formAssistantMessage( String args = arguments != null && !arguments.isEmpty() ? arguments.entrySet().stream() - .map(entry -> String.format("\"%s\": \"%s\"", - StringEscapeUtils.escapeJson(entry.getKey()), - StringEscapeUtils.escapeJson( - entry.getValue().toPromptString()))) - .collect(Collectors.joining(",", "{", "}")) + .map(entry -> String.format("\"%s\": \"%s\"", + StringEscapeUtils.escapeJson(entry.getKey()), + StringEscapeUtils.escapeJson( + entry.getValue().toPromptString()))) + .collect(Collectors.joining(",", "{", "}")) : "{}"; String prefix = ""; diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 256e4cb7..f6256767 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -90,18 +90,18 @@ com.mysql mysql-connector-j - 9.0.0 + 9.6.0 test org.postgresql postgresql - 42.7.3 + 42.7.10 org.xerial sqlite-jdbc - 3.46.1.0 + 3.53.0.0 @@ -147,7 +147,7 @@ org.hsqldb hsqldb - 2.7.3 + 2.7.4 test @@ -162,7 +162,7 @@ org.testcontainers testcontainers-bom - 1.21.4 + 2.0.4 pom import diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ImportingMultiplePluginsTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ImportingMultiplePluginsTest.java index a555937e..e9e5c6d4 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ImportingMultiplePluginsTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/ImportingMultiplePluginsTest.java @@ -4,7 +4,6 @@ import com.microsoft.semantickernel.Kernel; import com.microsoft.semantickernel.plugin.KernelPlugin; import com.microsoft.semantickernel.plugin.KernelPluginFactory; -import org.junit.Ignore; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 077d4da6..883d0643 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -66,17 +66,17 @@ org.postgresql postgresql - 42.7.7 + 42.7.10 org.xerial sqlite-jdbc - 3.47.0.0 + 3.53.0.0 com.oracle.database.jdbc ojdbc11 - 23.7.0.25.01 + 23.26.1.0.0 \ No newline at end of file diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml index e2c93443..98644f9a 100644 --- a/data/semantickernel-data-oracle/pom.xml +++ b/data/semantickernel-data-oracle/pom.xml @@ -61,12 +61,12 @@ com.oracle.database.jdbc ojdbc11 - 23.7.0.25.01 + 23.26.1.0.0 com.oracle.database.jdbc ojdbc-provider-jackson-oson - 1.0.4 + 1.0.6 org.junit.jupiter diff --git a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java index 0f93ff7f..ef6ac824 100644 --- a/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java +++ b/data/semantickernel-data-oracle/src/test/java/com/microsoft/semantickernel/data/jdbc/oracle/Hotel.java @@ -8,12 +8,8 @@ import com.microsoft.semantickernel.data.vectorstorage.annotations.VectorStoreRecordVector; import com.microsoft.semantickernel.data.vectorstorage.definition.DistanceFunction; import com.microsoft.semantickernel.data.vectorstorage.definition.IndexKind; - import java.util.List; -import static com.fasterxml.jackson.annotation.JsonCreator.Mode.DELEGATING; -import static com.fasterxml.jackson.annotation.JsonCreator.Mode.PROPERTIES; - public class Hotel { @VectorStoreRecordKey private final String id; @@ -52,13 +48,8 @@ public class Hotel { @VectorStoreRecordData private double rating; - @JsonCreator(mode = DELEGATING) - public Hotel() { - this(null, null, 0, 0d, null, null, null, null, null, null, 0.0); - } - - @JsonCreator(mode = PROPERTIES) - protected Hotel( + @JsonCreator + public Hotel( @JsonProperty("id") String id, @JsonProperty("name") String name, @JsonProperty("code") int code, diff --git a/data/semantickernel-data-postgres/pom.xml b/data/semantickernel-data-postgres/pom.xml index 8f22241d..ee56c3f2 100644 --- a/data/semantickernel-data-postgres/pom.xml +++ b/data/semantickernel-data-postgres/pom.xml @@ -51,7 +51,7 @@ org.postgresql postgresql - 42.7.7 + 42.7.10 \ No newline at end of file diff --git a/data/semantickernel-data-sqlite/pom.xml b/data/semantickernel-data-sqlite/pom.xml index fc4d8018..f490c247 100644 --- a/data/semantickernel-data-sqlite/pom.xml +++ b/data/semantickernel-data-sqlite/pom.xml @@ -52,7 +52,7 @@ org.xerial sqlite-jdbc - 3.47.0.0 + 3.53.0.0 \ No newline at end of file diff --git a/pom.xml b/pom.xml index 9a52016c..52630ac7 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ 1.0.0-beta.16 - 10.18.2 + 13.4.2 0.10.21 false 2.19.1 @@ -42,8 +42,7 @@ 3.10.1 3.5.0 2.4.0 - - 3.27.0 + 3.28.0 3.8.0 0.16.1 3.1.1 @@ -59,12 +58,11 @@ 2.17.1 5.14.2 0.9.1 - - 7.22.0 + 7.23.0 UTF-8 microsoft/semantic-kernel git@github.com:${project.github.repository}.git - 4.8.6 + 4.9.8 @@ -203,13 +201,32 @@ org.wiremock wiremock - 3.9.2 + 3.13.2 test org.mockito mockito-junit-jupiter - 5.14.2 + 5.23.0 + test + + + + org.testcontainers + junit-jupiter + 1.21.4 + test + + + org.testcontainers + postgresql + 1.21.4 + test + + + org.testcontainers + mysql + 1.21.4 test diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index bd5b298c..60d46fe7 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -29,7 +29,7 @@ io.opentelemetry.instrumentation opentelemetry-reactor-3.1 - 2.9.0-alpha + 2.26.1-alpha com.microsoft.semantic-kernel @@ -140,20 +140,20 @@ org.apache.pdfbox pdfbox - 3.0.3 + 3.0.7 com.google.cloud google-cloud-vertexai - 1.6.0 + 1.52.0 compile com.mysql mysql-connector-j - 9.0.0 + 9.6.0 com.github.victools diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example62_CustomAIServiceSelector.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example62_CustomAIServiceSelector.java index 5e01d13a..c3ebd6a0 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example62_CustomAIServiceSelector.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example62_CustomAIServiceSelector.java @@ -50,7 +50,7 @@ public static void main(String[] args) { var openAIChatCompletion = OpenAIChatCompletion.builder() .withOpenAIAsyncClient(client) .withServiceId("AzureOpenAIChat") - .withModelId("gpt-35-turbo-2") + .withModelId("gpt-35-turbo") .build(); var textGenerationService = TextGenerationService.builder() diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/responseschema/Example_ChatWithResponseFormat.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/responseschema/Example_ChatWithResponseFormat.java index 5d0aad36..8a5d0233 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/responseschema/Example_ChatWithResponseFormat.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/responseschema/Example_ChatWithResponseFormat.java @@ -55,6 +55,7 @@ public static void main(String[] args) throws InterruptedException, JsonProcessi .setResponseFormat(Pet.class) .setName("Pet") .build()) + .withMaxCompletionTokensEnable(true) .build(); FunctionResult response = kernel diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/responseschema/Example_ChatWithResponseFormatToolCall.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/responseschema/Example_ChatWithResponseFormatToolCall.java index cd227ebd..a3126eff 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/responseschema/Example_ChatWithResponseFormatToolCall.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/chatcompletion/responseschema/Example_ChatWithResponseFormatToolCall.java @@ -58,6 +58,7 @@ public static void main(String[] args) throws InterruptedException, JsonProcessi PromptExecutionSettings promptExecutionSettings = PromptExecutionSettings.builder() .withJsonSchemaResponseFormat(Pet.class) + .withMaxCompletionTokensEnable(true) .build(); FunctionResult response = kernel.invokePromptAsync("Get pet with id 1234") diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example59_OpenAIFunctionCalling.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example59_OpenAIFunctionCalling.java index e921bb78..d021caf8 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example59_OpenAIFunctionCalling.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/functions/Example59_OpenAIFunctionCalling.java @@ -39,7 +39,7 @@ public class Example59_OpenAIFunctionCalling { // Only required if AZURE_CLIENT_KEY is set private static final String CLIENT_ENDPOINT = System.getenv("CLIENT_ENDPOINT"); private static final String MODEL_ID = System.getenv() - .getOrDefault("MODEL_ID", "gpt-4o"); + .getOrDefault("MODEL_ID", "gpt-35-turbo"); // Define functions that can be called by the model public static class HelperFunctions { diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index 60bc7ffb..05c97048 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -43,7 +43,7 @@ com.microsoft.graph microsoft-graph - 6.13.0 + 6.62.0 diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index 8db11dcd..0d3ba178 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -31,40 +31,40 @@ org.springframework.boot spring-boot-test - 3.3.2 + 4.1.0-M4 test org.assertj assertj-core - 3.27.7 + 4.0.0-M1 test org.springframework.boot spring-boot-autoconfigure - 3.3.2 + 4.1.0-M4 org.springframework.boot spring-boot - 3.3.11 + 4.1.0-M4 org.springframework spring-test - 6.1.10 + 7.0.6 test com.azure azure-identity - 1.12.2 + 1.18.2 org.junit.jupiter junit-jupiter-api - 5.10.3 + 6.1.0-M1 test diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 8d5130b2..22b76e1b 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -89,7 +89,7 @@ com.mysql mysql-connector-j - 9.0.0 + 9.6.0 compile diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index f0b57642..7f763a06 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -62,7 +62,7 @@ io.swagger.parser.v3 swagger-parser - 2.1.22 + 2.1.40 com.microsoft.semantic-kernel diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index ace57557..2b912eb9 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -42,7 +42,7 @@ io.opentelemetry.instrumentation opentelemetry-reactor-3.1 - 2.9.0-alpha + 2.26.1-alpha com.azure diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/KernelHook.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/KernelHook.java index e4d3f530..586c93c5 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/KernelHook.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/KernelHook.java @@ -104,6 +104,7 @@ static ChatCompletionsOptions cloneOptionsWithMessages( .setFrequencyPenalty(options.getFrequencyPenalty()) .setLogitBias(options.getLogitBias()) .setMaxTokens(options.getMaxTokens()) + .setMaxCompletionTokens(options.getMaxCompletionTokens()) .setModel(options.getModel()) .setStop(options.getStop()) .setTemperature(options.getTemperature()) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/PromptExecutionSettings.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/PromptExecutionSettings.java index 19dfbd0d..9209123c 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/PromptExecutionSettings.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/PromptExecutionSettings.java @@ -82,6 +82,7 @@ public class PromptExecutionSettings { private static final String PRESENCE_PENALTY = "presence_penalty"; private static final String FREQUENCY_PENALTY = "frequency_penalty"; private static final String MAX_TOKENS = "max_tokens"; + private static final String MAX_COMPLETION_TOKENS = "max_completion_tokens"; private static final String BEST_OF = "best_of"; private static final String USER = "user"; private static final String STOP_SEQUENCES = "stop_sequences"; @@ -89,13 +90,19 @@ public class PromptExecutionSettings { private static final String TOKEN_SELECTION_BIASES = "token_selection_biases"; private static final String RESPONSE_FORMAT = "response_format"; + private static final String MAX_COMPLETION_TOKENS_ENABLE = "MAX_COMPLETION_TOKENS_ENABLE"; + private static final String DEFAULT_MAX_COMPLETION_TOKENS_ENABLE = System.getenv( + MAX_COMPLETION_TOKENS_ENABLE); + private final String serviceId; private final String modelId; private final double temperature; private final double topP; private final double presencePenalty; private final double frequencyPenalty; - private final int maxTokens; + private final boolean maxCompletionTokensEnable; + private final Integer maxCompletionTokens; + private final Integer maxTokens; private final int bestOf; private final int resultsPerPrompt; private final String user; @@ -135,14 +142,24 @@ public PromptExecutionSettings( @JsonProperty(USER) String user, @Nullable @JsonProperty(STOP_SEQUENCES) List stopSequences, @Nullable @JsonProperty(TOKEN_SELECTION_BIASES) Map tokenSelectionBiases, - @Nullable @JsonProperty(RESPONSE_FORMAT) ResponseFormat responseFormat) { + @Nullable @JsonProperty(RESPONSE_FORMAT) ResponseFormat responseFormat, + @JsonProperty(value = MAX_COMPLETION_TOKENS_ENABLE, defaultValue = "false") String maxCompletionTokensEnable) { this.serviceId = serviceId != null ? serviceId : DEFAULT_SERVICE_ID; this.modelId = modelId != null ? modelId : ""; this.temperature = clamp(temperature, 0d, 2d, DEFAULT_TEMPERATURE); this.topP = clamp(topP, 0d, 1d, DEFAULT_TOP_P); this.presencePenalty = clamp(presencePenalty, -2d, 2d, DEFAULT_PRESENCE_PENALTY); this.frequencyPenalty = clamp(frequencyPenalty, -2d, 2d, DEFAULT_FREQUENCY_PENALTY); - this.maxTokens = clamp(maxTokens, 1, Integer.MAX_VALUE, DEFAULT_MAX_TOKENS); + + this.maxCompletionTokensEnable = isMaxCompletionTokensEnable(maxCompletionTokensEnable); + + if (this.maxCompletionTokensEnable) { + this.maxCompletionTokens = clamp(maxTokens, 1, Integer.MAX_VALUE, DEFAULT_MAX_TOKENS); + this.maxTokens = null; + } else { + this.maxTokens = clamp(maxTokens, 1, Integer.MAX_VALUE, DEFAULT_MAX_TOKENS); + this.maxCompletionTokens = null; + } this.resultsPerPrompt = clamp(resultsPerPrompt, 1, Integer.MAX_VALUE, DEFAULT_RESULTS_PER_PROMPT); this.bestOf = clamp(bestOf, 1, Integer.MAX_VALUE, DEFAULT_BEST_OF); @@ -161,6 +178,22 @@ public PromptExecutionSettings( } } + private boolean isMaxCompletionTokensEnable(String maxCompletionTokensEnable) { + final boolean maxCompletionTokensEnabled; + if (maxCompletionTokensEnable != null && !maxCompletionTokensEnable.isEmpty()) { + maxCompletionTokensEnabled = Boolean.parseBoolean(maxCompletionTokensEnable); + } else { + if (DEFAULT_MAX_COMPLETION_TOKENS_ENABLE != null + && DEFAULT_MAX_COMPLETION_TOKENS_ENABLE.isEmpty()) { + maxCompletionTokensEnabled = Boolean.parseBoolean( + DEFAULT_MAX_COMPLETION_TOKENS_ENABLE); + } else { + maxCompletionTokensEnabled = false; + } + } + return maxCompletionTokensEnabled; + } + /** * Create a new builder for PromptExecutionSettings. * @@ -257,7 +290,7 @@ public double getFrequencyPenalty() { * @return The maximum number of tokens to generate in the output. */ @JsonProperty(MAX_TOKENS) - public int getMaxTokens() { + public Integer getMaxTokens() { return maxTokens; } @@ -383,6 +416,12 @@ public ResponseFormat getResponseFormat() { return responseFormat; } + + @JsonProperty(MAX_COMPLETION_TOKENS) + public Integer getMaxCompletionTokens() { + return maxCompletionTokens; + } + /** * Builder for PromptExecutionSettings. */ @@ -480,6 +519,18 @@ public Builder withMaxTokens(int maxTokens) { return this; } + /** + * Enables the use of max_completion_tokens config parameter rather than the older + * max_completion + * + * @param enable Whether to enable + * @return This builder + */ + public Builder withMaxCompletionTokensEnable(boolean enable) { + settings.put(MAX_COMPLETION_TOKENS_ENABLE, Boolean.toString(enable)); + return this; + } + /** * Set the number of results to generate for each prompt. The value is clamped to the range * [1, Integer.MAX_VALUE], and the default is 1. @@ -620,7 +671,9 @@ public PromptExecutionSettings build() { (List) settings.getOrDefault(STOP_SEQUENCES, Collections.emptyList()), (Map) settings.getOrDefault(TOKEN_SELECTION_BIASES, Collections.emptyMap()), - (ResponseFormat) settings.getOrDefault(RESPONSE_FORMAT, new TextResponseFormat())); + (ResponseFormat) settings.getOrDefault(RESPONSE_FORMAT, new TextResponseFormat()), + (String) settings.getOrDefault(MAX_COMPLETION_TOKENS_ENABLE, + DEFAULT_MAX_COMPLETION_TOKENS_ENABLE)); } } } diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 303e08b7..47b0a1c0 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -13,7 +13,7 @@ https://www.github.com/microsoft/semantic-kernel - 2.18.6 + 2.21.2 @@ -52,7 +52,7 @@ io.opentelemetry opentelemetry-bom - 1.43.0 + 1.61.0 pom import @@ -150,17 +150,17 @@ com.azure azure-identity - 1.14.0 + 1.18.2 com.azure azure-core - 1.53.0 + 1.57.1 com.azure azure-search-documents - 11.8.0-beta.1 + 11.8.1 com.azure @@ -171,13 +171,13 @@ redis.clients jedis - 5.2.0 + 7.4.1 com.fasterxml.jackson.core jackson-annotations - ${com.fasterxml.jackson.core.version} + 2.21 com.fasterxml.jackson.core @@ -201,7 +201,7 @@ com.github.jknack handlebars - 4.3.1 + 4.5.0 jakarta.inject @@ -211,7 +211,7 @@ org.slf4j slf4j-api - 2.0.16 + 2.0.17 com.google.code.findbugs @@ -230,36 +230,36 @@ com.github.spotbugs spotbugs-annotations - 4.8.6 + 4.9.8 org.apache.commons commons-text - 1.14.0 + 1.15.0 com.google.cloud google-cloud-vertexai - 1.12.0 + 1.52.0 com.github.victools jsonschema-generator - 4.36.0 + 4.38.0 true com.github.victools jsonschema-module-jackson - 4.36.0 + 4.38.0 true io.projectreactor reactor-core - 3.7.8 + 3.8.5 diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 5cdbd534..86c0aee7 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -115,12 +115,12 @@ org.postgresql postgresql - 42.7.7 + 42.7.10 org.xerial sqlite-jdbc - 3.47.0.0 + 3.53.0.0 From bd492a6e880aa07257f6fbc2f5d2d22e96595424 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Thu, 16 Apr 2026 17:26:38 +0000 Subject: [PATCH 261/277] Bump versions --- agents/semantickernel-agents-core/pom.xml | 2 +- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- data/semantickernel-data-azureaisearch/pom.xml | 2 +- data/semantickernel-data-hsqldb/pom.xml | 2 +- data/semantickernel-data-jdbc/pom.xml | 2 +- data/semantickernel-data-mysql/pom.xml | 2 +- data/semantickernel-data-oracle/pom.xml | 2 +- data/semantickernel-data-postgres/pom.xml | 2 +- data/semantickernel-data-redis/pom.xml | 2 +- data/semantickernel-data-sqlite/pom.xml | 2 +- pom.xml | 2 +- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-demos/semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- .../semantickernel-text-splitter-plugin/pom.xml | 2 +- semantickernel-api-ai-services/pom.xml | 2 +- semantickernel-api-builders/pom.xml | 2 +- semantickernel-api-data/pom.xml | 2 +- semantickernel-api-exceptions/pom.xml | 2 +- semantickernel-api-localization/pom.xml | 2 +- semantickernel-api-textembedding-services/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-bom/pom.xml | 2 +- semantickernel-experimental/pom.xml | 2 +- 36 files changed, 36 insertions(+), 36 deletions(-) diff --git a/agents/semantickernel-agents-core/pom.xml b/agents/semantickernel-agents-core/pom.xml index 1270d00d..e237c85a 100644 --- a/agents/semantickernel-agents-core/pom.xml +++ b/agents/semantickernel-agents-core/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../../pom.xml diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index f56bcab7..abe67c86 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index 152aba54..139c98a9 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index 992629a4..96e01cd1 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index f6256767..c919de19 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index 587dfe5b..6b6e2ff6 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../pom.xml diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml index c0f0a8c6..b7b30067 100644 --- a/data/semantickernel-data-azureaisearch/pom.xml +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-hsqldb/pom.xml b/data/semantickernel-data-hsqldb/pom.xml index 1cd33179..5296d116 100644 --- a/data/semantickernel-data-hsqldb/pom.xml +++ b/data/semantickernel-data-hsqldb/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 883d0643..68452053 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-mysql/pom.xml b/data/semantickernel-data-mysql/pom.xml index 3d6d40e6..7826fd50 100644 --- a/data/semantickernel-data-mysql/pom.xml +++ b/data/semantickernel-data-mysql/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml index 98644f9a..f8376c9c 100644 --- a/data/semantickernel-data-oracle/pom.xml +++ b/data/semantickernel-data-oracle/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-postgres/pom.xml b/data/semantickernel-data-postgres/pom.xml index ee56c3f2..5ed076d3 100644 --- a/data/semantickernel-data-postgres/pom.xml +++ b/data/semantickernel-data-postgres/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-redis/pom.xml b/data/semantickernel-data-redis/pom.xml index 885084d3..8dffac89 100644 --- a/data/semantickernel-data-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-sqlite/pom.xml b/data/semantickernel-data-sqlite/pom.xml index f490c247..79a8b15a 100644 --- a/data/semantickernel-data-sqlite/pom.xml +++ b/data/semantickernel-data-sqlite/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../../pom.xml diff --git a/pom.xml b/pom.xml index 52630ac7..cb089c6c 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT pom https://www.github.com/microsoft/semantic-kernel diff --git a/samples/pom.xml b/samples/pom.xml index d2bea638..5dac1dfb 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index 0b51d125..bd7e7358 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 60d46fe7..55b6a06e 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index 05c97048..08b39c25 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index c19477a7..a37b5822 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index 0d3ba178..b8656b4f 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index 079b5af0..974f4b5b 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 22b76e1b..f7ffcb6a 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index 0abacea7..893e5a0d 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index 7f763a06..8785c238 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index 10326e6e..30e912bb 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml index 1bf80430..32eab45e 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../pom.xml diff --git a/semantickernel-api-ai-services/pom.xml b/semantickernel-api-ai-services/pom.xml index 6187b56b..3f3d7f61 100644 --- a/semantickernel-api-ai-services/pom.xml +++ b/semantickernel-api-ai-services/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../pom.xml diff --git a/semantickernel-api-builders/pom.xml b/semantickernel-api-builders/pom.xml index be51b46f..41bcd8d1 100644 --- a/semantickernel-api-builders/pom.xml +++ b/semantickernel-api-builders/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT com.microsoft.semantic-kernel diff --git a/semantickernel-api-data/pom.xml b/semantickernel-api-data/pom.xml index 13bd1993..972317a1 100644 --- a/semantickernel-api-data/pom.xml +++ b/semantickernel-api-data/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../pom.xml diff --git a/semantickernel-api-exceptions/pom.xml b/semantickernel-api-exceptions/pom.xml index b07b001c..a64467ec 100644 --- a/semantickernel-api-exceptions/pom.xml +++ b/semantickernel-api-exceptions/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../pom.xml diff --git a/semantickernel-api-localization/pom.xml b/semantickernel-api-localization/pom.xml index f1c3c246..3ea4a60b 100644 --- a/semantickernel-api-localization/pom.xml +++ b/semantickernel-api-localization/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../pom.xml diff --git a/semantickernel-api-textembedding-services/pom.xml b/semantickernel-api-textembedding-services/pom.xml index eb839135..5b2e5181 100644 --- a/semantickernel-api-textembedding-services/pom.xml +++ b/semantickernel-api-textembedding-services/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index 2b912eb9..ad029049 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 47b0a1c0..cee100ed 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT pom Semantic Kernel Java BOM diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 86c0aee7..61e60f64 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.4-RC3-SNAPSHOT + 1.4.5-SNAPSHOT semantickernel-experimental From 5377046b34c50fa214d7a3dc6c7743696a92cd47 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Thu, 16 Apr 2026 18:28:52 +0000 Subject: [PATCH 262/277] Debump handlebars --- semantickernel-bom/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index cee100ed..9b436175 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -197,11 +197,11 @@ ${com.fasterxml.jackson.core.version} runtime - + com.github.jknack handlebars - 4.5.0 + 4.3.1 jakarta.inject From ca6445440dc24eb4e26eacf1a651c8c9cd672447 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Mon, 20 Apr 2026 16:19:04 +0000 Subject: [PATCH 263/277] Add allow/block list Fix typo Tidy up code Further cleanup --- .../openai/chatcompletion/OpenAIFunction.java | 6 +- .../plugin/KernelPluginFactory.java | 131 ++++++++++++++++-- 2 files changed, 124 insertions(+), 13 deletions(-) diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java index a5c5c662..4ef550c2 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIFunction.java @@ -7,22 +7,20 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.microsoft.semantickernel.exceptions.SKException; import com.microsoft.semantickernel.orchestration.responseformat.ResponseSchemaGenerator; +import com.microsoft.semantickernel.plugin.KernelPluginFactory; import com.microsoft.semantickernel.semanticfunctions.InputVariable; import com.microsoft.semantickernel.semanticfunctions.KernelFunctionMetadata; -import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import org.apache.commons.lang3.StringUtils; class OpenAIFunction { diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/plugin/KernelPluginFactory.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/plugin/KernelPluginFactory.java index afa31cb0..e0bc517b 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/plugin/KernelPluginFactory.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/plugin/KernelPluginFactory.java @@ -27,10 +27,14 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Set; +import java.util.function.Predicate; import java.util.stream.Collectors; import javax.annotation.Nullable; import org.reactivestreams.Publisher; @@ -67,6 +71,12 @@ public class KernelPluginFactory { COMMON_CLASS_NAMES.put("map", HashMap.class); COMMON_CLASS_NAMES.put("set", HashSet.class); + COMMON_CLASS_NAMES.put(Integer.class.getName(), int.class); + COMMON_CLASS_NAMES.put(String.class.getName(), String.class); + COMMON_CLASS_NAMES.put(List.class.getName(), ArrayList.class); + COMMON_CLASS_NAMES.put(Map.class.getName(), HashMap.class); + COMMON_CLASS_NAMES.put(Set.class.getName(), HashSet.class); + BOXED_FROM_PRIMITIVE.put(void.class, Void.class); BOXED_FROM_PRIMITIVE.put(int.class, Integer.class); BOXED_FROM_PRIMITIVE.put(double.class, Double.class); @@ -240,21 +250,16 @@ public static Class getTypeForName(String className) { return clazz; } + if (!checkClassName(className)) { + throw new SKException("Requested type is not allowed: " + className); + } + try { clazz = Thread.currentThread().getContextClassLoader().loadClass(className); } catch (ClassNotFoundException e) { // ignore } - if (clazz == null) { - try { - // Seems that in tests specifically we need to use the class loader of the class itself - clazz = KernelPluginFactory.class.getClassLoader().loadClass(className); - } catch (ClassNotFoundException e) { - // ignore - } - } - if (clazz == null) { throw new SKException("Requested type could not be found: " + className + ". This needs to be a fully qualified class name, e.g. 'java.lang.String'."); @@ -262,6 +267,10 @@ public static Class getTypeForName(String className) { return clazz; } + public static boolean checkClassName(String className) { + return ClassFilter.CLASS_CHECKER.test(className); + } + /** * Creates a plugin from the provided name and function collection. * @@ -429,6 +438,7 @@ private static KernelFunction getKernelFunction( /** * Imports a plugin from a resource directory on the filesystem. + * * @param parentDirectory The parent directory containing the plugin directories. * @param pluginDirectoryName The name of the plugin directory. * @param functionName The name of the function to import. @@ -552,4 +562,107 @@ private static PromptTemplateConfig getPromptTemplateConfig( return null; } } + + // Filters allowed classes that can be used as types in plugins + public static class ClassFilter { + + // Selects which filter type to use, allow list or ban list + public static final String CLASS_BLOCK_TYPE_PROPERTY_NAME = "semantic-kernel.class-block-type"; + public static final String CLASS_BLOCK_LIST_PROPERTY_NAME = "semantic-kernel.class-block-list"; + public static final String CLASS_ALLOW_LIST_PROPERTY_NAME = "semantic-kernel.class-allow-list"; + + // allow nothing by default (other than java primitives and collections) + private static final List CLASS_ALLOW_LIST; + private static final List CLASS_ALLOW_LIST_DEFAULT = Collections.emptyList(); + + // block Java classes by default (other than java primitives and collections) + private static final List CLASS_BLOCK_LIST; + private static final List CLASS_BLOCK_LIST_DEFAULT = Arrays.asList( + "java\\..*", + "com\\.sun\\..*", + "javax\\..*", + "jdk\\..*", + "org\\.xml\\..*", + "org\\.w3c\\..*" + ); + + static Predicate CLASS_CHECKER; + + private enum BlockType { + BLOCK, + ALLOW + } + + static { + // Default to blocking type + String classFilterType = System.getProperty(CLASS_BLOCK_TYPE_PROPERTY_NAME, + BlockType.BLOCK.name()); + CLASS_BLOCK_LIST = getList(CLASS_BLOCK_LIST_PROPERTY_NAME, CLASS_BLOCK_LIST_DEFAULT); + CLASS_ALLOW_LIST = getList(CLASS_ALLOW_LIST_PROPERTY_NAME, CLASS_ALLOW_LIST_DEFAULT); + + BlockType type; + + try { + type = BlockType.valueOf(classFilterType.toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + type = BlockType.BLOCK; + } + + switch (type) { + case ALLOW: + CLASS_CHECKER = ClassFilter::evaluateAllow; + break; + case BLOCK: + default: + CLASS_CHECKER = ClassFilter::evaluateBlock; + break; + } + } + + private static List getList(String propertyName, List defaultList) { + String blockList = System.getProperty(propertyName); + + if (blockList != null) { + return Arrays.asList(blockList.split(",")); + } else { + return defaultList; + } + } + + // Block classes/packages classes (other than common Java primitives and collections) + private static boolean evaluateBlock(String className) { + if (className == null || className.isEmpty()) { + return false; + } + + for (String ban : CLASS_BLOCK_LIST) { + if (className.matches(ban)) { + LOGGER.warn( + "Skipping class not allowed by class block list {}, if you wish to unblock this class update the property: {}", + className, CLASS_BLOCK_LIST_PROPERTY_NAME); + return false; + } + } + + return true; + } + + // Only allow explicitly allowed classes/packages (other than common Java primitives and collections) + private static boolean evaluateAllow(String className) { + if (className == null || className.isEmpty()) { + return false; + } + + for (String allow : CLASS_ALLOW_LIST) { + if (className.matches(allow)) { + return true; + } + } + + LOGGER.warn( + "Skipping class not allowed by class allow list {}, if you wish to allow this class update the property: {}", + className, CLASS_ALLOW_LIST_DEFAULT); + return false; + } + } } From f879a77131d0c3df48703c3ab71ae0587e7b5349 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Wed, 22 Apr 2026 16:53:55 +0000 Subject: [PATCH 264/277] Fix null check --- .../orchestration/PromptExecutionSettings.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/PromptExecutionSettings.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/PromptExecutionSettings.java index 9209123c..bfa2bb66 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/PromptExecutionSettings.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/PromptExecutionSettings.java @@ -384,7 +384,10 @@ public boolean equals(Object obj) { if (Double.compare(frequencyPenalty, other.frequencyPenalty) != 0) { return false; } - if (maxTokens != other.maxTokens) { + if (!Objects.equals(maxTokens, other.maxTokens)) { + return false; + } + if (!Objects.equals(maxCompletionTokens, other.maxCompletionTokens)) { return false; } if (bestOf != other.bestOf) { From 021faabd01f36887a6a9c11e0cdb8089849b6743 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Wed, 22 Apr 2026 17:21:22 +0000 Subject: [PATCH 265/277] Fix spotbugs issues --- .../services/HuggingFacePromptExecutionSettings.java | 5 +++-- .../jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java | 2 -- .../data/jdbc/JDBCVectorStoreQueryProvider.java | 6 ++---- .../data/jdbc/JDBCVectorStoreRecordCollection.java | 1 - .../data/jdbc/mysql/MySQLVectorStoreQueryProvider.java | 6 ++---- .../postgres/PostgreSQLVectorStoreQueryProvider.java | 10 ++++------ .../jdbc/sqlite/SQLiteVectorStoreQueryProvider.java | 6 ++---- .../contextvariables/ContextVariableTypeConverter.java | 2 -- ...CollectionVariableContextVariableTypeConverter.java | 3 --- .../semantickernel/hooks/FunctionInvokedEvent.java | 3 +-- .../semantickernel/hooks/PreToolCallEvent.java | 3 +-- .../semantickernel/hooks/PromptRenderedEvent.java | 1 - .../orchestration/FunctionInvocation.java | 5 +---- .../handlebars/HandlebarsPromptTemplate.java | 4 +--- 14 files changed, 17 insertions(+), 40 deletions(-) diff --git a/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFacePromptExecutionSettings.java b/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFacePromptExecutionSettings.java index 4ee23f26..ac4f4c09 100644 --- a/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFacePromptExecutionSettings.java +++ b/aiservices/huggingface/src/main/java/com/microsoft/semantickernel/aiservices/huggingface/services/HuggingFacePromptExecutionSettings.java @@ -103,10 +103,11 @@ public HuggingFacePromptExecutionSettings( @Nullable Boolean logProbs, @Nullable Integer topLogProbs, @Nullable Long seed, - @Nullable Boolean maxCompletionTokens) { + @Nullable Boolean maxCompletionTokensEnable) { super( serviceId, modelId, temperature, topP, presencePenalty, frequencyPenalty, maxTokens, - resultsPerPrompt, bestOf, user, stopSequences, tokenSelectionBiases, responseFormat, Boolean.toString(maxCompletionTokens)); + resultsPerPrompt, bestOf, user, stopSequences, tokenSelectionBiases, responseFormat, + Boolean.toString(Boolean.TRUE.equals(maxCompletionTokensEnable))); this.topK = topK; this.repetitionPenalty = repetitionPenalty; diff --git a/data/semantickernel-data-hsqldb/src/main/java/com/microsoft/semantickernel/data/jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java b/data/semantickernel-data-hsqldb/src/main/java/com/microsoft/semantickernel/data/jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java index 48431316..e105a20d 100644 --- a/data/semantickernel-data-hsqldb/src/main/java/com/microsoft/semantickernel/data/jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java +++ b/data/semantickernel-data-hsqldb/src/main/java/com/microsoft/semantickernel/data/jdbc/hsqldb/HSQLDBVectorStoreQueryProvider.java @@ -12,7 +12,6 @@ import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; - import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -32,7 +31,6 @@ public class HSQLDBVectorStoreQueryProvider extends JDBCVectorStoreQueryProvider private final ObjectMapper objectMapper; - @SuppressFBWarnings("EI_EXPOSE_REP2") private HSQLDBVectorStoreQueryProvider( DataSource dataSource, String collectionsTable, diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java index 2fcc2d5b..3441ccfe 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -29,8 +29,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; @@ -59,7 +57,6 @@ public class JDBCVectorStoreQueryProvider private final Object dbCreationLock = new Object(); - @SuppressFBWarnings("EI_EXPOSE_REP2") // DataSource is not exposed protected JDBCVectorStoreQueryProvider( @Nonnull DataSource dataSource, @Nonnull String collectionsTable, @@ -102,8 +99,9 @@ protected JDBCVectorStoreQueryProvider( * @param supportedDataTypes the supported data types * @param supportedVectorTypes the supported vector types */ + @SuppressFBWarnings("EI_EXPOSE_REP2") public JDBCVectorStoreQueryProvider( - @SuppressFBWarnings("EI_EXPOSE_REP2") @Nonnull DataSource dataSource, + @Nonnull DataSource dataSource, @Nonnull String collectionsTable, @Nonnull String prefixForCollectionTables, @Nonnull Map, String> supportedKeyTypes, diff --git a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java index 1d6b3e09..fc421c2b 100644 --- a/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/data/semantickernel-data-jdbc/src/main/java/com/microsoft/semantickernel/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -46,7 +46,6 @@ public class JDBCVectorStoreRecordCollection * @param collectionName the name of the collection * @param options the options */ - @SuppressFBWarnings("EI_EXPOSE_REP2") // DataSource is not exposed public JDBCVectorStoreRecordCollection( @Nonnull DataSource dataSource, @Nonnull String collectionName, diff --git a/data/semantickernel-data-mysql/src/main/java/com/microsoft/semantickernel/data/jdbc/mysql/MySQLVectorStoreQueryProvider.java b/data/semantickernel-data-mysql/src/main/java/com/microsoft/semantickernel/data/jdbc/mysql/MySQLVectorStoreQueryProvider.java index feb6dc4f..1ff0f706 100644 --- a/data/semantickernel-data-mysql/src/main/java/com/microsoft/semantickernel/data/jdbc/mysql/MySQLVectorStoreQueryProvider.java +++ b/data/semantickernel-data-mysql/src/main/java/com/microsoft/semantickernel/data/jdbc/mysql/MySQLVectorStoreQueryProvider.java @@ -13,14 +13,13 @@ import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; - -import javax.annotation.Nonnull; -import javax.sql.DataSource; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.List; import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.sql.DataSource; /** * The MySQL vector store query provider. @@ -32,7 +31,6 @@ public class MySQLVectorStoreQueryProvider extends private final ObjectMapper objectMapper; - @SuppressFBWarnings("EI_EXPOSE_REP2") private MySQLVectorStoreQueryProvider( @Nonnull DataSource dataSource, @Nonnull String collectionsTable, diff --git a/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java b/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java index 1f3273eb..e65bb9eb 100644 --- a/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/data/semantickernel-data-postgres/src/main/java/com/microsoft/semantickernel/data/jdbc/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -4,10 +4,10 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreQueryProvider; -import com.microsoft.semantickernel.data.jdbc.SQLVectorStoreQueryProvider; import com.microsoft.semantickernel.data.filter.AnyTagEqualToFilterClause; import com.microsoft.semantickernel.data.filter.EqualToFilterClause; +import com.microsoft.semantickernel.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.jdbc.SQLVectorStoreQueryProvider; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchFilter; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult; import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResults; @@ -22,9 +22,6 @@ import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; - -import javax.annotation.Nonnull; -import javax.sql.DataSource; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -38,6 +35,8 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.sql.DataSource; /** * The MySQL vector store query provider. @@ -50,7 +49,6 @@ public class PostgreSQLVectorStoreQueryProvider extends private final String prefixForCollectionTables; private final ObjectMapper objectMapper; - @SuppressFBWarnings("EI_EXPOSE_REP2") private PostgreSQLVectorStoreQueryProvider( @Nonnull DataSource dataSource, @Nonnull String collectionsTable, diff --git a/data/semantickernel-data-sqlite/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java b/data/semantickernel-data-sqlite/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java index 57de1225..2d88702e 100644 --- a/data/semantickernel-data-sqlite/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java +++ b/data/semantickernel-data-sqlite/src/main/java/com/microsoft/semantickernel/data/jdbc/sqlite/SQLiteVectorStoreQueryProvider.java @@ -13,13 +13,12 @@ import com.microsoft.semantickernel.data.vectorstorage.options.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; - -import javax.annotation.Nonnull; -import javax.sql.DataSource; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.List; +import javax.annotation.Nonnull; +import javax.sql.DataSource; /** * A query provider for a vector store in SQLite. @@ -30,7 +29,6 @@ public class SQLiteVectorStoreQueryProvider extends private final DataSource dataSource; private final ObjectMapper objectMapper; - @SuppressFBWarnings("EI_EXPOSE_REP2") private SQLiteVectorStoreQueryProvider( @Nonnull DataSource dataSource, @Nonnull String collectionsTable, diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypeConverter.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypeConverter.java index 22d53ec0..3082ecf8 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypeConverter.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypeConverter.java @@ -3,7 +3,6 @@ import com.microsoft.semantickernel.exceptions.SKException; import com.microsoft.semantickernel.localization.SemanticKernelResources; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -341,7 +340,6 @@ public static class Builder { * * @param clazz the class of the type */ - @SuppressFBWarnings("CT_CONSTRUCTOR_THROW") public Builder(Class clazz) { this.clazz = clazz; fromObject = x -> ContextVariableTypes.convert(x, clazz); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/CollectionVariableContextVariableTypeConverter.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/CollectionVariableContextVariableTypeConverter.java index d7bf72e0..5b266197 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/CollectionVariableContextVariableTypeConverter.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/CollectionVariableContextVariableTypeConverter.java @@ -7,7 +7,6 @@ import com.microsoft.semantickernel.contextvariables.ContextVariableType; import com.microsoft.semantickernel.contextvariables.ContextVariableTypeConverter; import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.Collection; import java.util.stream.Collectors; @@ -24,7 +23,6 @@ public class CollectionVariableContextVariableTypeConverter extends * Creates a new instance of the {@link CollectionVariableContextVariableTypeConverter} class. * @param delimiter The delimiter to use joining elements of the collection. */ - @SuppressFBWarnings("CT_CONSTRUCTOR_THROW") public CollectionVariableContextVariableTypeConverter(String delimiter) { super( Collection.class, @@ -38,7 +36,6 @@ public CollectionVariableContextVariableTypeConverter(String delimiter) { /** * Creates a new instance of the {@link CollectionVariableContextVariableTypeConverter} class. */ - @SuppressFBWarnings("CT_CONSTRUCTOR_THROW") public CollectionVariableContextVariableTypeConverter() { this(","); } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/FunctionInvokedEvent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/FunctionInvokedEvent.java index 7a39cdfe..715e7118 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/FunctionInvokedEvent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/FunctionInvokedEvent.java @@ -2,8 +2,8 @@ package com.microsoft.semantickernel.hooks; import com.microsoft.semantickernel.orchestration.FunctionResult; -import com.microsoft.semantickernel.semanticfunctions.KernelFunction; import com.microsoft.semantickernel.semanticfunctions.KernelArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelFunction; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.annotation.Nullable; @@ -60,7 +60,6 @@ public KernelArguments getArguments() { * * @return the result */ - @SuppressFBWarnings("EI_EXPOSE_REP") public FunctionResult getResult() { return result; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PreToolCallEvent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PreToolCallEvent.java index 42430f08..e5850228 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PreToolCallEvent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PreToolCallEvent.java @@ -2,8 +2,8 @@ package com.microsoft.semantickernel.hooks; import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; -import com.microsoft.semantickernel.semanticfunctions.KernelFunction; import com.microsoft.semantickernel.semanticfunctions.KernelArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelFunction; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.annotation.Nullable; @@ -52,7 +52,6 @@ public KernelArguments getArguments() { * Get the tool call function. * @return The tool call function. */ - @SuppressFBWarnings("EI_EXPOSE_REP2") public KernelFunction getFunction() { return function; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PromptRenderedEvent.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PromptRenderedEvent.java index 348d3bf1..fbd8cb80 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PromptRenderedEvent.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/PromptRenderedEvent.java @@ -55,7 +55,6 @@ public KernelArguments getArguments() { * * @return the prompt */ - @SuppressFBWarnings("EI_EXPOSE_REP") public String getPrompt() { return prompt; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java index 9b8a518c..8641f201 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java @@ -14,9 +14,8 @@ import com.microsoft.semantickernel.hooks.KernelHooks.UnmodifiableKernelHooks; import com.microsoft.semantickernel.implementation.telemetry.SemanticKernelTelemetry; import com.microsoft.semantickernel.localization.SemanticKernelResources; -import com.microsoft.semantickernel.semanticfunctions.KernelFunction; import com.microsoft.semantickernel.semanticfunctions.KernelArguments; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import com.microsoft.semantickernel.semanticfunctions.KernelFunction; import java.util.NoSuchElementException; import java.util.function.BiConsumer; import javax.annotation.Nullable; @@ -63,7 +62,6 @@ public class FunctionInvocation extends Mono> { * @param kernel The kernel to invoke the function on. * @param function The function to invoke. */ - @SuppressFBWarnings("EI_EXPOSE_REP2") public FunctionInvocation( Kernel kernel, KernelFunction function) { @@ -80,7 +78,6 @@ public FunctionInvocation( * @param function The function to invoke. * @param resultType The type of the result of the function invocation. */ - @SuppressFBWarnings("EI_EXPOSE_REP2") public FunctionInvocation( Kernel kernel, KernelFunction function, diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplate.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplate.java index 460e4815..bd8df138 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplate.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplate.java @@ -18,13 +18,12 @@ import com.microsoft.semantickernel.orchestration.InvocationContext; import com.microsoft.semantickernel.orchestration.ToolCallBehavior; import com.microsoft.semantickernel.plugin.KernelPlugin; -import com.microsoft.semantickernel.semanticfunctions.KernelFunction; import com.microsoft.semantickernel.semanticfunctions.KernelArguments; +import com.microsoft.semantickernel.semanticfunctions.KernelFunction; import com.microsoft.semantickernel.semanticfunctions.PromptTemplate; import com.microsoft.semantickernel.semanticfunctions.PromptTemplateConfig; import com.microsoft.semantickernel.semanticfunctions.PromptTemplateOption; import com.microsoft.semantickernel.services.chatcompletion.ChatMessageContent; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -192,7 +191,6 @@ private class HandleBarsPromptTemplateHandler { private final String template; private final Handlebars handlebars; - @SuppressFBWarnings("CT_CONSTRUCTOR_THROW") // Think this is a false positive public HandleBarsPromptTemplateHandler( Kernel kernel, String template, From 70a199818a60713cc834e451e4f80f6e1b9297d4 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Wed, 22 Apr 2026 18:51:08 +0000 Subject: [PATCH 266/277] Allow disabling filter --- .../Example69_MutableKernelPlugin.java | 2 ++ .../plugin/KernelPluginFactory.java | 25 ++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example69_MutableKernelPlugin.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example69_MutableKernelPlugin.java index 23bd3778..591bab7e 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example69_MutableKernelPlugin.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/Example69_MutableKernelPlugin.java @@ -3,6 +3,7 @@ import com.microsoft.semantickernel.Kernel; import com.microsoft.semantickernel.plugin.KernelPlugin; +import com.microsoft.semantickernel.plugin.KernelPluginFactory; import com.microsoft.semantickernel.semanticfunctions.KernelFunction; import com.microsoft.semantickernel.semanticfunctions.annotations.DefineKernelFunction; @@ -16,6 +17,7 @@ public class Example69_MutableKernelPlugin { */ public static void main(String[] args) throws NoSuchMethodException { System.out.println("======== Example69_MutableKernelPlugin ========"); + KernelPluginFactory.setTypeFilterEnable(false); KernelPlugin plugin = new KernelPlugin("Plugin", "Mutable plugin", null); plugin.addFunction(KernelFunction.createFromMethod( diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/plugin/KernelPluginFactory.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/plugin/KernelPluginFactory.java index e0bc517b..fa31819f 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/plugin/KernelPluginFactory.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/plugin/KernelPluginFactory.java @@ -25,6 +25,7 @@ import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; +import java.time.temporal.Temporal; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -54,7 +55,13 @@ public class KernelPluginFactory { private static final CaseInsensitiveMap> COMMON_CLASS_NAMES = new CaseInsensitiveMap<>(); private static final Map, Class> BOXED_FROM_PRIMITIVE = new HashMap<>(); + public static final String CLASS_FILTER_ENABLE_PROPERTY = "semantic-kernel.class-filter-enable"; + private static Boolean CLASS_FILTER_ENABLE; + static { + CLASS_FILTER_ENABLE = Boolean.parseBoolean( + System.getProperty(CLASS_FILTER_ENABLE_PROPERTY, "true")); + PRIMITIVE_CLASS_NAMES.put("void", void.class); PRIMITIVE_CLASS_NAMES.put("int", int.class); PRIMITIVE_CLASS_NAMES.put("double", double.class); @@ -76,6 +83,9 @@ public class KernelPluginFactory { COMMON_CLASS_NAMES.put(List.class.getName(), ArrayList.class); COMMON_CLASS_NAMES.put(Map.class.getName(), HashMap.class); COMMON_CLASS_NAMES.put(Set.class.getName(), HashSet.class); + COMMON_CLASS_NAMES.put(Temporal.class.getName(), Temporal.class); + COMMON_CLASS_NAMES.put(java.time.OffsetDateTime.class.getName(), java.time.OffsetDateTime.class); + COMMON_CLASS_NAMES.put(java.time.ZonedDateTime.class.getName(), java.time.ZonedDateTime.class); BOXED_FROM_PRIMITIVE.put(void.class, Void.class); BOXED_FROM_PRIMITIVE.put(int.class, Integer.class); @@ -89,6 +99,10 @@ public class KernelPluginFactory { } + public static void setTypeFilterEnable(boolean enable) { + KernelPluginFactory.CLASS_FILTER_ENABLE = enable; + } + /** * Creates a plugin that wraps the specified target object. Methods decorated with * {@code {@literal @}DefineSKFunction} will be included in the plugin. @@ -268,6 +282,9 @@ public static Class getTypeForName(String className) { } public static boolean checkClassName(String className) { + if (CLASS_FILTER_ENABLE == false) { + return true; + } return ClassFilter.CLASS_CHECKER.test(className); } @@ -638,8 +655,8 @@ private static boolean evaluateBlock(String className) { for (String ban : CLASS_BLOCK_LIST) { if (className.matches(ban)) { LOGGER.warn( - "Skipping class not allowed by class block list {}, if you wish to unblock this class update the property: {}", - className, CLASS_BLOCK_LIST_PROPERTY_NAME); + "Skipping class not allowed by class block list {}, if you wish to unblock this class update the property: {}. Filtering can also be controlled with {} and KernelPluginFactory.setTypeFilterEnable", + className, CLASS_BLOCK_LIST_PROPERTY_NAME, CLASS_FILTER_ENABLE_PROPERTY); return false; } } @@ -660,8 +677,8 @@ private static boolean evaluateAllow(String className) { } LOGGER.warn( - "Skipping class not allowed by class allow list {}, if you wish to allow this class update the property: {}", - className, CLASS_ALLOW_LIST_DEFAULT); + "Skipping class not allowed by class allow list {}, if you wish to allow this class update the property: {}. Filtering can also be controlled with {} and KernelPluginFactory.setTypeFilterEnable", + className, CLASS_ALLOW_LIST_DEFAULT, CLASS_FILTER_ENABLE_PROPERTY); return false; } } From cec7e78f495060011e57a4b739c7d9bcadf50470 Mon Sep 17 00:00:00 2001 From: GitHub Date: Wed, 22 Apr 2026 20:40:14 +0000 Subject: [PATCH 267/277] [maven-release-plugin] prepare release java-1.5.0 --- agents/semantickernel-agents-core/pom.xml | 2 +- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- data/semantickernel-data-azureaisearch/pom.xml | 5 ++--- data/semantickernel-data-hsqldb/pom.xml | 2 +- data/semantickernel-data-jdbc/pom.xml | 2 +- data/semantickernel-data-mysql/pom.xml | 2 +- data/semantickernel-data-oracle/pom.xml | 2 +- data/semantickernel-data-postgres/pom.xml | 2 +- data/semantickernel-data-redis/pom.xml | 2 +- data/semantickernel-data-sqlite/pom.xml | 2 +- pom.xml | 4 ++-- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- .../semantickernel-text-splitter-plugin/pom.xml | 2 +- semantickernel-api-ai-services/pom.xml | 2 +- semantickernel-api-builders/pom.xml | 2 +- semantickernel-api-data/pom.xml | 2 +- semantickernel-api-exceptions/pom.xml | 2 +- semantickernel-api-localization/pom.xml | 2 +- semantickernel-api-textembedding-services/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-bom/pom.xml | 4 ++-- semantickernel-experimental/pom.xml | 2 +- 36 files changed, 39 insertions(+), 40 deletions(-) diff --git a/agents/semantickernel-agents-core/pom.xml b/agents/semantickernel-agents-core/pom.xml index e237c85a..d23bd406 100644 --- a/agents/semantickernel-agents-core/pom.xml +++ b/agents/semantickernel-agents-core/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 ../../pom.xml diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index abe67c86..59521e77 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index 139c98a9..bcfd47ad 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index 96e01cd1..51a0a4e6 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index c919de19..4542a075 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.4.5-SNAPSHOT + 1.5.0 ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index 6b6e2ff6..9bbf5c59 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 ../pom.xml diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml index b7b30067..e3a48b25 100644 --- a/data/semantickernel-data-azureaisearch/pom.xml +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -1,11 +1,10 @@ - + 4.0.0 com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 ../../pom.xml diff --git a/data/semantickernel-data-hsqldb/pom.xml b/data/semantickernel-data-hsqldb/pom.xml index 5296d116..da17a5c0 100644 --- a/data/semantickernel-data-hsqldb/pom.xml +++ b/data/semantickernel-data-hsqldb/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 ../../pom.xml diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 68452053..098cc08c 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 ../../pom.xml diff --git a/data/semantickernel-data-mysql/pom.xml b/data/semantickernel-data-mysql/pom.xml index 7826fd50..adbd51b1 100644 --- a/data/semantickernel-data-mysql/pom.xml +++ b/data/semantickernel-data-mysql/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 ../../pom.xml diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml index f8376c9c..40a760ab 100644 --- a/data/semantickernel-data-oracle/pom.xml +++ b/data/semantickernel-data-oracle/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 ../../pom.xml diff --git a/data/semantickernel-data-postgres/pom.xml b/data/semantickernel-data-postgres/pom.xml index 5ed076d3..5e6ca4dc 100644 --- a/data/semantickernel-data-postgres/pom.xml +++ b/data/semantickernel-data-postgres/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 ../../pom.xml diff --git a/data/semantickernel-data-redis/pom.xml b/data/semantickernel-data-redis/pom.xml index 8dffac89..5c8a79db 100644 --- a/data/semantickernel-data-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 ../../pom.xml diff --git a/data/semantickernel-data-sqlite/pom.xml b/data/semantickernel-data-sqlite/pom.xml index 79a8b15a..12d6c008 100644 --- a/data/semantickernel-data-sqlite/pom.xml +++ b/data/semantickernel-data-sqlite/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 ../../pom.xml diff --git a/pom.xml b/pom.xml index 27018142..a7c3832e 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 pom https://www.github.com/microsoft/semantic-kernel @@ -947,6 +947,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - HEAD + java-1.5.0 diff --git a/samples/pom.xml b/samples/pom.xml index 5dac1dfb..f4122eec 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index bd7e7358..ff5b86e6 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.5-SNAPSHOT + 1.5.0 ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 55b6a06e..c62276fe 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.4.5-SNAPSHOT + 1.5.0 ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index 08b39c25..b219fa33 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.5-SNAPSHOT + 1.5.0 ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index a37b5822..86ccc67d 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.5-SNAPSHOT + 1.5.0 ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index b8656b4f..37026b27 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.5-SNAPSHOT + 1.5.0 ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index 974f4b5b..ead8e792 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.4.5-SNAPSHOT + 1.5.0 ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index f7ffcb6a..ecc8138e 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.5-SNAPSHOT + 1.5.0 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index 893e5a0d..3cfedcb7 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.4.5-SNAPSHOT + 1.5.0 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index 8785c238..b3664ab6 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.5-SNAPSHOT + 1.5.0 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index 30e912bb..6edcde9f 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.5-SNAPSHOT + 1.5.0 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml index 32eab45e..ba34af35 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.4.5-SNAPSHOT + 1.5.0 ../pom.xml diff --git a/semantickernel-api-ai-services/pom.xml b/semantickernel-api-ai-services/pom.xml index 3f3d7f61..1412e3cb 100644 --- a/semantickernel-api-ai-services/pom.xml +++ b/semantickernel-api-ai-services/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 ../pom.xml diff --git a/semantickernel-api-builders/pom.xml b/semantickernel-api-builders/pom.xml index 41bcd8d1..ffd6774d 100644 --- a/semantickernel-api-builders/pom.xml +++ b/semantickernel-api-builders/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 com.microsoft.semantic-kernel diff --git a/semantickernel-api-data/pom.xml b/semantickernel-api-data/pom.xml index 972317a1..15dc8210 100644 --- a/semantickernel-api-data/pom.xml +++ b/semantickernel-api-data/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 ../pom.xml diff --git a/semantickernel-api-exceptions/pom.xml b/semantickernel-api-exceptions/pom.xml index a64467ec..a5ee92a0 100644 --- a/semantickernel-api-exceptions/pom.xml +++ b/semantickernel-api-exceptions/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 ../pom.xml diff --git a/semantickernel-api-localization/pom.xml b/semantickernel-api-localization/pom.xml index 3ea4a60b..82becc44 100644 --- a/semantickernel-api-localization/pom.xml +++ b/semantickernel-api-localization/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 ../pom.xml diff --git a/semantickernel-api-textembedding-services/pom.xml b/semantickernel-api-textembedding-services/pom.xml index 5b2e5181..dfed9b1e 100644 --- a/semantickernel-api-textembedding-services/pom.xml +++ b/semantickernel-api-textembedding-services/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index ad029049..7f25cdc0 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 9b436175..ed3c8707 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.4.5-SNAPSHOT + 1.5.0 pom Semantic Kernel Java BOM @@ -326,6 +326,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - HEAD + java-1.5.0 diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 61e60f64..7dbcca0f 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.4.5-SNAPSHOT + 1.5.0 semantickernel-experimental From c62ca9ef6b1e048326cb858577fd94ed6445dba6 Mon Sep 17 00:00:00 2001 From: GitHub Date: Wed, 22 Apr 2026 20:40:15 +0000 Subject: [PATCH 268/277] [maven-release-plugin] prepare for next development iteration --- agents/semantickernel-agents-core/pom.xml | 2 +- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- data/semantickernel-data-azureaisearch/pom.xml | 2 +- data/semantickernel-data-hsqldb/pom.xml | 2 +- data/semantickernel-data-jdbc/pom.xml | 2 +- data/semantickernel-data-mysql/pom.xml | 2 +- data/semantickernel-data-oracle/pom.xml | 2 +- data/semantickernel-data-postgres/pom.xml | 2 +- data/semantickernel-data-redis/pom.xml | 2 +- data/semantickernel-data-sqlite/pom.xml | 2 +- pom.xml | 4 ++-- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- .../semantickernel-text-splitter-plugin/pom.xml | 2 +- semantickernel-api-ai-services/pom.xml | 2 +- semantickernel-api-builders/pom.xml | 2 +- semantickernel-api-data/pom.xml | 2 +- semantickernel-api-exceptions/pom.xml | 2 +- semantickernel-api-localization/pom.xml | 2 +- semantickernel-api-textembedding-services/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-bom/pom.xml | 4 ++-- semantickernel-experimental/pom.xml | 2 +- 36 files changed, 38 insertions(+), 38 deletions(-) diff --git a/agents/semantickernel-agents-core/pom.xml b/agents/semantickernel-agents-core/pom.xml index d23bd406..d5376209 100644 --- a/agents/semantickernel-agents-core/pom.xml +++ b/agents/semantickernel-agents-core/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT ../../pom.xml diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index 59521e77..e136a3eb 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index bcfd47ad..82f5cd52 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index 51a0a4e6..4b46bdd4 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 4542a075..9b5c3883 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.5.0 + 1.5.1-SNAPSHOT ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index 9bbf5c59..7b023c30 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT ../pom.xml diff --git a/data/semantickernel-data-azureaisearch/pom.xml b/data/semantickernel-data-azureaisearch/pom.xml index e3a48b25..714c9948 100644 --- a/data/semantickernel-data-azureaisearch/pom.xml +++ b/data/semantickernel-data-azureaisearch/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-hsqldb/pom.xml b/data/semantickernel-data-hsqldb/pom.xml index da17a5c0..bb06dfe4 100644 --- a/data/semantickernel-data-hsqldb/pom.xml +++ b/data/semantickernel-data-hsqldb/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 098cc08c..74de0e2c 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-mysql/pom.xml b/data/semantickernel-data-mysql/pom.xml index adbd51b1..2ad3ec2d 100644 --- a/data/semantickernel-data-mysql/pom.xml +++ b/data/semantickernel-data-mysql/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml index 40a760ab..de631318 100644 --- a/data/semantickernel-data-oracle/pom.xml +++ b/data/semantickernel-data-oracle/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-postgres/pom.xml b/data/semantickernel-data-postgres/pom.xml index 5e6ca4dc..9949118d 100644 --- a/data/semantickernel-data-postgres/pom.xml +++ b/data/semantickernel-data-postgres/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-redis/pom.xml b/data/semantickernel-data-redis/pom.xml index 5c8a79db..db151484 100644 --- a/data/semantickernel-data-redis/pom.xml +++ b/data/semantickernel-data-redis/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT ../../pom.xml diff --git a/data/semantickernel-data-sqlite/pom.xml b/data/semantickernel-data-sqlite/pom.xml index 12d6c008..7b4faa3d 100644 --- a/data/semantickernel-data-sqlite/pom.xml +++ b/data/semantickernel-data-sqlite/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT ../../pom.xml diff --git a/pom.xml b/pom.xml index a7c3832e..611f4472 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT pom https://www.github.com/microsoft/semantic-kernel @@ -947,6 +947,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.5.0 + HEAD diff --git a/samples/pom.xml b/samples/pom.xml index f4122eec..6ddaea22 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index ff5b86e6..99d85497 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.5.0 + 1.5.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index c62276fe..8d42b664 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.5.0 + 1.5.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index b219fa33..ad08549e 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.5.0 + 1.5.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index 86ccc67d..ba42e0db 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.5.0 + 1.5.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index 37026b27..29454883 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.5.0 + 1.5.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index ead8e792..2684a88c 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.5.0 + 1.5.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index ecc8138e..420b98e3 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.5.0 + 1.5.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index 3cfedcb7..c09c6ea8 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.5.0 + 1.5.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index b3664ab6..4016488a 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.5.0 + 1.5.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index 6edcde9f..af97964d 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.5.0 + 1.5.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml index ba34af35..5236bcff 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-text-splitter-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.5.0 + 1.5.1-SNAPSHOT ../pom.xml diff --git a/semantickernel-api-ai-services/pom.xml b/semantickernel-api-ai-services/pom.xml index 1412e3cb..e1fd17ef 100644 --- a/semantickernel-api-ai-services/pom.xml +++ b/semantickernel-api-ai-services/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT ../pom.xml diff --git a/semantickernel-api-builders/pom.xml b/semantickernel-api-builders/pom.xml index ffd6774d..cbd35cb8 100644 --- a/semantickernel-api-builders/pom.xml +++ b/semantickernel-api-builders/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT com.microsoft.semantic-kernel diff --git a/semantickernel-api-data/pom.xml b/semantickernel-api-data/pom.xml index 15dc8210..9e055c1e 100644 --- a/semantickernel-api-data/pom.xml +++ b/semantickernel-api-data/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT ../pom.xml diff --git a/semantickernel-api-exceptions/pom.xml b/semantickernel-api-exceptions/pom.xml index a5ee92a0..bb60dead 100644 --- a/semantickernel-api-exceptions/pom.xml +++ b/semantickernel-api-exceptions/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT ../pom.xml diff --git a/semantickernel-api-localization/pom.xml b/semantickernel-api-localization/pom.xml index 82becc44..b57ad77f 100644 --- a/semantickernel-api-localization/pom.xml +++ b/semantickernel-api-localization/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT ../pom.xml diff --git a/semantickernel-api-textembedding-services/pom.xml b/semantickernel-api-textembedding-services/pom.xml index dfed9b1e..3cba6dd4 100644 --- a/semantickernel-api-textembedding-services/pom.xml +++ b/semantickernel-api-textembedding-services/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index 7f25cdc0..dcebfbc6 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index ed3c8707..424a84fa 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.5.0 + 1.5.1-SNAPSHOT pom Semantic Kernel Java BOM @@ -326,6 +326,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.5.0 + HEAD diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 7dbcca0f..63031771 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.5.0 + 1.5.1-SNAPSHOT semantickernel-experimental From 8213b74a99e9e6cd0ba3144647669c3e742ef7c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 May 2026 15:15:32 +0000 Subject: [PATCH 269/277] Bump org.postgresql:postgresql in /data/semantickernel-data-postgres Bumps [org.postgresql:postgresql](https://github.com/pgjdbc/pgjdbc) from 42.7.10 to 42.7.11. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.7.10...REL42.7.11) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-version: 42.7.11 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- data/semantickernel-data-postgres/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/semantickernel-data-postgres/pom.xml b/data/semantickernel-data-postgres/pom.xml index 9949118d..810a9f53 100644 --- a/data/semantickernel-data-postgres/pom.xml +++ b/data/semantickernel-data-postgres/pom.xml @@ -51,7 +51,7 @@ org.postgresql postgresql - 42.7.10 + 42.7.11 \ No newline at end of file From d042f6a02241fd50c525f6e52c3e9bc03d7c82f1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jun 2026 07:52:58 +0000 Subject: [PATCH 270/277] Bump actions/checkout from 6 to 7 Bumps [actions/checkout](https://github.com/actions/checkout) from 6 to 7. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/java-build.yml | 2 +- .github/workflows/java-integration-tests.yml | 2 +- .github/workflows/java-publish-package.yml | 2 +- .github/workflows/markdown-link-check.yml | 2 +- .github/workflows/typos.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index dee4d1c4..5fe9e853 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -30,7 +30,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/java-build.yml b/.github/workflows/java-build.yml index d63a0d44..bf35695a 100644 --- a/.github/workflows/java-build.yml +++ b/.github/workflows/java-build.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 # Need to use JDK 11 to build for JDK 8 - name: Set JDK diff --git a/.github/workflows/java-integration-tests.yml b/.github/workflows/java-integration-tests.yml index 17edd75e..45afb219 100644 --- a/.github/workflows/java-integration-tests.yml +++ b/.github/workflows/java-integration-tests.yml @@ -22,7 +22,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 # Need to use JDK 11 to build for JDK 8 - name: Set JDK diff --git a/.github/workflows/java-publish-package.yml b/.github/workflows/java-publish-package.yml index 1f73e688..38512904 100644 --- a/.github/workflows/java-publish-package.yml +++ b/.github/workflows/java-publish-package.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@v7 # Sets up the specified JDK version from the matrix - uses: actions/setup-java@v5 diff --git a/.github/workflows/markdown-link-check.yml b/.github/workflows/markdown-link-check.yml index bc1a1fa7..530cb5f4 100644 --- a/.github/workflows/markdown-link-check.yml +++ b/.github/workflows/markdown-link-check.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest # check out the latest version of the code steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 # Checks the status of hyperlinks in .md files in verbose mode - name: Check links diff --git a/.github/workflows/typos.yaml b/.github/workflows/typos.yaml index 532f4d5d..c6845f2a 100644 --- a/.github/workflows/typos.yaml +++ b/.github/workflows/typos.yaml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Use custom config file uses: crate-ci/typos@master From 82ddc409b6b0e44d79a90925c8392ffaf5b98a7c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 Jun 2026 03:28:00 +0000 Subject: [PATCH 271/277] Bump com.fasterxml.jackson.core:jackson-databind in /semantickernel-bom Bumps [com.fasterxml.jackson.core:jackson-databind](https://github.com/FasterXML/jackson) from 2.21.2 to 2.22.0. - [Commits](https://github.com/FasterXML/jackson/commits) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.core:jackson-databind dependency-version: 2.22.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- semantickernel-bom/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 424a84fa..1d4947b3 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -13,7 +13,7 @@ https://www.github.com/microsoft/semantic-kernel - 2.21.2 + 2.22.0 From 7b6ff97bf7d58a80f991dcc08a52dbbe62241494 Mon Sep 17 00:00:00 2001 From: John <1615532+johnoliver@users.noreply.github.com> Date: Thu, 9 Jul 2026 21:50:54 +0100 Subject: [PATCH 272/277] Add profile that allows skiping container tests for when running in environments that cannot pull docker images (#365) --- data/semantickernel-data-oracle/pom.xml | 7 +++++++ pom.xml | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/data/semantickernel-data-oracle/pom.xml b/data/semantickernel-data-oracle/pom.xml index de631318..42ec5b2c 100644 --- a/data/semantickernel-data-oracle/pom.xml +++ b/data/semantickernel-data-oracle/pom.xml @@ -119,6 +119,13 @@ true + + org.apache.maven.plugins + maven-surefire-plugin + + ${skipTests.oracle} + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 611f4472..fd834bd3 100644 --- a/pom.xml +++ b/pom.xml @@ -837,6 +837,15 @@ api-test + + skip-container-tests + + false + + + true + + release From a7c6a33472d868b428250be02054cf37dee7d725 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 14:16:09 +0000 Subject: [PATCH 273/277] Bump org.postgresql:postgresql in /data/semantickernel-data-jdbc Bumps [org.postgresql:postgresql](https://github.com/pgjdbc/pgjdbc) from 42.7.10 to 42.7.12. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.7.10...REL42.7.12) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-version: 42.7.12 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- data/semantickernel-data-jdbc/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/semantickernel-data-jdbc/pom.xml b/data/semantickernel-data-jdbc/pom.xml index 74de0e2c..cf8069ec 100644 --- a/data/semantickernel-data-jdbc/pom.xml +++ b/data/semantickernel-data-jdbc/pom.xml @@ -66,7 +66,7 @@ org.postgresql postgresql - 42.7.10 + 42.7.12 org.xerial From 1e7b081ec2706f26029e7c736cb4836e3b2d6d9f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 14:18:40 +0000 Subject: [PATCH 274/277] Bump org.postgresql:postgresql in /data/semantickernel-data-postgres Bumps [org.postgresql:postgresql](https://github.com/pgjdbc/pgjdbc) from 42.7.11 to 42.7.12. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.7.11...REL42.7.12) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-version: 42.7.12 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- data/semantickernel-data-postgres/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/semantickernel-data-postgres/pom.xml b/data/semantickernel-data-postgres/pom.xml index 810a9f53..cfc838f7 100644 --- a/data/semantickernel-data-postgres/pom.xml +++ b/data/semantickernel-data-postgres/pom.xml @@ -51,7 +51,7 @@ org.postgresql postgresql - 42.7.11 + 42.7.12 \ No newline at end of file From 1f5f2d5f65cff85530cffdd44b4110130e7cca0f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 22:57:19 +0000 Subject: [PATCH 275/277] Bump com.fasterxml.jackson.core:jackson-core in /semantickernel-bom Bumps [com.fasterxml.jackson.core:jackson-core](https://github.com/FasterXML/jackson-core) from 2.22.0 to 2.22.1. - [Commits](https://github.com/FasterXML/jackson-core/compare/jackson-core-2.22.0...jackson-core-2.22.1) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.core:jackson-core dependency-version: 2.22.1 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- semantickernel-bom/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 1d4947b3..d0eb376b 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -13,7 +13,7 @@ https://www.github.com/microsoft/semantic-kernel - 2.22.0 + 2.22.1 From 11f19e41603b63b14f9f67bc0a43a16744d76179 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 22:57:21 +0000 Subject: [PATCH 276/277] Bump com.fasterxml.jackson.core:jackson-databind in /semantickernel-bom Bumps [com.fasterxml.jackson.core:jackson-databind](https://github.com/FasterXML/jackson) from 2.22.0 to 2.22.1. - [Commits](https://github.com/FasterXML/jackson/commits) --- updated-dependencies: - dependency-name: com.fasterxml.jackson.core:jackson-databind dependency-version: 2.22.1 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- semantickernel-bom/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 1d4947b3..d0eb376b 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -13,7 +13,7 @@ https://www.github.com/microsoft/semantic-kernel - 2.22.0 + 2.22.1 From 7b607f7f860fe7b55380f3c0cb4068fc8697e736 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 08:01:03 +0000 Subject: [PATCH 277/277] Bump actions/stale from 10 to 11 Bumps [actions/stale](https://github.com/actions/stale) from 10 to 11. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v10...v11) --- updated-dependencies: - dependency-name: actions/stale dependency-version: '11' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/close-inactive-issues.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/close-inactive-issues.yml b/.github/workflows/close-inactive-issues.yml index 4e6ebcce..bbd6a87c 100644 --- a/.github/workflows/close-inactive-issues.yml +++ b/.github/workflows/close-inactive-issues.yml @@ -10,7 +10,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@v10 + - uses: actions/stale@v11 with: days-before-issue-stale: 90 days-before-issue-close: 14