From b84e42e34362c13db851704a4b8f3c190fe2cb76 Mon Sep 17 00:00:00 2001 From: GitHub Date: Wed, 24 Jul 2024 17:27:21 +0100 Subject: [PATCH 01/28] [maven-release-plugin] prepare for next development iteration --- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- pom.xml | 4 ++-- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-bom/pom.xml | 4 ++-- semantickernel-experimental/pom.xml | 2 +- 20 files changed, 22 insertions(+), 22 deletions(-) diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index ef878373..0ca6600e 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.0 + 1.2.1-SNAPSHOT ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index f53983c9..7743b989 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.0 + 1.2.1-SNAPSHOT ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index a7b12c09..dbb1980c 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.0 + 1.2.1-SNAPSHOT ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 48f5b608..126b3741 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.2.0 + 1.2.1-SNAPSHOT ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index 7b49dd77..7effdac3 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.0 + 1.2.1-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 313a7869..a846ccab 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.0 + 1.2.1-SNAPSHOT pom https://www.github.com/microsoft/semantic-kernel @@ -812,6 +812,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.2.0 + HEAD diff --git a/samples/pom.xml b/samples/pom.xml index 61a1dada..a153ef2f 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.0 + 1.2.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index a79a2371..673abe6d 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.0 + 1.2.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 9cedbbf0..bea43f73 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.2.0 + 1.2.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index c427bb4e..0247bef0 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.2.0 + 1.2.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index 4ef911aa..586c7969 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.0 + 1.2.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index acd56827..ac17a0db 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.2.0 + 1.2.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index 11dd2ae6..40dcdbc0 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.2.0 + 1.2.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 63ee1288..e7a1b516 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.0 + 1.2.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index 31bd11cc..1a526043 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.0 + 1.2.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index 6ac975d2..5424f254 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.2.0 + 1.2.1-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index cd3c0431..952ff6af 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.2.0 + 1.2.1-SNAPSHOT ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index 2bb0a050..6171818b 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.0 + 1.2.1-SNAPSHOT ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index b7f52070..180d8da8 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.2.0 + 1.2.1-SNAPSHOT pom Semantic Kernel Java BOM @@ -256,6 +256,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.2.0 + HEAD diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 327ff1e6..20fa172a 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.0 + 1.2.1-SNAPSHOT semantickernel-experimental From b5a05bd2c013a0bac5ed51128de11d8cce56f853 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Fri, 26 Jul 2024 12:46:56 +0100 Subject: [PATCH 02/28] Update build docs for new repository --- BUILD.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/BUILD.md b/BUILD.md index 7c45218d..bc0c61ce 100644 --- a/BUILD.md +++ b/BUILD.md @@ -36,11 +36,11 @@ To build the Semantic Kernel for Java, you will need: 1. Clone this repository - git clone -b java-v1 https://github.com/microsoft/semantic-kernel/ + git clone https://github.com/microsoft/semantic-kernel-java 2. Build the project with the Maven Wrapper - cd semantic-kernel/java + cd semantic-kernel ./mvnw install 3. (Optional) To run a FULL build including static analysis and end-to-end tests that might require a valid OpenAI key, @@ -104,8 +104,7 @@ Also ensure that: - All new code is covered by unit tests - All new code is covered by integration tests -Once your proposal is ready, submit a pull request to the `java-v1` branch. The pull request will be reviewed by the -project maintainers. +Once your proposal is ready, submit a pull request. The pull request will be reviewed by the project maintainers. Make sure your pull request has an objective title and a clear description explaining the problem and solution. From 844f2c942809f2c9e6bc5a34efbfab1f8ac9275a Mon Sep 17 00:00:00 2001 From: Takshi Uesaka Date: Sat, 27 Jul 2024 21:36:25 +0900 Subject: [PATCH 03/28] Iadded two overload methods to the invokePromptAsync method 1. One method accepts KernelFunctionArguments 2. The other method accepts KernelFunctionArguments and InvocationContext --- .../com/microsoft/semantickernel/Kernel.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/Kernel.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/Kernel.java index 24b838bc..4ab1553f 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/Kernel.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/Kernel.java @@ -163,6 +163,24 @@ public FunctionInvocation invokePromptAsync(@Nonnull String prompt) { return invokeAsync(KernelFunction.createFromPrompt(prompt).build()); } + public FunctionInvocation invokePromptAsync(@Nonnull String prompt, + @Nonnull KernelFunctionArguments arguments) { + KernelFunction function = KernelFunction.createFromPrompt(prompt).build(); + + return function.invokeAsync(this) + .withArguments(arguments); + } + + public FunctionInvocation invokePromptAsync(@Nonnull String prompt, + @Nonnull KernelFunctionArguments arguments, @Nonnull InvocationContext invocationContext) { + + KernelFunction function = KernelFunction.createFromPrompt(prompt).build(); + + return function.invokeAsync(this) + .withArguments(arguments) + .withInvocationContext(invocationContext); + } + /** * Invokes a {@code KernelFunction}. * From caff8b665d098ee5e74973c97bf22559c391f3e0 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Fri, 26 Jul 2024 12:07:48 +0100 Subject: [PATCH 04/28] Fix some global kernel hooks not being executed --- .../chatcompletion/OpenAIChatCompletion.java | 33 ++++++++++++------- .../samples/demos/lights/App.java | 22 +++++++++++++ .../memory/InMemory_DataStorage.java | 3 +- .../semantickernel/hooks/KernelHooks.java | 27 ++++++++++++++- .../KernelFunctionFromMethod.java | 7 ++-- .../KernelFunctionFromPrompt.java | 7 ++-- 6 files changed, 78 insertions(+), 21 deletions(-) diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java index 6bdb4f1c..ce988126 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java @@ -311,6 +311,7 @@ private Mono internalChatMessageContentsAsync( ChatCompletionsOptions options = executeHook( invocationContext, + kernel, new PreChatCompletionEvent( getCompletionsOptions( this, @@ -349,7 +350,7 @@ private Mono internalChatMessageContentsAsync( .collect(Collectors.toList()); // execute post chat completion hook - executeHook(invocationContext, new PostChatCompletionEvent(completions)); + executeHook(invocationContext, kernel, new PostChatCompletionEvent(completions)); // Just return the result: // If we don't want to attempt to invoke any functions @@ -517,11 +518,12 @@ private Mono> invokeFunctionTool( pluginName, openAIFunctionToolCall.getFunctionName()); - PreToolCallEvent hookResult = executeHook(invocationContext, new PreToolCallEvent( - openAIFunctionToolCall.getFunctionName(), - openAIFunctionToolCall.getArguments(), - function, - contextVariableTypes)); + PreToolCallEvent hookResult = executeHook(invocationContext, kernel, + new PreToolCallEvent( + openAIFunctionToolCall.getFunctionName(), + openAIFunctionToolCall.getArguments(), + function, + contextVariableTypes)); function = hookResult.getFunction(); KernelFunctionArguments arguments = hookResult.getArguments(); @@ -537,12 +539,21 @@ private Mono> invokeFunctionTool( private static T executeHook( @Nullable InvocationContext invocationContext, + @Nullable Kernel kernel, T event) { - KernelHooks kernelHooks = invocationContext != null - && invocationContext.getKernelHooks() != null - ? invocationContext.getKernelHooks() - : new KernelHooks(); - + KernelHooks kernelHooks = null; + if (kernel == null) { + if (invocationContext != null) { + kernelHooks = invocationContext.getKernelHooks(); + } + } else { + kernelHooks = KernelHooks.merge( + kernel.getGlobalKernelHooks(), + invocationContext != null ? invocationContext.getKernelHooks() : null); + } + if (kernelHooks == null) { + return event; + } return kernelHooks.executeHooks(event); } diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/App.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/App.java index 08fb2697..08a8f3b5 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/App.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/demos/lights/App.java @@ -10,6 +10,7 @@ import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIChatCompletion; import com.microsoft.semantickernel.contextvariables.ContextVariableTypeConverter; import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; +import com.microsoft.semantickernel.hooks.KernelHooks; import com.microsoft.semantickernel.orchestration.InvocationContext; import com.microsoft.semantickernel.orchestration.InvocationContext.Builder; import com.microsoft.semantickernel.orchestration.InvocationReturnMode; @@ -73,6 +74,27 @@ public static void main(String[] args) throws Exception { .toPromptString(new Gson()::toJson) .build()); + KernelHooks hook = new KernelHooks(); + + hook.addPreToolCallHook((context) -> { + System.out.println("Pre-tool call hook"); + return context; + }); + + hook.addPreChatCompletionHook( + (context) -> { + System.out.println("Pre-chat completion hook"); + return context; + }); + + hook.addPostChatCompletionHook( + (context) -> { + System.out.println("Post-chat completion hook"); + return context; + }); + + kernel.getGlobalKernelHooks().addHooks(hook); + // Enable planning InvocationContext invocationContext = new Builder() .withReturnMode(InvocationReturnMode.LAST_MESSAGE_ONLY) diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java index b189081b..d92c1e06 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java @@ -105,7 +105,8 @@ public static void main(String[] args) { inMemoryDataStorage(embeddingGeneration); } - public static void inMemoryDataStorage(OpenAITextEmbeddingGenerationService embeddingGeneration) { + public static void inMemoryDataStorage( + OpenAITextEmbeddingGenerationService embeddingGeneration) { // Create a new Volatile vector store var volatileVectorStore = new VolatileVectorStore(); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/KernelHooks.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/KernelHooks.java index f690fe26..959dda5a 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/KernelHooks.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/hooks/KernelHooks.java @@ -66,7 +66,7 @@ public UnmodifiableKernelHooks unmodifiableClone() { * * @return an unmodifiable map of the hooks */ - private Map> getHooks() { + protected Map> getHooks() { return Collections.unmodifiableMap(hooks); } @@ -224,6 +224,31 @@ public boolean isEmpty() { return hooks.isEmpty(); } + /** + * Builds the list of hooks to be invoked for the given context, by merging the hooks in this + * collection with the hooks in the context. Duplicate hooks in b will override hooks in a. + * + * @param a hooks to merge + * @param b hooks to merge + * @return the list of hooks to be invoked + */ + public static KernelHooks merge(@Nullable KernelHooks a, @Nullable KernelHooks b) { + KernelHooks hooks = a; + if (hooks == null) { + hooks = new KernelHooks(); + } + + if (b == null) { + return hooks; + } else if (hooks.isEmpty()) { + return b; + } else { + HashMap> merged = new HashMap<>(hooks.getHooks()); + merged.putAll(b.getHooks()); + return new KernelHooks(merged); + } + } + /** * A wrapper for KernelHooks that disables mutating methods. */ diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java index 6d9d1166..9a7b09dc 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromMethod.java @@ -157,10 +157,9 @@ public static ImplementationFunc getFunction(Method method, Object instan } // kernelHooks must be effectively final for lambda - KernelHooks kernelHooks = context.getKernelHooks() != null - ? context.getKernelHooks() - : kernel.getGlobalKernelHooks(); - assert kernelHooks != null : "getGlobalKernelHooks() should never return null!"; + KernelHooks kernelHooks = KernelHooks.merge( + kernel.getGlobalKernelHooks(), + context.getKernelHooks()); FunctionInvokingEvent updatedState = kernelHooks .executeHooks( diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java index 1d754d65..e3be61e8 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java @@ -102,10 +102,9 @@ private Flux> invokeInternalAsync( : InvocationContext.builder().build(); // must be effectively final for lambda - KernelHooks kernelHooks = context.getKernelHooks() != null - ? context.getKernelHooks() - : kernel.getGlobalKernelHooks(); - assert kernelHooks != null : "getGlobalKernelHooks() should never return null"; + KernelHooks kernelHooks = KernelHooks.merge( + kernel.getGlobalKernelHooks(), + context.getKernelHooks()); PromptRenderingEvent preRenderingHookResult = kernelHooks .executeHooks(new PromptRenderingEvent(this, argumentsIn)); From 4e242a513a3f7ed4775a28f041522ad1f59b33a9 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Mon, 29 Jul 2024 17:09:02 +0100 Subject: [PATCH 05/28] Add opt-out for the semantic-kernel user-agent addition to openai requests --- .../implementation/OpenAIRequestSettings.java | 45 ++++++++++++++++--- .../memory/InMemory_DataStorage.java | 3 +- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/implementation/OpenAIRequestSettings.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/implementation/OpenAIRequestSettings.java index 8da85ed9..9b92e0f8 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/implementation/OpenAIRequestSettings.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/implementation/OpenAIRequestSettings.java @@ -8,6 +8,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.Properties; +import javax.annotation.Nullable; import org.slf4j.Logger; /** @@ -20,14 +21,36 @@ public final class OpenAIRequestSettings { private static final String SEMANTIC_KERNEL_VERSION_PROPERTY_NAME = "semantic-kernel.version"; private static final String SEMANTIC_KERNEL_VERSION_PROPERTIES_FILE = "semantic-kernel-version.properties"; + @Nullable private static final String useragent; + @Nullable private static final String header; + public static final String SEMANTIC_KERNEL_DISABLE_USERAGENT_PROPERTY = "semantic-kernel.useragent-disable"; + static { - String version = loadVersion(); - useragent = "semantic-kernel-java/" + version; - header = "java/" + version; + boolean disabled = isDisabled(); + + if (!disabled) { + String version = loadVersion(); + useragent = "semantic-kernel-java/" + version; + header = "java/" + version; + } else { + useragent = null; + header = null; + } + } + + private static boolean isDisabled() { + boolean disable = false; + try { + disable = Boolean.parseBoolean( + System.getProperty(SEMANTIC_KERNEL_DISABLE_USERAGENT_PROPERTY, "false")); + } catch (Exception e) { + LOGGER.error("Failed to parse system property 'semantic-kernel.disable-useragent'.", e); + } + return disable; } private static String loadVersion() { @@ -58,9 +81,19 @@ private static String loadVersion() { * @return The request options */ public static RequestOptions getRequestOptions() { - return new RequestOptions() - .setHeader(HttpHeaderName.fromString("Semantic-Kernel-Version"), header) - .setContext( + RequestOptions requestOptions = new RequestOptions(); + + if (header != null) { + requestOptions.setHeader( + HttpHeaderName.fromString("Semantic-Kernel-Version"), header); + } + + if (useragent != null) { + requestOptions.setContext( new Context(UserAgentPolicy.APPEND_USER_AGENT_CONTEXT_KEY, useragent)); + } + + return requestOptions; + } } diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java index b189081b..d92c1e06 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java @@ -105,7 +105,8 @@ public static void main(String[] args) { inMemoryDataStorage(embeddingGeneration); } - public static void inMemoryDataStorage(OpenAITextEmbeddingGenerationService embeddingGeneration) { + public static void inMemoryDataStorage( + OpenAITextEmbeddingGenerationService embeddingGeneration) { // Create a new Volatile vector store var volatileVectorStore = new VolatileVectorStore(); From b679197f893495150e6f96c7abbe27d1d0e089d0 Mon Sep 17 00:00:00 2001 From: Bruno Borges Date: Mon, 29 Jul 2024 13:32:51 -0700 Subject: [PATCH 06/28] Select Java tab in MS Learn docs Fix URL parameter to select Java tab in documentation --- README.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 715b979a..b748c348 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,12 @@ # Semantic Kernel for Java -Welcome to the Semantic Kernel for Java. For detailed documentation, visit [Microsoft Learn](https://learn.microsoft.com/en-us/semantic-kernel/overview/?tabs=Java). +Welcome to the Semantic Kernel for Java. For detailed documentation, visit [Microsoft Learn](https://learn.microsoft.com/en-us/semantic-kernel/overview/?tabs=Java&pivots=programming-language-java). [Semantic Kernel](https://learn.microsoft.com/en-us/semantic-kernel/overview/) is an SDK that integrates Large Language Models (LLMs) like [OpenAI](https://platform.openai.com/docs/introduction), [Azure OpenAI](https://azure.microsoft.com/en-us/products/ai-services/openai-service), and [Hugging Face](https://huggingface.co/) -with conventional programming languages like C#, Python, and Java. Semantic Kernel achieves this by allowing you to define [plugins](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/plugins) that can be chained together in just a [few lines of code](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/chaining-functions?tabs=Java#using-the-runasync-method-to-simplify-your-code). +with conventional programming languages like C#, Python, and Java. Semantic Kernel achieves this by allowing you to define [plugins](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/plugins??tabs=Java&pivots=programming-language-java) that can be chained together in just a [few lines of code](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/chaining-functions?tabs=Java&pivots=programming-language-java#using-the-runasync-method-to-simplify-your-code). -What makes Semantic Kernel _special_, however, is its ability to _automatically_ orchestrate plugins with AI. With Semantic Kernel [planners](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/planner), you can ask an LLM to generate a plan that achieves a user's unique goal. Afterwards, Semantic Kernel will execute the plan for the user. +What makes Semantic Kernel _special_, however, is its ability to _automatically_ orchestrate plugins with AI. With Semantic Kernel [planners](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/planner?tabs=Java&pivots=programming-language-java), you can ask an LLM to generate a plan that achieves a user's unique goal. Afterwards, Semantic Kernel will execute the plan for the user. For C#, Python and other language support, see [microsoft/semantic-kernel](https://github.com/microsoft/semantic-kernel). @@ -23,20 +23,20 @@ For C#, Python and other language support, see [microsoft/semantic-kernel](https The quickest way to get started with the basics is to get an API key from either OpenAI or Azure OpenAI and to run one of the Java console applications/scripts below. 1. Clone the repository: `git clone https://github.com/microsoft/semantic-kernel-java.git` -2. Follow the instructions [Start learning how to use Semantic Kernel](https://learn.microsoft.com/en-us/semantic-kernel/get-started/quick-start-guide?tabs=Java). +2. Follow the instructions [Start learning how to use Semantic Kernel](https://learn.microsoft.com/en-us/semantic-kernel/get-started/quick-start-guide?tabs=Java&pivots=programming-language-java). ## Documentation: Learning how to use Semantic Kernel The fastest way to learn how to use Semantic Kernel is with our walkthroughs on our Learn site. -1. 📖 [Overview of the kernel](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/?tabs=Java) -1. 🔌 [Understanding AI plugins](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/plugins?tabs=Java) -1. 👄 [Creating semantic functions](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/semantic-functions?tabs=Java) -1. 💽 [Creating native functions](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/native-functions?tabs=Java) -1. ⛓️ [Chaining functions together](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/chaining-functions?tabs=Java) -1. 🤖 [Auto create plans with planner](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/planner?tabs=Java) -1. 💡 [Create and run a ChatGPT plugin](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/chatgpt-plugins?tabs=Java) +1. 📖 [Overview of the kernel](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/?tabs=Java&pivots=programming-language-java) +1. 🔌 [Understanding AI plugins](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/plugins?tabs=Java&pivots=programming-language-java) +1. 👄 [Creating semantic functions](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/semantic-functions?tabs=Java&pivots=programming-language-java) +1. 💽 [Creating native functions](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/native-functions?tabs=Java&pivots=programming-language-java) +1. ⛓️ [Chaining functions together](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/chaining-functions?tabs=Java&pivots=programming-language-java) +1. 🤖 [Auto create plans with planner](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/planner?tabs=Java&pivots=programming-language-java) +1. 💡 [Create and run a ChatGPT plugin](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/chatgpt-plugins?tabs=Java&pivots=programming-language-java) ## Join the community @@ -50,8 +50,8 @@ in a different direction, but also to consider the impact on the larger ecosyste To learn more and get started: -- Read the [documentation](https://learn.microsoft.com/en-us/semantic-kernel/overview/?tabs=Java) -- Learn how to [contribute](https://learn.microsoft.com/en-us/semantic-kernel/get-started/contributing) to the project +- Read the [documentation](https://learn.microsoft.com/en-us/semantic-kernel/overview/?tabs=Java&pivots=programming-language-java) +- Learn how to [contribute](https://learn.microsoft.com/en-us/semantic-kernel/get-started/contributing?tabs=Java&pivots=programming-language-java) to the project - Join the [Discord community](https://aka.ms/SKDiscord) - Attend [regular office hours and SK community events](COMMUNITY.md) - Follow the team on our [blog](https://aka.ms/sk/blog) From 7df0fd07990604acacf8f17411212148b46e7b24 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Tue, 30 Jul 2024 12:44:24 +0100 Subject: [PATCH 07/28] Fixes from review --- .../implementation/OpenAIRequestSettings.java | 46 ++++++------------- 1 file changed, 14 insertions(+), 32 deletions(-) diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/implementation/OpenAIRequestSettings.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/implementation/OpenAIRequestSettings.java index 9b92e0f8..35d0d3f3 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/implementation/OpenAIRequestSettings.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/implementation/OpenAIRequestSettings.java @@ -8,7 +8,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.Properties; -import javax.annotation.Nullable; import org.slf4j.Logger; /** @@ -21,36 +20,24 @@ public final class OpenAIRequestSettings { private static final String SEMANTIC_KERNEL_VERSION_PROPERTY_NAME = "semantic-kernel.version"; private static final String SEMANTIC_KERNEL_VERSION_PROPERTIES_FILE = "semantic-kernel-version.properties"; - @Nullable - private static final String useragent; - @Nullable + private static final String useragent; private static final String header; public static final String SEMANTIC_KERNEL_DISABLE_USERAGENT_PROPERTY = "semantic-kernel.useragent-disable"; + private static final boolean disabled; + static { - boolean disabled = isDisabled(); - - if (!disabled) { - String version = loadVersion(); - useragent = "semantic-kernel-java/" + version; - header = "java/" + version; - } else { - useragent = null; - header = null; - } + disabled = isDisabled(); + String version = loadVersion(); + useragent = "semantic-kernel-java/" + version; + header = "java/" + version; } private static boolean isDisabled() { - boolean disable = false; - try { - disable = Boolean.parseBoolean( - System.getProperty(SEMANTIC_KERNEL_DISABLE_USERAGENT_PROPERTY, "false")); - } catch (Exception e) { - LOGGER.error("Failed to parse system property 'semantic-kernel.disable-useragent'.", e); - } - return disable; + return Boolean.parseBoolean( + System.getProperty(SEMANTIC_KERNEL_DISABLE_USERAGENT_PROPERTY, "false")); } private static String loadVersion() { @@ -83,17 +70,12 @@ private static String loadVersion() { public static RequestOptions getRequestOptions() { RequestOptions requestOptions = new RequestOptions(); - if (header != null) { - requestOptions.setHeader( - HttpHeaderName.fromString("Semantic-Kernel-Version"), header); - } - - if (useragent != null) { - requestOptions.setContext( - new Context(UserAgentPolicy.APPEND_USER_AGENT_CONTEXT_KEY, useragent)); + if (disabled) { + return requestOptions; } - return requestOptions; - + return requestOptions + .setHeader(HttpHeaderName.fromString("Semantic-Kernel-Version"), header) + .setContext(new Context(UserAgentPolicy.APPEND_USER_AGENT_CONTEXT_KEY, useragent)); } } From 95ba9567d289468f8a154d4fecf6811f21208696 Mon Sep 17 00:00:00 2001 From: David Grieve Date: Tue, 30 Jul 2024 10:02:44 -0400 Subject: [PATCH 08/28] Generic client type for OpenAiService --- .../aiservices/openai/OpenAiService.java | 9 ++++----- .../openai/audio/OpenAiAudioToTextService.java | 2 +- .../openai/audio/OpenAiTextToAudioService.java | 2 +- .../chatcompletion/OpenAIChatCompletion.java | 4 ++-- .../OpenAITextGenerationService.java | 2 +- .../OpenAITextEmbeddingGenerationService.java | 4 ++-- .../services/audio/AudioToTextService.java | 3 ++- .../services/audio/TextToAudioService.java | 3 ++- .../services/openai/OpenAiServiceBuilder.java | 14 +++++++++----- .../textcompletion/TextGenerationService.java | 3 ++- 10 files changed, 26 insertions(+), 20 deletions(-) diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/OpenAiService.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/OpenAiService.java index 4da7b67b..4a7a196b 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/OpenAiService.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/OpenAiService.java @@ -1,23 +1,22 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.aiservices.openai; -import com.azure.ai.openai.OpenAIAsyncClient; import com.microsoft.semantickernel.services.AIService; import javax.annotation.Nullable; /** * Provides OpenAI service. */ -public abstract class OpenAiService implements AIService { +public abstract class OpenAiService implements AIService { - private final OpenAIAsyncClient client; + private final Client client; @Nullable private final String serviceId; private final String modelId; private final String deploymentName; protected OpenAiService( - OpenAIAsyncClient client, + Client client, @Nullable String serviceId, String modelId, String deploymentName) { @@ -39,7 +38,7 @@ public String getServiceId() { return serviceId; } - protected OpenAIAsyncClient getClient() { + protected Client getClient() { return client; } diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiAudioToTextService.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiAudioToTextService.java index 07fdc76f..7becd67e 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiAudioToTextService.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiAudioToTextService.java @@ -18,7 +18,7 @@ /** * Provides OpenAi implementation of audio to text service. */ -public class OpenAiAudioToTextService extends OpenAiService implements AudioToTextService { +public class OpenAiAudioToTextService extends OpenAiService implements AudioToTextService { private static final Logger LOGGER = LoggerFactory.getLogger(OpenAiAudioToTextService.class); diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiTextToAudioService.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiTextToAudioService.java index b4f4dafd..25071ca9 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiTextToAudioService.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiTextToAudioService.java @@ -17,7 +17,7 @@ /** * Provides OpenAi implementation of text to audio service. */ -public class OpenAiTextToAudioService extends OpenAiService implements TextToAudioService { +public class OpenAiTextToAudioService extends OpenAiService implements TextToAudioService { private static final Logger LOGGER = LoggerFactory.getLogger(OpenAiTextToAudioService.class); diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java index 6bdb4f1c..9c0510ca 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java @@ -79,7 +79,7 @@ /** * OpenAI chat completion service. */ -public class OpenAIChatCompletion extends OpenAiService implements ChatCompletionService { +public class OpenAIChatCompletion extends OpenAiService implements ChatCompletionService { private static final Logger LOGGER = LoggerFactory.getLogger(OpenAIChatCompletion.class); @@ -1044,7 +1044,7 @@ static ChatRequestMessage getChatRequestMessage( /** * Builder for creating a new instance of {@link OpenAIChatCompletion}. */ - public static class Builder extends OpenAiServiceBuilder { + public static class Builder extends OpenAiServiceBuilder { @Override public OpenAIChatCompletion build() { diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAITextGenerationService.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAITextGenerationService.java index 57e3dd1f..5c418649 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAITextGenerationService.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAITextGenerationService.java @@ -30,7 +30,7 @@ /** * An OpenAI implementation of a {@link TextGenerationService}. */ -public class OpenAITextGenerationService extends OpenAiService implements TextGenerationService { +public class OpenAITextGenerationService extends OpenAiService implements TextGenerationService { private static final Logger LOGGER = LoggerFactory.getLogger(OpenAITextGenerationService.class); diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textembedding/OpenAITextEmbeddingGenerationService.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textembedding/OpenAITextEmbeddingGenerationService.java index a46540c0..cd2c7aa8 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textembedding/OpenAITextEmbeddingGenerationService.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textembedding/OpenAITextEmbeddingGenerationService.java @@ -23,7 +23,7 @@ * An OpenAI implementation of a {@link TextEmbeddingGenerationService}. * */ -public class OpenAITextEmbeddingGenerationService extends OpenAiService +public class OpenAITextEmbeddingGenerationService extends OpenAiService implements TextEmbeddingGenerationService { private static final Logger LOGGER = LoggerFactory .getLogger(OpenAITextEmbeddingGenerationService.class); @@ -87,7 +87,7 @@ protected Mono> internalGenerateTextEmbeddingsAsync(List * A builder for creating a {@link OpenAITextEmbeddingGenerationService}. */ public static class Builder extends - OpenAiServiceBuilder { + OpenAiServiceBuilder { private int dimensions = DEFAULT_DIMENSIONS; /** diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/AudioToTextService.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/AudioToTextService.java index 91e14466..70224596 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/AudioToTextService.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/AudioToTextService.java @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.services.audio; +import com.azure.ai.openai.OpenAIAsyncClient; import com.microsoft.semantickernel.implementation.ServiceLoadUtil; import com.microsoft.semantickernel.services.AIService; import com.microsoft.semantickernel.services.openai.OpenAiServiceBuilder; @@ -32,7 +33,7 @@ static Builder builder() { /** * Builder for the AudioToTextService. */ - abstract class Builder extends OpenAiServiceBuilder { + abstract class Builder extends OpenAiServiceBuilder { } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/TextToAudioService.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/TextToAudioService.java index 80a42436..ff2cd40a 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/TextToAudioService.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/TextToAudioService.java @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.services.audio; +import com.azure.ai.openai.OpenAIAsyncClient; import com.microsoft.semantickernel.implementation.ServiceLoadUtil; import com.microsoft.semantickernel.services.AIService; import com.microsoft.semantickernel.services.openai.OpenAiServiceBuilder; @@ -36,7 +37,7 @@ static Builder builder() { * Builder for the TextToAudioService. */ abstract class Builder extends - OpenAiServiceBuilder { + OpenAiServiceBuilder { } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/openai/OpenAiServiceBuilder.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/openai/OpenAiServiceBuilder.java index 0197a55a..e42b2c6e 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/openai/OpenAiServiceBuilder.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/openai/OpenAiServiceBuilder.java @@ -1,20 +1,24 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.services.openai; -import com.azure.ai.openai.OpenAIAsyncClient; +import com.microsoft.semantickernel.services.AIService; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import javax.annotation.Nullable; /** * Builder for an OpenAI service. - */ -public abstract class OpenAiServiceBuilder> implements + * @param The client type + * @param The service type + * @param The builder type +*/ +public abstract class OpenAiServiceBuilder> implements + SemanticKernelBuilder { @Nullable protected String modelId; @Nullable - protected OpenAIAsyncClient client; + protected C client; @Nullable protected String serviceId; @Nullable @@ -51,7 +55,7 @@ public U withDeploymentName(String deploymentName) { * @param client The OpenAI client * @return The builder */ - public U withOpenAIAsyncClient(OpenAIAsyncClient client) { + public U withOpenAIAsyncClient(C client) { this.client = client; return (U) this; } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textcompletion/TextGenerationService.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textcompletion/TextGenerationService.java index ab592c86..8010b67f 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textcompletion/TextGenerationService.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textcompletion/TextGenerationService.java @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.services.textcompletion; +import com.azure.ai.openai.OpenAIAsyncClient; import com.microsoft.semantickernel.Kernel; import com.microsoft.semantickernel.implementation.ServiceLoadUtil; import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; @@ -60,6 +61,6 @@ Flux getStreamingTextContentsAsync( /** * Builder for a TextGenerationService */ - abstract class Builder extends OpenAiServiceBuilder { + abstract class Builder extends OpenAiServiceBuilder { } } From 659859f7b975b7f492abe5c2605218ae63a3be08 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Tue, 30 Jul 2024 15:19:20 +0100 Subject: [PATCH 09/28] add example for custom type conversion. Fix bug in per-invocation type conversion. Add fluent calls to ChatHistory --- CHANGELOG.md | 12 ++ .../java/CustomTypes_Example.java | 180 ++++++++++++++++++ .../ContextVariableTypeConverter.java | 4 +- .../ContextVariableJacksonConverter.java | 79 ++++++++ .../DateTimeContextVariableTypeConverter.java | 4 +- .../orchestration/FunctionInvocation.java | 8 +- .../services/chatcompletion/ChatHistory.java | 24 ++- 7 files changed, 293 insertions(+), 18 deletions(-) create mode 100644 samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/CustomTypes_Example.java create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/ContextVariableJacksonConverter.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ad56c82..c8e406be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +# 1.2.1 + +- Fix bug in `FunctionInvocation` not using per-invocation type conversion when calling `withResultType`. +- Fix bug in Global Hooks not being invoked under certain circumstances. +- Add fluent returns to `ChatHistory` `addXMessage` methods. +- Add user agent opt-out for OpenAI requests by setting the property `semantic-kernel.useragent-disable` to `true`. +- Add several convenience `invokePromptAsync` methods to `Kernel`. + +#### Non-API Changes + +- Add custom type Conversion example, CustomTypes_Example + # 1.2.0 - Add ability to use image_url as content for a OpenAi chat completion diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/CustomTypes_Example.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/CustomTypes_Example.java new file mode 100644 index 00000000..e809474d --- /dev/null +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/java/CustomTypes_Example.java @@ -0,0 +1,180 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.samples.syntaxexamples.java; + +import com.azure.ai.openai.OpenAIAsyncClient; +import com.azure.ai.openai.OpenAIClientBuilder; +import com.azure.core.credential.AzureKeyCredential; +import com.azure.core.credential.KeyCredential; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.microsoft.semantickernel.Kernel; +import com.microsoft.semantickernel.aiservices.openai.chatcompletion.OpenAIChatCompletion; +import com.microsoft.semantickernel.contextvariables.ContextVariableTypeConverter; +import com.microsoft.semantickernel.contextvariables.ContextVariableTypes; +import com.microsoft.semantickernel.contextvariables.converters.ContextVariableJacksonConverter; +import com.microsoft.semantickernel.exceptions.ConfigurationException; +import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; +import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class CustomTypes_Example { + + private static final String CLIENT_KEY = System.getenv("CLIENT_KEY"); + private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY"); + + // Only required if AZURE_CLIENT_KEY is set + private static final String CLIENT_ENDPOINT = System.getenv("CLIENT_ENDPOINT"); + private static final String MODEL_ID = System.getenv() + .getOrDefault("MODEL_ID", "gpt-35-turbo-2"); + + public static void main(String[] args) throws ConfigurationException, IOException { + + OpenAIAsyncClient client; + + if (AZURE_CLIENT_KEY != null) { + client = new OpenAIClientBuilder() + .credential(new AzureKeyCredential(AZURE_CLIENT_KEY)) + .endpoint(CLIENT_ENDPOINT) + .buildAsyncClient(); + } else { + client = new OpenAIClientBuilder() + .credential(new KeyCredential(CLIENT_KEY)) + .buildAsyncClient(); + } + + ChatCompletionService chatCompletionService = OpenAIChatCompletion.builder() + .withOpenAIAsyncClient(client) + .withModelId(MODEL_ID) + .build(); + + exampleBuildingCustomConverter(chatCompletionService); + exampleUsingJackson(chatCompletionService); + exampleUsingGlobalTypes(chatCompletionService); + } + + public record Pet(String name, int age, String species) { + + @JsonCreator + public Pet( + @JsonProperty("name") String name, + @JsonProperty("age") int age, + @JsonProperty("species") String species) { + this.name = name; + this.age = age; + this.species = species; + } + + @Override + public String toString() { + return name + " " + species + " " + age; + } + } + + private static void exampleBuildingCustomConverter( + ChatCompletionService chatCompletionService) { + Pet sandy = new Pet("Sandy", 3, "Dog"); + + Kernel kernel = Kernel.builder() + .withAIService(ChatCompletionService.class, chatCompletionService) + .build(); + + // Format: + // name: Sandy + // age: 3 + // species: Dog + + // Custom serializer + Function petToString = pet -> "name: " + pet.name() + "\n" + + "age: " + pet.age() + "\n" + + "species: " + pet.species() + "\n"; + + // Custom deserializer + Function stringToPet = prompt -> { + Map properties = Arrays.stream(prompt.split("\n")) + .collect(Collectors.toMap( + line -> line.split(":")[0].trim(), + line -> line.split(":")[1].trim())); + + return new Pet( + properties.get("name"), + Integer.parseInt(properties.get("age")), + properties.get("species")); + }; + + // create custom converter + ContextVariableTypeConverter typeConverter = ContextVariableTypeConverter.builder( + Pet.class) + .toPromptString(petToString) + .fromPromptString(stringToPet) + .build(); + + Pet updated = kernel.invokePromptAsync( + "Change Sandy's name to Daisy:\n{{$Sandy}}", + KernelFunctionArguments.builder() + .withVariable("Sandy", sandy, typeConverter) + .build()) + .withTypeConverter(typeConverter) + .withResultType(Pet.class) + .block() + .getResult(); + + System.out.println("Sandy's updated record: " + updated); + } + + public static void exampleUsingJackson(ChatCompletionService chatCompletionService) { + Pet sandy = new Pet("Sandy", 3, "Dog"); + + Kernel kernel = Kernel.builder() + .withAIService(ChatCompletionService.class, chatCompletionService) + .build(); + + // Create a converter that defaults to using jackson for serialization + ContextVariableTypeConverter typeConverter = ContextVariableJacksonConverter.create( + Pet.class); + + // Invoke the prompt with the custom converter + Pet updated = kernel.invokePromptAsync( + "Increase Sandy's age by a year:\n{{$Sandy}}", + KernelFunctionArguments.builder() + .withVariable("Sandy", sandy, typeConverter) + .build()) + .withTypeConverter(typeConverter) + .withResultType(Pet.class) + .block() + .getResult(); + + System.out.println("Sandy's updated record: " + updated); + } + + public static void exampleUsingGlobalTypes(ChatCompletionService chatCompletionService) { + Pet sandy = new Pet("Sandy", 3, "Dog"); + + Kernel kernel = Kernel.builder() + .withAIService(ChatCompletionService.class, chatCompletionService) + .build(); + + // Create a converter that defaults to using jackson for serialization + ContextVariableTypeConverter typeConverter = ContextVariableJacksonConverter.create( + Pet.class); + + // Add converter to global types + ContextVariableTypes.addGlobalConverter(typeConverter); + + // No need to explicitly tell the invocation how to convert the type + Pet updated = kernel.invokePromptAsync( + "Sandy's is actually a cat correct this:\n{{$Sandy}}", + KernelFunctionArguments.builder() + .withVariable("Sandy", sandy) + .build()) + .withResultType(Pet.class) + .block() + .getResult(); + + System.out.println("Sandy's updated record: " + updated); + } + +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypeConverter.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypeConverter.java index 0a4bc758..687ef302 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypeConverter.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/ContextVariableTypeConverter.java @@ -309,9 +309,7 @@ public static class Builder { @SuppressFBWarnings("CT_CONSTRUCTOR_THROW") public Builder(Class clazz) { this.clazz = clazz; - fromObject = x -> { - throw new UnsupportedOperationException("fromObject not implemented"); - }; + fromObject = x -> ContextVariableTypes.convert(x, clazz); toPromptString = (a, b) -> { throw new UnsupportedOperationException("toPromptString not implemented"); }; diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/ContextVariableJacksonConverter.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/ContextVariableJacksonConverter.java new file mode 100644 index 00000000..ca4bfc44 --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/ContextVariableJacksonConverter.java @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.contextvariables.converters; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.microsoft.semantickernel.contextvariables.ContextVariableTypeConverter; +import com.microsoft.semantickernel.contextvariables.ContextVariableTypeConverter.Builder; +import com.microsoft.semantickernel.exceptions.SKException; + +/** + * A utility class for creating {@link ContextVariableTypeConverter} instances that use Jackson for + * serialization and deserialization. + */ +public final class ContextVariableJacksonConverter { + + /** + * Creates a new {@link ContextVariableTypeConverter} that uses Jackson for serialization and + * deserialization. + * + * @param type the type of the context variable + * @param mapper the {@link ObjectMapper} to use for serialization and deserialization + * @param the type of the context variable + * @return a new {@link ContextVariableTypeConverter} + */ + public static ContextVariableTypeConverter create(Class type, ObjectMapper mapper) { + return builder(type, mapper).build(); + } + + /** + * Creates a new {@link ContextVariableTypeConverter} that uses Jackson for serialization and + * deserialization. + * + * @param type the type of the context variable + * @param the type of the context variable + * @return a new {@link ContextVariableTypeConverter} + */ + public static ContextVariableTypeConverter create(Class type) { + return create(type, new ObjectMapper()); + } + + /** + * Creates a new {@link Builder} for a {@link ContextVariableTypeConverter} that uses Jackson + * for serialization and deserialization. + * + * @param type the type of the context variable + * @param the type of the context variable + * @return a new {@link Builder} + */ + public static Builder builder(Class type) { + return builder(type, new ObjectMapper()); + } + + /** + * Creates a new {@link Builder} for a {@link ContextVariableTypeConverter} that uses Jackson + * for serialization and deserialization. + * + * @param type the type of the context variable + * @param mapper the {@link ObjectMapper} to use for serialization and deserialization + * @param the type of the context variable + * @return a new {@link Builder} + */ + public static Builder builder(Class type, ObjectMapper mapper) { + return ContextVariableTypeConverter.builder(type) + .fromPromptString(str -> { + try { + return mapper.readValue(str, type); + } catch (JsonProcessingException e) { + throw new SKException("Failed to deserialize object", e); + } + }) + .toPromptString(obj -> { + try { + return mapper.writerWithDefaultPrettyPrinter().writeValueAsString(obj); + } catch (JsonProcessingException e) { + throw new SKException("Failed to serialize object", e); + } + }); + } +} diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/DateTimeContextVariableTypeConverter.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/DateTimeContextVariableTypeConverter.java index 6a64ccd2..34fc741f 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/DateTimeContextVariableTypeConverter.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/contextvariables/converters/DateTimeContextVariableTypeConverter.java @@ -33,9 +33,7 @@ public DateTimeContextVariableTypeConverter() { return null; }, Object::toString, - o -> { - return ZonedDateTime.parse(o).toOffsetDateTime(); - }, + o -> ZonedDateTime.parse(o).toOffsetDateTime(), Arrays.asList( new DefaultConverter(OffsetDateTime.class, Instant.class) { @Override diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java index 276f2c18..4cd6ae2a 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/orchestration/FunctionInvocation.java @@ -133,7 +133,7 @@ private static BiConsumer, SynchronousSink FunctionInvocation withResultType(ContextVariableType resultTyp * @return A new {@code FunctionInvocation} for fluent chaining. */ public FunctionInvocation withResultType(Class resultType) { - return withResultType(ContextVariableTypes.getGlobalVariableTypeForClass(resultType)); + try { + return withResultType(contextVariableTypes.getVariableTypeForSuperClass(resultType)); + } catch (SKException e) { + return withResultType(ContextVariableTypes.getGlobalVariableTypeForClass(resultType)); + } } /** diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java index d2f391ff..8d6bdce6 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java @@ -113,7 +113,7 @@ public Spliterator> spliterator() { * @param encoding The encoding of the message * @param metadata The metadata of the message */ - public void addMessage(AuthorRole authorRole, String content, Charset encoding, + public ChatHistory addMessage(AuthorRole authorRole, String content, Charset encoding, FunctionResultMetadata metadata) { chatMessageContents.add( ChatMessageTextContent.builder() @@ -122,6 +122,7 @@ public void addMessage(AuthorRole authorRole, String content, Charset encoding, .withEncoding(encoding) .withMetadata(metadata) .build()); + return this; } /** @@ -130,12 +131,13 @@ public void addMessage(AuthorRole authorRole, String content, Charset encoding, * @param authorRole The role of the author of the message * @param content The content of the message */ - public void addMessage(AuthorRole authorRole, String content) { + public ChatHistory addMessage(AuthorRole authorRole, String content) { chatMessageContents.add( ChatMessageTextContent.builder() .withAuthorRole(authorRole) .withContent(content) .build()); + return this; } /** @@ -143,8 +145,9 @@ public void addMessage(AuthorRole authorRole, String content) { * * @param content The content of the message */ - public void addMessage(ChatMessageContent content) { + public ChatHistory addMessage(ChatMessageContent content) { chatMessageContents.add(content); + return this; } /** @@ -152,8 +155,8 @@ public void addMessage(ChatMessageContent content) { * * @param content The content of the user message */ - public void addUserMessage(String content) { - addMessage(AuthorRole.USER, content); + public ChatHistory addUserMessage(String content) { + return addMessage(AuthorRole.USER, content); } /** @@ -161,8 +164,8 @@ public void addUserMessage(String content) { * * @param content The content of the assistant message */ - public void addAssistantMessage(String content) { - addMessage(AuthorRole.ASSISTANT, content); + public ChatHistory addAssistantMessage(String content) { + return addMessage(AuthorRole.ASSISTANT, content); } /** @@ -170,11 +173,12 @@ public void addAssistantMessage(String content) { * * @param content The content of the system message */ - public void addSystemMessage(String content) { - addMessage(AuthorRole.SYSTEM, content); + public ChatHistory addSystemMessage(String content) { + return addMessage(AuthorRole.SYSTEM, content); } - public void addAll(List> messages) { + public ChatHistory addAll(List> messages) { chatMessageContents.addAll(messages); + return this; } } From 21082b89dc6c5b5444a51315d67989dd97173d5c Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Wed, 31 Jul 2024 10:19:39 -0700 Subject: [PATCH 10/28] Update VectorStoreRecordCollection --- ...reAISearchVectorStoreRecordCollection.java | 15 ++- .../RedisVectorStoreRecordCollection.java | 10 +- .../data/VectorStoreRecordCollection.java | 11 +- .../VolatileVectorStoreRecordCollection.java | 5 +- .../VectorStoreRecordDefinition.java | 117 ++++++++---------- 5 files changed, 77 insertions(+), 81 deletions(-) diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index 9576b122..70d517ac 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -90,12 +90,15 @@ public AzureAISearchVectorStoreRecordCollection( : options.getRecordDefinition(); // Validate supported types - VectorStoreRecordDefinition.validateSupportedKeyTypes(this.options.getRecordClass(), - this.recordDefinition, supportedKeyTypes); - VectorStoreRecordDefinition.validateSupportedDataTypes(this.options.getRecordClass(), - this.recordDefinition, supportedDataTypes); - VectorStoreRecordDefinition.validateSupportedVectorTypes(this.options.getRecordClass(), - this.recordDefinition, supportedVectorTypes); + VectorStoreRecordDefinition.validateSupportedTypes( + Collections.singletonList(recordDefinition.getKeyDeclaredField(this.options.getRecordClass())), + supportedKeyTypes); + VectorStoreRecordDefinition.validateSupportedTypes( + recordDefinition.getDataDeclaredFields(this.options.getRecordClass()), + supportedDataTypes); + VectorStoreRecordDefinition.validateSupportedTypes( + recordDefinition.getVectorDeclaredFields(this.options.getRecordClass()), + supportedVectorTypes); // Add non-vector fields to the list nonVectorFields.add(this.recordDefinition.getKeyField().getName()); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollection.java index 8783320b..e8d9e1db 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollection.java @@ -81,10 +81,12 @@ public RedisVectorStoreRecordCollection( } // Validate supported types - VectorStoreRecordDefinition.validateSupportedKeyTypes(options.getRecordClass(), - recordDefinition, supportedKeyTypes); - VectorStoreRecordDefinition.validateSupportedVectorTypes(options.getRecordClass(), - recordDefinition, supportedVectorTypes); + VectorStoreRecordDefinition.validateSupportedTypes( + Collections.singletonList(recordDefinition.getKeyDeclaredField(this.options.getRecordClass())), + supportedKeyTypes); + VectorStoreRecordDefinition.validateSupportedTypes( + recordDefinition.getVectorDeclaredFields(this.options.getRecordClass()), + supportedVectorTypes); // If mapper is not provided, set a default one if (options.getVectorStoreRecordMapper() == null) { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollection.java index 867cbf16..1466ac35 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollection.java @@ -9,41 +9,40 @@ import java.util.List; public interface VectorStoreRecordCollection { - /** * Gets the name of the collection. * * @return The name of the collection. */ - public String getCollectionName(); + String getCollectionName(); /** * Checks if the collection exists in the store. * * @return A Mono emitting a boolean indicating if the collection exists. */ - public Mono collectionExistsAsync(); + Mono collectionExistsAsync(); /** * Creates the collection in the store. * * @return A Mono representing the completion of the creation operation. */ - public Mono createCollectionAsync(); + Mono createCollectionAsync(); /** * Creates the collection in the store if it does not exist. * * @return A Mono representing the completion of the creation operation. */ - public Mono createCollectionIfNotExistsAsync(); + Mono createCollectionIfNotExistsAsync(); /** * Deletes the collection from the store. * * @return A Mono representing the completion of the deletion operation. */ - public Mono deleteCollectionAsync(); + Mono deleteCollectionAsync(); /** * Gets a record from the store. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java index e675d4cd..8c1270ba 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java @@ -43,8 +43,9 @@ public VolatileVectorStoreRecordCollection(String collectionName, } // Validate the key type - VectorStoreRecordDefinition.validateSupportedKeyTypes(options.getRecordClass(), - recordDefinition, supportedKeyTypes); + VectorStoreRecordDefinition.validateSupportedTypes( + Collections.singletonList(recordDefinition.getKeyDeclaredField(options.getRecordClass())), + supportedKeyTypes); } VolatileVectorStoreRecordCollection(String collectionName, diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java index a1914d2c..e36a258b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java @@ -5,13 +5,12 @@ import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; /** @@ -50,6 +49,50 @@ public List getAllFields() { return fields; } + public List getNonVectorFields() { + List fields = new ArrayList<>(); + fields.add(keyField); + fields.addAll(dataFields); + return fields; + } + + private List getDeclaredFields(Class recordClass, List fields, String fieldType) { + List declaredFields = new ArrayList<>(); + for (VectorStoreRecordField field : fields) { + try { + Field declaredField = recordClass.getDeclaredField(field.getName()); + declaredFields.add(declaredField); + } catch (NoSuchFieldException e) { + throw new IllegalArgumentException( + String.format("%s field not found in record class: %s", fieldType, field.getName())); + } + } + return declaredFields; + } + + public Field getKeyDeclaredField(Class recordClass) { + try { + return recordClass.getDeclaredField(keyField.getName()); + } catch (NoSuchFieldException e) { + throw new IllegalArgumentException( + "Key field not found in record class: " + keyField.getName()); + } + } + + public List getDataDeclaredFields(Class recordClass) { + return getDeclaredFields( + recordClass, + dataFields.stream().map(f -> (VectorStoreRecordField) f).collect(Collectors.toList()), + "Data"); + } + + public List getVectorDeclaredFields(Class recordClass) { + return getDeclaredFields( + recordClass, + vectorFields.stream().map(f -> (VectorStoreRecordField) f).collect(Collectors.toList()), + "Vector"); + } + private VectorStoreRecordDefinition( VectorStoreRecordKeyField keyField, List dataFields, @@ -148,71 +191,19 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) return checkFields(keyFields, dataFields, vectorFields); } - private static String getSupportedTypesString(@Nullable HashSet> types) { - if (types == null || types.isEmpty()) { - return ""; - } - return types.stream().map(Class::getName).collect(Collectors.joining(", ")); - } - - public static void validateSupportedKeyTypes(@Nonnull Class recordClass, - @Nonnull VectorStoreRecordDefinition recordDefinition, - @Nonnull HashSet> supportedTypes) { - String supportedTypesString = getSupportedTypesString(supportedTypes); - - try { - Field declaredField = recordClass.getDeclaredField(recordDefinition.keyField.getName()); + public static void validateSupportedTypes(List declaredFields, Set> supportedTypes) { + Set> unsupportedTypes = new HashSet<>(); + for (Field declaredField : declaredFields) { if (!supportedTypes.contains(declaredField.getType())) { - throw new IllegalArgumentException( - "Unsupported key field type: " + declaredField.getType().getName() - + ". Supported types are: " + supportedTypesString); - } - } catch (NoSuchFieldException e) { - throw new IllegalArgumentException( - "Key field not found in record class: " + recordDefinition.keyField.getName()); - } - } - - public static void validateSupportedDataTypes(@Nonnull Class recordClass, - @Nonnull VectorStoreRecordDefinition recordDefinition, - @Nonnull HashSet> supportedTypes) { - String supportedTypesString = getSupportedTypesString(supportedTypes); - - for (VectorStoreRecordDataField field : recordDefinition.dataFields) { - try { - Field declaredField = recordClass.getDeclaredField(field.getName()); - - if (!supportedTypes.contains(declaredField.getType())) { - throw new IllegalArgumentException( - "Unsupported data field type: " + declaredField.getType().getName() - + ". Supported types are: " + supportedTypesString); - } - } catch (NoSuchFieldException e) { - throw new IllegalArgumentException( - "Data field not found in record class: " + field.getName()); + unsupportedTypes.add(declaredField.getType()); } } - } - - public static void validateSupportedVectorTypes(@Nonnull Class recordClass, - @Nonnull VectorStoreRecordDefinition recordDefinition, - @Nonnull HashSet> supportedTypes) { - String supportedTypesString = getSupportedTypesString(supportedTypes); - - for (VectorStoreRecordVectorField field : recordDefinition.vectorFields) { - try { - Field declaredField = recordClass.getDeclaredField(field.getName()); - - if (!supportedTypes.contains(declaredField.getType())) { - throw new IllegalArgumentException( - "Unsupported vector field type: " + declaredField.getType().getName() - + ". Supported types are: " + supportedTypesString); - } - } catch (NoSuchFieldException e) { - throw new IllegalArgumentException( - "Vector field not found in record class: " + field.getName()); - } + if (!unsupportedTypes.isEmpty()) { + throw new IllegalArgumentException( + String.format("Unsupported field types found in record class: %s. Supported types: %s", + unsupportedTypes.stream().map(Class::getName).collect(Collectors.joining(", ")), + supportedTypes.stream().map(Class::getName).collect(Collectors.joining(", ")))); } } } From 4d17c1f5bb0ca5209afa64bc5d190e6981c75995 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 1 Aug 2024 00:07:02 -0700 Subject: [PATCH 11/28] Add JDBC Vector Store --- .../JDBCVectorStoreRecordCollectionTest.java | 248 +++++++++++++ .../memory/jdbc/JDBCVectorStoreTest.java | 64 ++++ .../connectors/data/jdbc/JDBCVectorStore.java | 171 +++++++++ .../JDBCVectorStoreDefaultQueryProvider.java | 304 ++++++++++++++++ .../data/jdbc/JDBCVectorStoreOptions.java | 101 ++++++ .../jdbc/JDBCVectorStoreQueryProvider.java | 120 +++++++ .../jdbc/JDBCVectorStoreRecordCollection.java | 331 ++++++++++++++++++ ...DBCVectorStoreRecordCollectionFactory.java | 19 + ...DBCVectorStoreRecordCollectionOptions.java | 128 +++++++ .../jdbc/JDBCVectorStoreRecordMapper.java | 150 ++++++++ .../jdbc/MySQLVectorStoreQueryProvider.java | 104 ++++++ .../connectors/data/jdbc/SQLVectorStore.java | 15 + .../jdbc/SQLVectorStoreRecordCollection.java | 14 + 13 files changed, 1769 insertions(+) create mode 100644 api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java create mode 100644 api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/MySQLVectorStoreQueryProvider.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java new file mode 100644 index 00000000..efbe9638 --- /dev/null +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -0,0 +1,248 @@ +package com.microsoft.semantickernel.tests.connectors.memory.jdbc; + +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollection; +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; +import com.microsoft.semantickernel.connectors.data.jdbc.MySQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; +import com.microsoft.semantickernel.tests.connectors.memory.Hotel; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import javax.annotation.Nonnull; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +@Testcontainers +public class JDBCVectorStoreRecordCollectionTest { + @Container + private static final MySQLContainer CONTAINER = new MySQLContainer<>("mysql:5.7.34"); + private static final String MYSQL_USER = "test"; + private static final String MYSQL_PASSWORD = "test"; + private static Connection connection; + @BeforeAll + static void setup() throws SQLException { + connection = DriverManager.getConnection(CONTAINER.getJdbcUrl(), MYSQL_USER, MYSQL_PASSWORD); + } + + private JDBCVectorStoreRecordCollection buildRecordCollection(@Nonnull String collectionName) { + JDBCVectorStoreRecordCollection recordCollection = new JDBCVectorStoreRecordCollection<>( + connection, + collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .withQueryProvider(MySQLVectorStoreQueryProvider.builder() + .withConnection(connection) + .build()) + .build()); + + recordCollection.prepareAsync().block(); + recordCollection.createCollectionIfNotExistsAsync().block(); + return recordCollection; + } + + @Test + public void buildRecordCollection() { + assertNotNull(buildRecordCollection("buildTest")); + } + + private List getHotels() { + return List.of( + new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(1.0f, 2.0f, 3.0f), 4.0), + new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(1.0f, 2.0f, 3.0f), 3.0), + new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(1.0f, 2.0f, 3.0f), 5.0), + new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(1.0f, 2.0f, 3.0f), 4.0), + new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(1.0f, 2.0f, 3.0f), 5.0) + ); + } + + @Test + public void upsertAndGetRecordAsync() { + String collectionName = "upsertAndGetRecordAsync"; + JDBCVectorStoreRecordCollection recordStore = buildRecordCollection(collectionName); + + List hotels = getHotels(); + for (Hotel hotel : hotels) { + recordStore.upsertAsync(hotel, null).block(); + } + + for (Hotel hotel : hotels) { + Hotel retrievedHotel = recordStore.getAsync(hotel.getId(), null).block(); + assertNotNull(retrievedHotel); + assertEquals(hotel.getId(), retrievedHotel.getId()); + } + } + + @Test + public void getBatchAsync() { + String collectionName = "getBatchAsync"; + JDBCVectorStoreRecordCollection recordStore = buildRecordCollection(collectionName); + + List hotels = getHotels(); + for (Hotel hotel : hotels) { + recordStore.upsertAsync(hotel, null).block(); + } + + List keys = new ArrayList<>(); + for (Hotel hotel : hotels) { + keys.add(hotel.getId()); + } + + List retrievedHotels = recordStore.getBatchAsync(keys, null).block(); + assertNotNull(retrievedHotels); + assertEquals(hotels.size(), retrievedHotels.size()); + } + + @Test + public void upsertBatchAndGetBatchAsync() { + String collectionName = "upsertBatchAndGetBatchAsync"; + JDBCVectorStoreRecordCollection recordStore = buildRecordCollection(collectionName); + + List hotels = getHotels(); + recordStore.upsertBatchAsync(hotels, null).block(); + + List keys = new ArrayList<>(); + for (Hotel hotel : hotels) { + keys.add(hotel.getId()); + } + + List retrievedHotels = recordStore.getBatchAsync(keys, null).block(); + assertNotNull(retrievedHotels); + assertEquals(hotels.size(), retrievedHotels.size()); + } + + @Test + public void insertAndReplaceAsync() { + String collectionName = "insertAndReplaceAsync"; + JDBCVectorStoreRecordCollection recordStore = buildRecordCollection(collectionName); + + List hotels = getHotels(); + recordStore.upsertBatchAsync(hotels, null).block(); + recordStore.upsertBatchAsync(hotels, null).block(); + recordStore.upsertBatchAsync(hotels, null).block(); + + List keys = new ArrayList<>(); + for (Hotel hotel : hotels) { + keys.add(hotel.getId()); + } + + List retrievedHotels = recordStore.getBatchAsync(keys, null).block(); + assertNotNull(retrievedHotels); + assertEquals(hotels.size(), retrievedHotels.size()); + } + + @Test + public void deleteRecordAsync() { + String collectionName = "deleteRecordAsync"; + JDBCVectorStoreRecordCollection recordStore = buildRecordCollection(collectionName); + + List hotels = getHotels(); + recordStore.upsertBatchAsync(hotels, null).block(); + + for (Hotel hotel : hotels) { + recordStore.deleteAsync(hotel.getId(), null).block(); + Hotel retrievedHotel = recordStore.getAsync(hotel.getId(), null).block(); + assertNull(retrievedHotel); + } + } + + @Test + public void deleteBatchAsync() { + String collectionName = "deleteBatchAsync"; + JDBCVectorStoreRecordCollection recordStore = buildRecordCollection(collectionName); + + List hotels = getHotels(); + recordStore.upsertBatchAsync(hotels, null).block(); + + List keys = new ArrayList<>(); + for (Hotel hotel : hotels) { + keys.add(hotel.getId()); + } + + recordStore.deleteBatchAsync(keys, null).block(); + + for (String key : keys) { + Hotel retrievedHotel = recordStore.getAsync(key, null).block(); + assertNull(retrievedHotel); + } + } + + @Test + public void getWithNoVectors() { + String collectionName = "getWithNoVectors"; + JDBCVectorStoreRecordCollection recordStore = buildRecordCollection(collectionName); + + List hotels = getHotels(); + recordStore.upsertBatchAsync(hotels, null).block(); + + GetRecordOptions options = GetRecordOptions.builder() + .includeVectors(false) + .build(); + + for (Hotel hotel : hotels) { + Hotel retrievedHotel = recordStore.getAsync(hotel.getId(), options).block(); + assertNotNull(retrievedHotel); + assertEquals(hotel.getId(), retrievedHotel.getId()); + assertNull(retrievedHotel.getDescriptionEmbedding()); + } + + options = GetRecordOptions.builder() + .includeVectors(true) + .build(); + + for (Hotel hotel : hotels) { + Hotel retrievedHotel = recordStore.getAsync(hotel.getId(), options).block(); + assertNotNull(retrievedHotel); + assertEquals(hotel.getId(), retrievedHotel.getId()); + assertNotNull(retrievedHotel.getDescriptionEmbedding()); + } + } + + @Test + public void getBatchWithNoVectors() { + String collectionName = "getBatchWithNoVectors"; + JDBCVectorStoreRecordCollection recordStore = buildRecordCollection(collectionName); + + List hotels = getHotels(); + recordStore.upsertBatchAsync(hotels, null).block(); + + GetRecordOptions options = GetRecordOptions.builder() + .includeVectors(false) + .build(); + + List keys = new ArrayList<>(); + for (Hotel hotel : hotels) { + keys.add(hotel.getId()); + } + + List retrievedHotels = recordStore.getBatchAsync(keys, options).block(); + assertNotNull(retrievedHotels); + assertEquals(hotels.size(), retrievedHotels.size()); + + for (Hotel hotel : retrievedHotels) { + assertNull(hotel.getDescriptionEmbedding()); + } + + options = GetRecordOptions.builder() + .includeVectors(true) + .build(); + + retrievedHotels = recordStore.getBatchAsync(keys, options).block(); + assertNotNull(retrievedHotels); + assertEquals(hotels.size(), retrievedHotels.size()); + + for (Hotel hotel : retrievedHotels) { + assertNotNull(hotel.getDescriptionEmbedding()); + } + } +} diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java new file mode 100644 index 00000000..0ebeed42 --- /dev/null +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java @@ -0,0 +1,64 @@ +package com.microsoft.semantickernel.tests.connectors.memory.jdbc; + +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.connectors.data.jdbc.MySQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.tests.connectors.memory.Hotel; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Testcontainers +public class JDBCVectorStoreTest { + @Container + private static final MySQLContainer CONTAINER = new MySQLContainer<>("mysql:5.7.34"); + private static final String MYSQL_USER = "test"; + private static final String MYSQL_PASSWORD = "test"; + private static Connection connection; + + @BeforeAll + static void setup() throws SQLException { + connection = DriverManager.getConnection(CONTAINER.getJdbcUrl(), MYSQL_USER, MYSQL_PASSWORD); + } + + @Test + public void getCollectionNamesAsync() { + JDBCVectorStoreOptions options = JDBCVectorStoreOptions.builder() + .withQueryProvider(MySQLVectorStoreQueryProvider.builder() + .withConnection(connection) + .build()) + .build(); + + JDBCVectorStore vectorStore = JDBCVectorStore.builder() + .withConnection(connection) + .withOptions(options) + .build(); + + vectorStore.getCollectionNamesAsync().block(); + + List collectionNames = Arrays.asList("collection1", "collection2", "collection3"); + + for (String collectionName : collectionNames) { + vectorStore.getCollection(collectionName, Hotel.class, null).createCollectionAsync().block(); + } + + List retrievedCollectionNames = vectorStore.getCollectionNamesAsync().block(); + assertNotNull(retrievedCollectionNames); + assertEquals(collectionNames.size(), retrievedCollectionNames.size()); + for (String collectionName : collectionNames) { + assertTrue(retrievedCollectionNames.contains(collectionName)); + } + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java new file mode 100644 index 00000000..df2155a0 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java @@ -0,0 +1,171 @@ +package com.microsoft.semantickernel.connectors.data.jdbc; + +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.exceptions.SKException; +import reactor.core.publisher.Mono; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +/** + * A JDBC vector store. + */ +public class JDBCVectorStore implements SQLVectorStore> { + private final Connection connection; + private final JDBCVectorStoreOptions options; + private final JDBCVectorStoreQueryProvider queryProvider; + + /** + * Creates a new instance of the {@link JDBCVectorStore}. + * If using this constructor, call {@link #prepareAsync()} before using the vector store. + * + * @param connection the connection + * @param options the options + */ + public JDBCVectorStore(@Nonnull Connection connection, @Nullable JDBCVectorStoreOptions options) { + this.connection = connection; + this.options = options; + + if (this.options != null && this.options.getQueryProvider() != null) { + this.queryProvider = this.options.getQueryProvider(); + } else { + this.queryProvider = new JDBCVectorStoreDefaultQueryProvider(connection); + } + } + + /** + * Creates a new builder for the vector store. + * + * @return the builder + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Gets a collection from the vector store. + * + * @param collectionName The name of the collection. + * @param recordClass The class type of the record. + * @param recordDefinition The record definition. + * @return The collection. + */ + @Override + public JDBCVectorStoreRecordCollection getCollection( + @Nonnull String collectionName, + @Nonnull Class recordClass, + @Nullable VectorStoreRecordDefinition recordDefinition) { + + if (this.options != null && this.options.getVectorStoreRecordCollectionFactory() != null) { + return this.options.getVectorStoreRecordCollectionFactory() + .createVectorStoreRecordCollection( + connection, + collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(recordClass) + .withRecordDefinition(recordDefinition) + .withQueryProvider(this.queryProvider) + .build()); + } + + return new JDBCVectorStoreRecordCollection<>( + connection, + collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(recordClass) + .withRecordDefinition(recordDefinition) + .withQueryProvider(this.queryProvider) + .build()); + } + + /** + * Gets the names of all collections in the vector store. + * + * @return A list of collection names. + */ + @Override + public Mono> getCollectionNamesAsync() { + return Mono.fromCallable(() -> { + List collectionNames = new ArrayList<>(); + try { + ResultSet resultSet = queryProvider.getCollectionNames(); + while (resultSet.next()) { + collectionNames.add(resultSet.getString(1)); + } + + return collectionNames; + } catch (SQLException e) { + throw new SKException("Failed to get collection names.", e); + } + }); + } + + @Override + public Mono prepareAsync() { + return Mono.fromRunnable(() -> { + try { + queryProvider.prepareVectorStore(); + } catch (SQLException e) { + throw new SKException("Failed to prepare vector store.", e); + } + }); + } + + /** + * Builder for creating a {@link JDBCVectorStore}. + */ + public static class Builder { + private Connection connection; + private JDBCVectorStoreOptions options; + + /** + * Sets the connection. + * + * @param connection the connection + * @return the builder + */ + public Builder withConnection(Connection connection) { + this.connection = connection; + return this; + } + + /** + * Sets the options. + * + * @param options the options + * @return the builder + */ + public Builder withOptions(JDBCVectorStoreOptions options) { + this.options = options; + return this; + } + + /** + * Builds the {@link JDBCVectorStore}. + * + * @return the {@link JDBCVectorStore} + */ + public JDBCVectorStore build() { + return buildAsync().block(); + } + + /** + * Builds the {@link JDBCVectorStore} asynchronously. + * + * @return the {@link Mono} with the {@link JDBCVectorStore} + */ + public Mono buildAsync() { + if (connection == null) { + throw new IllegalArgumentException("connection is required"); + } + + JDBCVectorStore vectorStore = new JDBCVectorStore(connection, options); + return vectorStore.prepareAsync().thenReturn(vectorStore); + } + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java new file mode 100644 index 00000000..ceb2c544 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java @@ -0,0 +1,304 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.jdbc; + +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.exceptions.SKException; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; +import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; +import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +import javax.annotation.Nonnull; +import java.lang.reflect.Field; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.OffsetDateTime; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class JDBCVectorStoreDefaultQueryProvider + implements JDBCVectorStoreQueryProvider { + private static final Map, String> supportedKeyTypes; + private static final Map, String> supportedDataTypes; + private static final Map, String> supportedVectorTypes; + + static { + supportedKeyTypes = new HashMap<>(); + supportedKeyTypes.put(String.class, "VARCHAR(255)"); + + supportedDataTypes = new HashMap<>(); + supportedDataTypes.put(String.class, "TEXT"); + supportedDataTypes.put(Integer.class, "INTEGER"); + supportedDataTypes.put(int.class, "INTEGER"); + supportedDataTypes.put(Long.class, "BIGINT"); + supportedDataTypes.put(long.class, "BIGINT"); + supportedDataTypes.put(Float.class, "REAL"); + supportedDataTypes.put(float.class, "REAL"); + supportedDataTypes.put(Double.class, "DOUBLE"); + supportedDataTypes.put(double.class, "DOUBLE"); + supportedDataTypes.put(Boolean.class, "BOOLEAN"); + supportedDataTypes.put(boolean.class, "BOOLEAN"); + supportedDataTypes.put(OffsetDateTime.class, "TIMESTAMPTZ"); + + supportedVectorTypes = new HashMap<>(); + supportedVectorTypes.put(String.class, "TEXT"); + supportedVectorTypes.put(List.class, "TEXT"); + supportedVectorTypes.put(Collection.class, "TEXT"); + } + + protected final Connection connection; + protected final String collectionsTable; + protected final String prefixForCollectionTables; + + @SuppressFBWarnings("EI_EXPOSE_REP2") + public JDBCVectorStoreDefaultQueryProvider( + @Nonnull Connection connection, + @Nonnull String collectionsTable, + @Nonnull String prefixForCollectionTables) { + this.connection = connection; + // Validate table name + if (!isValidSQLIdentifier(collectionsTable)) { + throw new IllegalArgumentException("Invalid collections table name: " + collectionsTable); + } + if (!isValidSQLIdentifier(prefixForCollectionTables)) { + throw new IllegalArgumentException("Invalid prefix for collection tables: " + prefixForCollectionTables); + } + + this.collectionsTable = collectionsTable; + this.prefixForCollectionTables = prefixForCollectionTables; + } + + public JDBCVectorStoreDefaultQueryProvider( + @Nonnull Connection connection) { + this(connection, DEFAULT_COLLECTIONS_TABLE, DEFAULT_PREFIX_FOR_COLLECTION_TABLES); + } + + /** + * Creates a new builder. + * @return the builder + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Formats a wildcard string for a query. + * @param wildcards the number of wildcards + * @return the formatted wildcard string + */ + protected String getWildcardString(int wildcards) { + StringBuilder wildcardString = new StringBuilder(); + for (int i = 0; i < wildcards; ++i) { + wildcardString.append("?"); + if (i < wildcards - 1) { + wildcardString.append(", "); + } + } + return wildcardString.toString(); + } + + /** + * Formats the query columns from a record definition. + * @param fields the fields to get the columns from + * @return the formatted query columns + */ + protected String getQueryColumnsFromFields(List fields) { + return fields.stream().map(VectorStoreRecordField::getName) + .collect(Collectors.joining(", ")); + } + + protected String getColumnNamesAndTypes(List fields, Map, String> types) { + List columns = fields.stream() + .map(field -> field.getName() + " " + types.get(field.getType())) + .collect(Collectors.toList()); + + return String.join(", ", columns); + } + + protected String getCollectionTableName(String collectionName) { + return prefixForCollectionTables + collectionName; + } + + @Override + public void prepareVectorStore() throws SQLException { + String createCollectionsTable = + "CREATE TABLE IF NOT EXISTS " + collectionsTable + + " (collectionId VARCHAR(255) PRIMARY KEY);"; + + PreparedStatement createTable = connection.prepareStatement(createCollectionsTable); + createTable.execute(); + } + + @Override + public void validateSupportedTypes(Class recordClass, VectorStoreRecordDefinition recordDefinition) { + VectorStoreRecordDefinition.validateSupportedTypes( + Collections.singletonList(recordDefinition.getKeyDeclaredField(recordClass)), supportedKeyTypes.keySet()); + VectorStoreRecordDefinition.validateSupportedTypes( + recordDefinition.getDataDeclaredFields(recordClass), supportedDataTypes.keySet()); + VectorStoreRecordDefinition.validateSupportedTypes( + recordDefinition.getVectorDeclaredFields(recordClass), supportedVectorTypes.keySet()); + } + + @Override + public boolean collectionExists(String collectionName) throws SQLException { + String query = "SELECT 1 FROM " + collectionsTable + " WHERE collectionId = ?"; + + PreparedStatement statement = connection.prepareStatement(query); + statement.setObject(1, collectionName); + + return statement.executeQuery().next(); + } + + @Override + public void createCollection(String collectionName, Class recordClass, VectorStoreRecordDefinition recordDefinition) throws SQLException { + Field keyDeclaredField = recordDefinition.getKeyDeclaredField(recordClass); + List dataDeclaredFields = recordDefinition.getDataDeclaredFields(recordClass); + List vectorDeclaredFields = recordDefinition.getVectorDeclaredFields(recordClass); + + String createStorageTable = + "CREATE TABLE IF NOT EXISTS " + getCollectionTableName(collectionName) + + " (" + keyDeclaredField.getName() + " VARCHAR(255) PRIMARY KEY, " + + getColumnNamesAndTypes(dataDeclaredFields, supportedDataTypes) + ", " + + getColumnNamesAndTypes(vectorDeclaredFields, supportedVectorTypes) + ");"; + + PreparedStatement createTable = connection.prepareStatement(createStorageTable); + + String insertCollectionQuery = "INSERT INTO " + collectionsTable + " (collectionId) VALUES (?)"; + PreparedStatement insert = connection.prepareStatement(insertCollectionQuery); + insert.setObject(1, collectionName); + + createTable.execute(); + insert.execute(); + } + + @Override + public void deleteCollection(String collectionName) throws SQLException { + String deleteCollectionOperation = "DELETE FROM " + collectionsTable + " WHERE collectionId = ?"; + String dropTableOperation = "DROP TABLE " + getCollectionTableName(collectionName); + + PreparedStatement deleteCollection = connection.prepareStatement(deleteCollectionOperation); + deleteCollection.setObject(1, collectionName); + + PreparedStatement dropTable = connection.prepareStatement(dropTableOperation); + + dropTable.execute(); + deleteCollection.execute(); + } + + @Override + public ResultSet getCollectionNames() throws SQLException { + String query = "SELECT collectionId FROM " + collectionsTable; + + return connection.prepareStatement(query).executeQuery(); + } + + @Override + public ResultSet getRecords(String collectionName, List keys, VectorStoreRecordDefinition recordDefinition, GetRecordOptions options) throws SQLException { + List fields; + if (options == null || options.includeVectors()) { + fields = recordDefinition.getAllFields(); + } else { + fields = recordDefinition.getNonVectorFields(); + } + + String query = "SELECT " + getQueryColumnsFromFields(fields) + + " FROM " + getCollectionTableName(collectionName) + + " WHERE " + recordDefinition.getKeyField().getName() + + " IN (" + getWildcardString(keys.size()) + ")"; + + PreparedStatement statement = connection.prepareStatement(query); + for (int i = 0; i < keys.size(); ++i) { + try { + statement.setObject(i + 1, keys.get(i)); + } catch (SQLException e) { + throw new SKException("Failed to set statement values", e); + } + } + + return statement.executeQuery(); + } + + @Override + public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) throws SQLException { + throw new UnsupportedOperationException( + "Upsert is not supported. Try with a specific query provider."); + } + + @Override + public void deleteRecords(String collectionName, List keys, VectorStoreRecordDefinition recordDefinition, DeleteRecordOptions options) throws SQLException { + String query = "DELETE FROM " + getCollectionTableName(collectionName) + + " WHERE " + recordDefinition.getKeyField().getName() + + " IN (" + getWildcardString(keys.size()) + ")"; + + PreparedStatement statement = connection.prepareStatement(query); + for (int i = 0; i < keys.size(); ++i) { + try { + statement.setObject(i + 1, keys.get(i)); + } catch (SQLException e) { + throw new SKException("Failed to set statement values", e); + } + } + + statement.execute(); + } + + public static boolean isValidSQLIdentifier(String identifier) { + return identifier.matches("[a-zA-Z_][a-zA-Z0-9_]*"); + } + + /** + * The builder for {@link JDBCVectorStoreDefaultQueryProvider}. + */ + public static class Builder + implements JDBCVectorStoreQueryProvider.Builder { + protected Connection connection; + protected String collectionsTable = DEFAULT_COLLECTIONS_TABLE; + protected String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; + + /** + * Sets the connection. + * @param connection the connection + * @return the builder + */ + public Builder withConnection(Connection connection) { + this.connection = connection; + return this; + } + + /** + * Sets the collections table name. + * @param collectionsTable the collections table name + * @return the builder + */ + public Builder withCollectionsTable(String collectionsTable) { + this.collectionsTable = collectionsTable; + return this; + } + + /** + * Sets the prefix for collection tables. + * @param prefixForCollectionTables the prefix for collection tables + * @return the builder + */ + public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { + this.prefixForCollectionTables = prefixForCollectionTables; + return this; + } + + @Override + public JDBCVectorStoreDefaultQueryProvider build() { + if (connection == null) { + throw new IllegalArgumentException("connection is required"); + } + + return new JDBCVectorStoreDefaultQueryProvider(connection, collectionsTable, prefixForCollectionTables); + } + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java new file mode 100644 index 00000000..6ecb59ea --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java @@ -0,0 +1,101 @@ +package com.microsoft.semantickernel.connectors.data.jdbc; + +import javax.annotation.Nullable; + +public class JDBCVectorStoreOptions { + @Nullable + private final JDBCVectorStoreRecordCollectionFactory vectorStoreRecordCollectionFactory; + @Nullable + private final JDBCVectorStoreQueryProvider queryProvider; + + /** + * Creates a new instance of the JDBC vector store options. + * + * @param vectorStoreRecordCollectionFactory The vector store record collection factory. + */ + public JDBCVectorStoreOptions( + @Nullable JDBCVectorStoreQueryProvider queryProvider, + @Nullable JDBCVectorStoreRecordCollectionFactory vectorStoreRecordCollectionFactory) { + this.queryProvider = queryProvider; + this.vectorStoreRecordCollectionFactory = vectorStoreRecordCollectionFactory; + } + + /** + * Creates a new instance of the JDBC vector store options. + */ + public JDBCVectorStoreOptions() { + this(null, null); + } + + /** + * Gets the query provider. + * + * @return the query provider + */ + @Nullable + public JDBCVectorStoreQueryProvider getQueryProvider() { + return queryProvider; + } + + /** + * Creates a new builder. + * + * @return the builder + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Gets the vector store record collection factory. + * + * @return the vector store record collection factory + */ + @Nullable + public JDBCVectorStoreRecordCollectionFactory getVectorStoreRecordCollectionFactory() { + return vectorStoreRecordCollectionFactory; + } + + /** + * Builder for JDBC vector store options. + * + */ + public static class Builder { + @Nullable + private JDBCVectorStoreQueryProvider queryProvider; + @Nullable + private JDBCVectorStoreRecordCollectionFactory vectorStoreRecordCollectionFactory; + + /** + * Sets the query provider. + * + * @param queryProvider The query provider. + * @return The updated builder instance. + */ + public Builder withQueryProvider(JDBCVectorStoreQueryProvider queryProvider) { + this.queryProvider = queryProvider; + return this; + } + + /** + * Sets the vector store record collection factory. + * + * @param vectorStoreRecordCollectionFactory The vector store record collection factory. + * @return The updated builder instance. + */ + public Builder withVectorStoreRecordCollectionFactory( + JDBCVectorStoreRecordCollectionFactory vectorStoreRecordCollectionFactory) { + this.vectorStoreRecordCollectionFactory = vectorStoreRecordCollectionFactory; + return this; + } + + /** + * Builds the JDBC vector store options. + * + * @return The JDBC vector store options. + */ + public JDBCVectorStoreOptions build() { + return new JDBCVectorStoreOptions(queryProvider, vectorStoreRecordCollectionFactory); + } + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java new file mode 100644 index 00000000..6104de8b --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.jdbc; + +import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; +import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; +import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; + +/** + * The JDBC vector store query provider. + * Provides the necessary methods to interact with a JDBC vector store and vector store collections. + */ +public interface JDBCVectorStoreQueryProvider { + /** + * The default name for the collections table. + */ + String DEFAULT_COLLECTIONS_TABLE = "SKCollections"; + + /** + * The prefix for collection tables. + */ + String DEFAULT_PREFIX_FOR_COLLECTION_TABLES = "SKCollection_"; + + /** + * Prepares the vector store. + * Executes any necessary setup steps for the vector store. + * + * @throws SQLException if an error occurs + */ + void prepareVectorStore() throws SQLException; + + /** + * Checks if the types of the record class fields are supported. + * + * @param recordClass the record class + * @param recordDefinition the record definition + */ + void validateSupportedTypes(Class recordClass, VectorStoreRecordDefinition recordDefinition); + + /** + * Checks if a collection exists. + * + * @param collectionName the collection name + * @return true if the collection exists, false otherwise + * @throws SQLException if an error occurs + */ + boolean collectionExists(String collectionName) throws SQLException; + + /** + * Creates a collection. + * + * @param collectionName the collection name + * @param recordClass the record class + * @param recordDefinition the record definition + * @throws SQLException if an error occurs + */ + void createCollection(String collectionName, Class recordClass, VectorStoreRecordDefinition recordDefinition) throws SQLException; + + /** + * Deletes a collection. + * + * @param collectionName the collection name + * @throws SQLException if an error occurs + */ + void deleteCollection(String collectionName) throws SQLException; + + /** + * Gets the names of the collections. + * + * @return the result set + * @throws SQLException if an error occurs + */ + ResultSet getCollectionNames() throws SQLException; + + /** + * Gets the records. + * + * @param collectionName the collection name + * @param keys the keys + * @param recordDefinition the record definition + * @param options the options + * @return the result set + * @throws SQLException if an error occurs + */ + ResultSet getRecords(String collectionName, List keys, VectorStoreRecordDefinition recordDefinition, GetRecordOptions options) throws SQLException; + + /** + * Upserts records. + * + * @param collectionName the collection name + * @param records the records + * @param vectorStoreRecordDefinition the record definition + * @param options the options + * @throws SQLException if an error occurs + */ + void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition vectorStoreRecordDefinition, UpsertRecordOptions options) throws SQLException; + + /** + * Deletes records. + * + * @param collectionName the collection name + * @param keys the keys + * @param recordDefinition the record definition + * @param options the options + * @throws SQLException if an error occurs + */ + void deleteRecords(String collectionName, List keys, VectorStoreRecordDefinition recordDefinition, DeleteRecordOptions options) throws SQLException; + + /** + * The builder for the JDBC vector store query provider. + */ + interface Builder extends SemanticKernelBuilder { + + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java new file mode 100644 index 00000000..54f0b76d --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -0,0 +1,331 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.jdbc; + +import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; +import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; +import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.exceptions.SKException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +import javax.annotation.Nonnull; +import java.lang.reflect.Field; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +public class JDBCVectorStoreRecordCollection + implements SQLVectorStoreRecordCollection { + private final String collectionName; + private final VectorStoreRecordDefinition recordDefinition; + private final JDBCVectorStoreRecordCollectionOptions options; + private final JDBCVectorStoreRecordMapper vectorStoreRecordMapper; + private final JDBCVectorStoreQueryProvider queryProvider; + + /** + * Creates a new instance of the JDBCVectorRecordStore. + * If using this constructor, call {@link #prepareAsync()} before using the record collection. + * + * @param connection The JDBC connection. + * @param options The options for the store. + */ + @SuppressFBWarnings("EI_EXPOSE_REP2") + public JDBCVectorStoreRecordCollection( + @Nonnull Connection connection, + @Nonnull String collectionName, + @Nonnull JDBCVectorStoreRecordCollectionOptions options) { + this.collectionName = collectionName; + this.options = options; + + // If record definition is not provided, create one from the record class + recordDefinition = options.getRecordDefinition() == null + ? VectorStoreRecordDefinition.fromRecordClass(options.getRecordClass()) + : options.getRecordDefinition(); + + // If mapper is not provided, set a default one + if (options.getVectorStoreRecordMapper() == null) { + vectorStoreRecordMapper = JDBCVectorStoreRecordMapper.builder() + .withRecordClass(options.getRecordClass()) + .withVectorStoreRecordDefinition(recordDefinition) + .build(); + } else { + vectorStoreRecordMapper = options.getVectorStoreRecordMapper(); + } + + // If the query provider is not provided, set a default one + if (options.getQueryProvider() == null) { + this.queryProvider = new JDBCVectorStoreDefaultQueryProvider(connection); + } else { + this.queryProvider = options.getQueryProvider(); + } + + // Check if the types are supported + queryProvider.validateSupportedTypes(options.getRecordClass(), recordDefinition); + } + + /** + * Gets the name of the collection. + * + * @return The name of the collection. + */ + @Override + public String getCollectionName() { + return collectionName; + } + + /** + * Checks if the collection exists in the store. + * + * @return A Mono emitting a boolean indicating if the collection exists. + */ + @Override + public Mono collectionExistsAsync() { + return Mono.fromCallable( + () -> { + try { + return queryProvider.collectionExists(this.collectionName); + } catch (SQLException e) { + throw new SKException("Failed to check if collection exists", e); + } + }) + .subscribeOn(Schedulers.boundedElastic()); + } + + /** + * Creates the collection in the store. + * + * @return A Mono representing the completion of the creation operation. + */ + @Override + public Mono createCollectionAsync() { + return Mono.fromRunnable( + () -> { + try { + queryProvider.createCollection(this.collectionName, options.getRecordClass(), recordDefinition); + } catch (SQLException e) { + throw new SKException("Failed to create collection", e); + } + }) + .subscribeOn(Schedulers.boundedElastic()) + .then(); + } + + /** + * Creates the collection in the store if it does not exist. + * + * @return A Mono representing the completion of the creation operation. + */ + @Override + public Mono createCollectionIfNotExistsAsync() { + return collectionExistsAsync().map( + exists -> { + if (!exists) { + return createCollectionAsync(); + } + return Mono.empty(); + }) + .flatMap(mono -> mono) + .then(); + } + + /** + * Deletes the collection from the store. + * + * @return A Mono representing the completion of the deletion operation. + */ + @Override + public Mono deleteCollectionAsync() { + return Mono.fromRunnable( + () -> { + try { + queryProvider.deleteCollection(this.collectionName); + } catch (SQLException e) { + throw new SKException("Failed to delete collection", e); + } + }).subscribeOn(Schedulers.boundedElastic()).then(); + } + + /** + * Gets a record from the store. + * + * @param key The key of the record to get. + * @param options The options for getting the record. + * @return A Mono emitting the record. + */ + @Override + public Mono getAsync(String key, GetRecordOptions options) { + return this.getBatchAsync(Collections.singletonList(key), options) + .mapNotNull(records -> { + if (records.isEmpty()) { + return null; + } + return records.get(0); + }); + } + + /** + * Gets a batch of records from the store. + * + * @param keys The keys of the records to get. + * @param options The options for getting the records. + * @return A Mono emitting a collection of records. + */ + @Override + public Mono> getBatchAsync(List keys, GetRecordOptions options) { + return Mono.fromCallable( + () -> { + List records = new ArrayList<>(); + + try { + ResultSet resultSet = queryProvider.getRecords(this.collectionName, keys, recordDefinition, options); + while (resultSet.next()) { + records.add(vectorStoreRecordMapper.mapStorageModeltoRecord(resultSet)); + } + } catch (SQLException e) { + throw new SKException("Failed to get records", e); + } + + return records; + }).subscribeOn(Schedulers.boundedElastic()); + } + + protected String getKeyFromRecord(Record data) { + try { + Field keyField = data.getClass().getDeclaredField(recordDefinition.getKeyField().getName()); + keyField.setAccessible(true); + return (String) keyField.get(data); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new SKException("Failed to get key from record", e); + } + } + + /** + * Inserts or updates a record in the store. + * + * @param data The record to upsert. + * @param options The options for upserting the record. + * @return A Mono emitting the key of the upserted record. + */ + @Override + public Mono upsertAsync(Record data, UpsertRecordOptions options) { + return this.upsertBatchAsync(Collections.singletonList(data), options) + .mapNotNull(keys -> { + if (keys.isEmpty()) { + return null; + } + return keys.get(0); + }); + } + + /** + * Inserts or updates a batch of records in the store. + * + * @param data The records to upsert. + * @param options The options for upserting the records. + * @return A Mono emitting a collection of keys of the upserted records. + */ + @Override + public Mono> upsertBatchAsync(List data, UpsertRecordOptions options) { + return Mono.fromCallable( + () -> { + try { + queryProvider.upsertRecords(this.collectionName, data, recordDefinition, options); + + return data.stream().map(this::getKeyFromRecord).collect(Collectors.toList()); + } catch (SQLException e) { + throw new SKException("Failed to upsert records", e); + } + }) + .subscribeOn(Schedulers.boundedElastic()); + } + + /** + * Deletes a record from the store. + * + * @param key The key of the record to delete. + * @param options The options for deleting the record. + * @return A Mono representing the completion of the deletion operation. + */ + @Override + public Mono deleteAsync(String key, DeleteRecordOptions options) { + return this.deleteBatchAsync(Collections.singletonList(key), options); + } + + /** + * Deletes a batch of records from the store. + * + * @param keys The keys of the records to delete. + * @param options The options for deleting the records. + * @return A Mono representing the completion of the deletion operation. + */ + @Override + public Mono deleteBatchAsync(List keys, DeleteRecordOptions options) { + return Mono.fromRunnable( + () -> { + try { + queryProvider.deleteRecords(this.collectionName, keys, recordDefinition, options); + } catch (SQLException e) { + throw new SKException("Failed to delete records", e); + } + }).subscribeOn(Schedulers.boundedElastic()).then(); + } + + /** + * Prepares the collection for use. + * + * @return A Mono representing the completion of the preparation operation. + */ + @Override + public Mono prepareAsync() { + return Mono.fromRunnable(() -> { + try { + queryProvider.prepareVectorStore(); + } catch (SQLException e) { + throw new SKException("Failed to prepare vector store record collection", e); + } + }).subscribeOn(Schedulers.boundedElastic()).then(); + } + + public static class Builder implements SemanticKernelBuilder> { + private Connection connection; + private String collectionName; + private JDBCVectorStoreRecordCollectionOptions options; + + public Builder withConnection(Connection connection) { + this.connection = connection; + return this; + } + + public Builder withCollectionName(String collectionName) { + this.collectionName = collectionName; + return this; + } + + public Builder withOptions(JDBCVectorStoreRecordCollectionOptions options) { + this.options = options; + return this; + } + + @Override + public JDBCVectorStoreRecordCollection build() { + if (connection == null) { + throw new IllegalArgumentException("connection is required"); + } + if (collectionName == null) { + throw new IllegalArgumentException("collectionName is required"); + } + if (options == null) { + throw new IllegalArgumentException("options is required"); + } + + return new JDBCVectorStoreRecordCollection<>(connection, collectionName, options); + } + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java new file mode 100644 index 00000000..4d92eb25 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java @@ -0,0 +1,19 @@ +package com.microsoft.semantickernel.connectors.data.jdbc; + +import java.sql.Connection; + +/** + * Factory for creating JDBC vector store record collections. + */ +public interface JDBCVectorStoreRecordCollectionFactory { + /** + * Creates a new JDBC vector store record collection. + * + * @param options The options for the collection. + * @return The new JDBC vector store record collection. + */ + JDBCVectorStoreRecordCollection createVectorStoreRecordCollection( + Connection connection, + String collectionName, + JDBCVectorStoreRecordCollectionOptions options); +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java new file mode 100644 index 00000000..6fe6dbaf --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.jdbc; + +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; + +public class JDBCVectorStoreRecordCollectionOptions { + private final Class recordClass; + private final JDBCVectorStoreRecordMapper vectorStoreRecordMapper; + private final VectorStoreRecordDefinition recordDefinition; + private final JDBCVectorStoreQueryProvider queryProvider; + + public JDBCVectorStoreRecordCollectionOptions( + Class recordClass, + VectorStoreRecordDefinition recordDefinition, + JDBCVectorStoreRecordMapper vectorStoreRecordMapper, + JDBCVectorStoreQueryProvider queryProvider) { + this.recordClass = recordClass; + this.recordDefinition = recordDefinition; + this.vectorStoreRecordMapper = vectorStoreRecordMapper; + this.queryProvider = queryProvider; + } + + /** + * Creates a new builder. + * @param the record type + * @return the builder + */ + public static Builder builder() { + return new Builder<>(); + } + + /** + * Gets the record class. + * @return the record class + */ + public Class getRecordClass() { + return recordClass; + } + + /** + * Gets the record definition. + * @return the record definition + */ + public VectorStoreRecordDefinition getRecordDefinition() { + return recordDefinition; + } + + /** + * Gets the vector store record mapper. + * @return the vector store record mapper + */ + public JDBCVectorStoreRecordMapper getVectorStoreRecordMapper() { + return vectorStoreRecordMapper; + } + + /** + * Gets the query provider. + * @return the query provider + */ + public JDBCVectorStoreQueryProvider getQueryProvider() { + return queryProvider; + } + + public static class Builder { + private Class recordClass; + private VectorStoreRecordDefinition recordDefinition; + private JDBCVectorStoreRecordMapper vectorStoreRecordMapper; + private JDBCVectorStoreQueryProvider queryProvider; + + /** + * Sets the record class. + * @param recordClass the record class + * @return the builder + */ + public Builder withRecordClass(Class recordClass) { + this.recordClass = recordClass; + return this; + } + + /** + * Sets the record definition. + * @param recordDefinition the record definition + * @return the builder + */ + public Builder withRecordDefinition(VectorStoreRecordDefinition recordDefinition) { + this.recordDefinition = recordDefinition; + return this; + } + + /** + * Sets the vector store record mapper. + * @param vectorStoreRecordMapper the vector store record mapper + * @return the builder + */ + public Builder withVectorStoreRecordMapper( + JDBCVectorStoreRecordMapper vectorStoreRecordMapper) { + this.vectorStoreRecordMapper = vectorStoreRecordMapper; + return this; + } + + /** + * Sets the query provider. + * @param queryProvider the query provider + * @return the builder + */ + public Builder withQueryProvider(JDBCVectorStoreQueryProvider queryProvider) { + this.queryProvider = queryProvider; + return this; + } + + /** + * Builds the options. + * @return the options + */ + public JDBCVectorStoreRecordCollectionOptions build() { + if (recordClass == null) { + throw new IllegalArgumentException("recordClass is required"); + } + + return new JDBCVectorStoreRecordCollectionOptions<>( + recordClass, + recordDefinition, + vectorStoreRecordMapper, + queryProvider + ); + } + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java new file mode 100644 index 00000000..6eff0c7d --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordMapper.java @@ -0,0 +1,150 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.jdbc; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.exceptions.SKException; +import com.microsoft.semantickernel.data.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; + +import java.sql.ResultSetMetaData; +import java.util.List; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.function.Function; + +public class JDBCVectorStoreRecordMapper + extends VectorStoreRecordMapper { + + /** + * Constructs a new instance of the VectorStoreRecordMapper. + * + * @param storageModelToRecordMapper the function to convert a storage model to a record + */ + protected JDBCVectorStoreRecordMapper(Function storageModelToRecordMapper) { + super(null, storageModelToRecordMapper); + } + + /** + * Creates a new builder. + * + * @param the record type + * @return the builder + */ + public static Builder builder() { + return new Builder<>(); + } + + /** + * Operation not supported. + */ + @Override + public ResultSet mapRecordToStorageModel(Record record) { + throw new UnsupportedOperationException("Not implemented"); + } + + public static class Builder + implements SemanticKernelBuilder> { + private Class recordClass; + private VectorStoreRecordDefinition vectorStoreRecordDefinition; + + /** + * Sets the record class. + * + * @param recordClass the record class + * @return the builder + */ + public Builder withRecordClass(Class recordClass) { + this.recordClass = recordClass; + return this; + } + + /** + * Sets the vector store record definition. + * + * @param vectorStoreRecordDefinition the vector store record definition + * @return the builder + */ + public Builder withVectorStoreRecordDefinition( + VectorStoreRecordDefinition vectorStoreRecordDefinition) { + this.vectorStoreRecordDefinition = vectorStoreRecordDefinition; + return this; + } + + /** + * Builds the {@link JDBCVectorStoreRecordMapper}. + * + * @return the {@link JDBCVectorStoreRecordMapper} + */ + public JDBCVectorStoreRecordMapper build() { + if (recordClass == null) { + throw new IllegalArgumentException("recordClass is required"); + } + if (vectorStoreRecordDefinition == null) { + throw new IllegalArgumentException("vectorStoreRecordDefinition is required"); + } + + return new JDBCVectorStoreRecordMapper<>( + resultSet -> { + try { + Constructor constructor = recordClass.getDeclaredConstructor(); + constructor.setAccessible(true); + Record record = (Record) constructor.newInstance(); + + // Select fields from the record definition. + // Check if vector fields are present in the result set. + List fields; + ResultSetMetaData metaData = resultSet.getMetaData(); + if (metaData.getColumnCount() == vectorStoreRecordDefinition.getAllFields() + .size()) { + fields = vectorStoreRecordDefinition.getAllFields(); + } else { + fields = vectorStoreRecordDefinition.getNonVectorFields(); + } + + for (VectorStoreRecordField field : fields) { + Object value = resultSet.getObject(field.getName()); + Field recordField = recordClass.getDeclaredField(field.getName()); + recordField.setAccessible(true); + + // If the field is a vector field, deserialize the JSON string + if (field instanceof VectorStoreRecordVectorField) { + Class vectorType = recordField.getType(); + + // If the vector type is a string, set the value directly + if (vectorType.equals(String.class)) { + recordField.set(record, value); + } else { + // Deserialize the JSON string to the vector type + recordField.set(record, + new ObjectMapper().readValue((String) value, vectorType)); + } + } else { + recordField.set(record, value); + } + } + + return record; + } catch (NoSuchMethodException e) { + throw new SKException("Default constructor not found.", e); + } catch (InstantiationException | InvocationTargetException e) { + throw new SKException(String.format( + "SK cannot instantiate %s. A custom mapper is required.", + recordClass.getName()), e); + } catch (JsonProcessingException e) { + throw new SKException(String.format( + "SK cannot deserialize %s. A custom mapper is required.", + recordClass.getName()), e); + } catch (SQLException | NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + }); + } + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/MySQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/MySQLVectorStoreQueryProvider.java new file mode 100644 index 00000000..f7211f24 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/MySQLVectorStoreQueryProvider.java @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.jdbc; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.exceptions.SKException; + +import java.lang.reflect.Field; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.List; + +public class MySQLVectorStoreQueryProvider extends + JDBCVectorStoreDefaultQueryProvider implements JDBCVectorStoreQueryProvider { + + public MySQLVectorStoreQueryProvider(Connection connection, String collectionsTable, String prefixForCollectionTables) { + super(connection, collectionsTable, prefixForCollectionTables); + } + + /** + * Creates a new builder. + * @return the builder + */ + public static Builder builder() { + return new Builder(); + } + + private void setStatementValues(PreparedStatement statement, Object record, List fields) { + for (int i = 0; i < fields.size(); ++i) { + VectorStoreRecordField field = fields.get(i); + try { + Field recordField = record.getClass().getDeclaredField(field.getName()); + recordField.setAccessible(true); + Object value = recordField.get(record); + + if (field instanceof VectorStoreRecordKeyField) { + statement.setObject(i + 1, (String) value); + } else if (field instanceof VectorStoreRecordVectorField) { + Class vectorType = record.getClass().getDeclaredField(field.getName()).getType(); + + // If the vector field is other than String, serialize it to JSON + if (vectorType.equals(String.class)) { + statement.setObject(i + 1, value); + } else { + // Serialize the vector to JSON + statement.setObject(i + 1, new ObjectMapper().writeValueAsString(value)); + } + } else { + statement.setObject(i + 1, value); + } + } catch (NoSuchFieldException | IllegalAccessException | SQLException e) { + throw new SKException("Failed to set statement values", e); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + } + + @Override + public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) throws SQLException { + List fields = recordDefinition.getAllFields(); + + StringBuilder onDuplicateKeyUpdate = new StringBuilder(); + for (int i = 0; i < fields.size(); ++i) { + VectorStoreRecordField field = fields.get(i); + if (i > 0) { + onDuplicateKeyUpdate.append(", "); + } + + onDuplicateKeyUpdate.append(field.getName()).append(" = VALUES(").append(field.getName()).append(")"); + } + + String query = "INSERT INTO " + getCollectionTableName(collectionName) + + " (" + getQueryColumnsFromFields(fields) + ")" + + " VALUES (" + getWildcardString(fields.size()) + ")" + + " ON DUPLICATE KEY UPDATE " + onDuplicateKeyUpdate; + + PreparedStatement statement = connection.prepareStatement(query); + + for (Object record : records) { + setStatementValues(statement, record, recordDefinition.getAllFields()); + statement.addBatch(); + } + + statement.executeBatch(); + } + + public static class Builder + extends JDBCVectorStoreDefaultQueryProvider.Builder { + public MySQLVectorStoreQueryProvider build() { + if (connection == null) { + throw new IllegalArgumentException("connection is required"); + } + + return new MySQLVectorStoreQueryProvider(connection, collectionsTable, prefixForCollectionTables); + } + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java new file mode 100644 index 00000000..dd19f787 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java @@ -0,0 +1,15 @@ +package com.microsoft.semantickernel.connectors.data.jdbc; + +import com.microsoft.semantickernel.data.VectorStore; +import com.microsoft.semantickernel.data.VectorStoreRecordCollection; +import reactor.core.publisher.Mono; + +public interface SQLVectorStore> extends VectorStore { + + /** + * Prepares the vector store. + * + * @return A {@link Mono} that completes when the vector store is prepared to be used. + */ + Mono prepareAsync(); +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java new file mode 100644 index 00000000..3583a273 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java @@ -0,0 +1,14 @@ +package com.microsoft.semantickernel.connectors.data.jdbc; + +import com.microsoft.semantickernel.data.VectorStoreRecordCollection; +import reactor.core.publisher.Mono; + +public interface SQLVectorStoreRecordCollection extends VectorStoreRecordCollection { + + /** + * Prepares the vector store record collection. + * + * @return A {@link Mono} that completes when the vector store record collection is prepared to be used. + */ + Mono prepareAsync(); +} From 4fe5a421ad4cb7d050216071a252d3611a8a1f52 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 1 Aug 2024 02:28:23 -0700 Subject: [PATCH 12/28] Format --- .../services/audio/AudioToTextService.java | 3 +- .../services/openai/OpenAiServiceBuilder.java | 5 +- .../textcompletion/TextGenerationService.java | 3 +- ...reAISearchVectorStoreRecordCollection.java | 13 +- .../connectors/data/jdbc/JDBCVectorStore.java | 69 +++--- .../JDBCVectorStoreDefaultQueryProvider.java | 214 +++++++++++------- .../data/jdbc/JDBCVectorStoreOptions.java | 1 + .../jdbc/JDBCVectorStoreQueryProvider.java | 41 ++-- .../jdbc/JDBCVectorStoreRecordCollection.java | 99 +++----- ...DBCVectorStoreRecordCollectionFactory.java | 1 + ...DBCVectorStoreRecordCollectionOptions.java | 72 +++++- .../jdbc/MySQLVectorStoreQueryProvider.java | 44 ++-- .../connectors/data/jdbc/SQLVectorStore.java | 4 +- .../jdbc/SQLVectorStoreRecordCollection.java | 4 +- .../RedisVectorStoreRecordCollection.java | 9 +- .../VolatileVectorStoreRecordCollection.java | 3 +- .../VectorStoreRecordDefinition.java | 31 +-- 17 files changed, 351 insertions(+), 265 deletions(-) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/AudioToTextService.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/AudioToTextService.java index 70224596..871d4cb4 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/AudioToTextService.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/audio/AudioToTextService.java @@ -33,7 +33,8 @@ static Builder builder() { /** * Builder for the AudioToTextService. */ - abstract class Builder extends OpenAiServiceBuilder { + abstract class Builder + extends OpenAiServiceBuilder { } } diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/openai/OpenAiServiceBuilder.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/openai/OpenAiServiceBuilder.java index e42b2c6e..5386cd83 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/openai/OpenAiServiceBuilder.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/openai/OpenAiServiceBuilder.java @@ -11,8 +11,9 @@ * @param The service type * @param The builder type */ -public abstract class OpenAiServiceBuilder> implements - +public abstract class OpenAiServiceBuilder> + implements + SemanticKernelBuilder { @Nullable diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textcompletion/TextGenerationService.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textcompletion/TextGenerationService.java index 8010b67f..0ab08f5f 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textcompletion/TextGenerationService.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/textcompletion/TextGenerationService.java @@ -61,6 +61,7 @@ Flux getStreamingTextContentsAsync( /** * Builder for a TextGenerationService */ - abstract class Builder extends OpenAiServiceBuilder { + abstract class Builder + extends OpenAiServiceBuilder { } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index 70d517ac..5155299d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -91,14 +91,15 @@ public AzureAISearchVectorStoreRecordCollection( // Validate supported types VectorStoreRecordDefinition.validateSupportedTypes( - Collections.singletonList(recordDefinition.getKeyDeclaredField(this.options.getRecordClass())), - supportedKeyTypes); + Collections + .singletonList(recordDefinition.getKeyDeclaredField(this.options.getRecordClass())), + supportedKeyTypes); VectorStoreRecordDefinition.validateSupportedTypes( - recordDefinition.getDataDeclaredFields(this.options.getRecordClass()), - supportedDataTypes); + recordDefinition.getDataDeclaredFields(this.options.getRecordClass()), + supportedDataTypes); VectorStoreRecordDefinition.validateSupportedTypes( - recordDefinition.getVectorDeclaredFields(this.options.getRecordClass()), - supportedVectorTypes); + recordDefinition.getVectorDeclaredFields(this.options.getRecordClass()), + supportedVectorTypes); // Add non-vector fields to the list nonVectorFields.add(this.recordDefinition.getKeyField().getName()); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java index df2155a0..7e02f4a8 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java @@ -1,15 +1,14 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import com.microsoft.semantickernel.exceptions.SKException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.List; /** @@ -27,14 +26,18 @@ public class JDBCVectorStore implements SQLVectorStore JDBCVectorStoreRecordCollection getCollection( - @Nonnull String collectionName, - @Nonnull Class recordClass, - @Nullable VectorStoreRecordDefinition recordDefinition) { + @Nonnull String collectionName, + @Nonnull Class recordClass, + @Nullable VectorStoreRecordDefinition recordDefinition) { if (this.options != null && this.options.getVectorStoreRecordCollectionFactory() != null) { return this.options.getVectorStoreRecordCollectionFactory() @@ -67,20 +70,20 @@ public JDBCVectorStoreRecordCollection getCollection( connection, collectionName, JDBCVectorStoreRecordCollectionOptions.builder() - .withRecordClass(recordClass) - .withRecordDefinition(recordDefinition) - .withQueryProvider(this.queryProvider) - .build()); - } - - return new JDBCVectorStoreRecordCollection<>( - connection, - collectionName, - JDBCVectorStoreRecordCollectionOptions.builder() .withRecordClass(recordClass) .withRecordDefinition(recordDefinition) .withQueryProvider(this.queryProvider) .build()); + } + + return new JDBCVectorStoreRecordCollection<>( + connection, + collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(recordClass) + .withRecordDefinition(recordDefinition) + .withQueryProvider(this.queryProvider) + .build()); } /** @@ -90,30 +93,17 @@ public JDBCVectorStoreRecordCollection getCollection( */ @Override public Mono> getCollectionNamesAsync() { - return Mono.fromCallable(() -> { - List collectionNames = new ArrayList<>(); - try { - ResultSet resultSet = queryProvider.getCollectionNames(); - while (resultSet.next()) { - collectionNames.add(resultSet.getString(1)); - } - - return collectionNames; - } catch (SQLException e) { - throw new SKException("Failed to get collection names.", e); - } - }); + return Mono.fromCallable(queryProvider::getCollectionNames) + .subscribeOn(Schedulers.boundedElastic()); } + /** + * Prepares the vector store. + */ @Override public Mono prepareAsync() { - return Mono.fromRunnable(() -> { - try { - queryProvider.prepareVectorStore(); - } catch (SQLException e) { - throw new SKException("Failed to prepare vector store.", e); - } - }); + return Mono.fromRunnable(queryProvider::prepareVectorStore) + .subscribeOn(Schedulers.boundedElastic()).then(); } /** @@ -129,6 +119,7 @@ public static class Builder { * @param connection the connection * @return the builder */ + @SuppressFBWarnings("EI_EXPOSE_REP2") public Builder withConnection(Connection connection) { this.connection = connection; return this; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java index ceb2c544..9bf75afd 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java @@ -16,6 +16,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.time.OffsetDateTime; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -52,34 +53,20 @@ public class JDBCVectorStoreDefaultQueryProvider supportedVectorTypes.put(List.class, "TEXT"); supportedVectorTypes.put(Collection.class, "TEXT"); } - protected final Connection connection; protected final String collectionsTable; protected final String prefixForCollectionTables; @SuppressFBWarnings("EI_EXPOSE_REP2") - public JDBCVectorStoreDefaultQueryProvider( - @Nonnull Connection connection, - @Nonnull String collectionsTable, - @Nonnull String prefixForCollectionTables) { + protected JDBCVectorStoreDefaultQueryProvider( + @Nonnull Connection connection, + @Nonnull String collectionsTable, + @Nonnull String prefixForCollectionTables) { this.connection = connection; - // Validate table name - if (!isValidSQLIdentifier(collectionsTable)) { - throw new IllegalArgumentException("Invalid collections table name: " + collectionsTable); - } - if (!isValidSQLIdentifier(prefixForCollectionTables)) { - throw new IllegalArgumentException("Invalid prefix for collection tables: " + prefixForCollectionTables); - } - this.collectionsTable = collectionsTable; this.prefixForCollectionTables = prefixForCollectionTables; } - public JDBCVectorStoreDefaultQueryProvider( - @Nonnull Connection connection) { - this(connection, DEFAULT_COLLECTIONS_TABLE, DEFAULT_PREFIX_FOR_COLLECTION_TABLES); - } - /** * Creates a new builder. * @return the builder @@ -116,8 +103,8 @@ protected String getQueryColumnsFromFields(List fields) protected String getColumnNamesAndTypes(List fields, Map, String> types) { List columns = fields.stream() - .map(field -> field.getName() + " " + types.get(field.getType())) - .collect(Collectors.toList()); + .map(field -> field.getName() + " " + types.get(field.getType())) + .collect(Collectors.toList()); return String.join(", ", columns); } @@ -127,80 +114,126 @@ protected String getCollectionTableName(String collectionName) { } @Override - public void prepareVectorStore() throws SQLException { - String createCollectionsTable = - "CREATE TABLE IF NOT EXISTS " + collectionsTable - + " (collectionId VARCHAR(255) PRIMARY KEY);"; - - PreparedStatement createTable = connection.prepareStatement(createCollectionsTable); - createTable.execute(); + public void prepareVectorStore() { + String createCollectionsTable = "CREATE TABLE IF NOT EXISTS " + collectionsTable + + " (collectionId VARCHAR(255) PRIMARY KEY);"; + + try (PreparedStatement createTable = connection.prepareStatement(createCollectionsTable)) { + createTable.execute(); + } catch (SQLException e) { + throw new SKException("Failed to prepare vector store", e); + } } @Override - public void validateSupportedTypes(Class recordClass, VectorStoreRecordDefinition recordDefinition) { + public void validateSupportedTypes(Class recordClass, + VectorStoreRecordDefinition recordDefinition) { VectorStoreRecordDefinition.validateSupportedTypes( - Collections.singletonList(recordDefinition.getKeyDeclaredField(recordClass)), supportedKeyTypes.keySet()); + Collections.singletonList(recordDefinition.getKeyDeclaredField(recordClass)), + supportedKeyTypes.keySet()); VectorStoreRecordDefinition.validateSupportedTypes( - recordDefinition.getDataDeclaredFields(recordClass), supportedDataTypes.keySet()); + recordDefinition.getDataDeclaredFields(recordClass), supportedDataTypes.keySet()); VectorStoreRecordDefinition.validateSupportedTypes( - recordDefinition.getVectorDeclaredFields(recordClass), supportedVectorTypes.keySet()); + recordDefinition.getVectorDeclaredFields(recordClass), supportedVectorTypes.keySet()); } @Override - public boolean collectionExists(String collectionName) throws SQLException { + public boolean collectionExists(String collectionName) { + validateSQLidentifier(collectionsTable); + String query = "SELECT 1 FROM " + collectionsTable + " WHERE collectionId = ?"; - PreparedStatement statement = connection.prepareStatement(query); - statement.setObject(1, collectionName); + try (PreparedStatement statement = connection.prepareStatement(query)) { + statement.setObject(1, collectionName); - return statement.executeQuery().next(); + return statement.executeQuery().next(); + } catch (SQLException e) { + throw new SKException("Failed to check if collection exists", e); + } } @Override - public void createCollection(String collectionName, Class recordClass, VectorStoreRecordDefinition recordDefinition) throws SQLException { + public void createCollection(String collectionName, Class recordClass, + VectorStoreRecordDefinition recordDefinition) { + validateSQLidentifier(collectionName); + Field keyDeclaredField = recordDefinition.getKeyDeclaredField(recordClass); List dataDeclaredFields = recordDefinition.getDataDeclaredFields(recordClass); List vectorDeclaredFields = recordDefinition.getVectorDeclaredFields(recordClass); - String createStorageTable = - "CREATE TABLE IF NOT EXISTS " + getCollectionTableName(collectionName) - + " (" + keyDeclaredField.getName() + " VARCHAR(255) PRIMARY KEY, " - + getColumnNamesAndTypes(dataDeclaredFields, supportedDataTypes) + ", " - + getColumnNamesAndTypes(vectorDeclaredFields, supportedVectorTypes) + ");"; + String createStorageTable = "CREATE TABLE IF NOT EXISTS " + + getCollectionTableName(collectionName) + + " (" + keyDeclaredField.getName() + " VARCHAR(255) PRIMARY KEY, " + + getColumnNamesAndTypes(dataDeclaredFields, supportedDataTypes) + ", " + + getColumnNamesAndTypes(vectorDeclaredFields, supportedVectorTypes) + ");"; - PreparedStatement createTable = connection.prepareStatement(createStorageTable); + String insertCollectionQuery = "INSERT INTO " + collectionsTable + + " (collectionId) VALUES (?)"; - String insertCollectionQuery = "INSERT INTO " + collectionsTable + " (collectionId) VALUES (?)"; - PreparedStatement insert = connection.prepareStatement(insertCollectionQuery); - insert.setObject(1, collectionName); + try (PreparedStatement createTable = connection.prepareStatement(createStorageTable)) { + createTable.execute(); + } catch (SQLException e) { + throw new SKException("Failed to create collection", e); + } - createTable.execute(); - insert.execute(); + try (PreparedStatement insert = connection.prepareStatement(insertCollectionQuery)) { + insert.setObject(1, collectionName); + insert.execute(); + } catch (SQLException e) { + throw new SKException("Failed to insert collection", e); + } } @Override - public void deleteCollection(String collectionName) throws SQLException { - String deleteCollectionOperation = "DELETE FROM " + collectionsTable + " WHERE collectionId = ?"; - String dropTableOperation = "DROP TABLE " + getCollectionTableName(collectionName); + public void deleteCollection(String collectionName) { + validateSQLidentifier(collectionsTable); + validateSQLidentifier(getCollectionTableName(collectionName)); - PreparedStatement deleteCollection = connection.prepareStatement(deleteCollectionOperation); - deleteCollection.setObject(1, collectionName); + String deleteCollectionOperation = "DELETE FROM " + collectionsTable + + " WHERE collectionId = ?"; + String dropTableOperation = "DROP TABLE " + getCollectionTableName(collectionName); - PreparedStatement dropTable = connection.prepareStatement(dropTableOperation); + try (PreparedStatement deleteCollection = connection + .prepareStatement(deleteCollectionOperation)) { + deleteCollection.setObject(1, collectionName); + deleteCollection.execute(); + } catch (SQLException e) { + throw new SKException("Failed to delete collection", e); + } - dropTable.execute(); - deleteCollection.execute(); + try (PreparedStatement dropTable = connection.prepareStatement(dropTableOperation)) { + dropTable.execute(); + } catch (SQLException e) { + throw new SKException("Failed to drop table", e); + } } @Override - public ResultSet getCollectionNames() throws SQLException { + public List getCollectionNames() { + validateSQLidentifier(collectionsTable); + String query = "SELECT collectionId FROM " + collectionsTable; - return connection.prepareStatement(query).executeQuery(); + try (PreparedStatement statement = connection.prepareStatement(query)) { + List collectionNames = new ArrayList<>(); + ResultSet resultSet = statement.executeQuery(); + + while (resultSet.next()) { + collectionNames.add(resultSet.getString(1)); + } + + return Collections.unmodifiableList(collectionNames); + } catch (SQLException e) { + throw new SKException("Failed to get collection names", e); + } } @Override - public ResultSet getRecords(String collectionName, List keys, VectorStoreRecordDefinition recordDefinition, GetRecordOptions options) throws SQLException { + public List getRecords(String collectionName, List keys, + VectorStoreRecordDefinition recordDefinition, JDBCVectorStoreRecordMapper mapper, + GetRecordOptions options) { + validateSQLidentifier(getCollectionTableName(collectionName)); + List fields; if (options == null || options.includeVectors()) { fields = recordDefinition.getAllFields(); @@ -209,48 +242,59 @@ public ResultSet getRecords(String collectionName, List keys, VectorStor } String query = "SELECT " + getQueryColumnsFromFields(fields) - + " FROM " + getCollectionTableName(collectionName) - + " WHERE " + recordDefinition.getKeyField().getName() - + " IN (" + getWildcardString(keys.size()) + ")"; + + " FROM " + getCollectionTableName(collectionName) + + " WHERE " + recordDefinition.getKeyField().getName() + + " IN (" + getWildcardString(keys.size()) + ")"; - PreparedStatement statement = connection.prepareStatement(query); - for (int i = 0; i < keys.size(); ++i) { - try { + try (PreparedStatement statement = connection.prepareStatement(query)) { + for (int i = 0; i < keys.size(); ++i) { statement.setObject(i + 1, keys.get(i)); - } catch (SQLException e) { - throw new SKException("Failed to set statement values", e); } - } - return statement.executeQuery(); + List records = new ArrayList<>(); + ResultSet resultSet = statement.executeQuery(); + + while (resultSet.next()) { + records.add(mapper.mapStorageModeltoRecord(resultSet)); + } + + return Collections.unmodifiableList(records); + } catch (SQLException e) { + throw new SKException("Failed to set statement values", e); + } } @Override - public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) throws SQLException { + public void upsertRecords(String collectionName, List records, + VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { throw new UnsupportedOperationException( - "Upsert is not supported. Try with a specific query provider."); + "Upsert is not supported. Try with a specific query provider."); } @Override - public void deleteRecords(String collectionName, List keys, VectorStoreRecordDefinition recordDefinition, DeleteRecordOptions options) throws SQLException { + public void deleteRecords(String collectionName, List keys, + VectorStoreRecordDefinition recordDefinition, DeleteRecordOptions options) { + validateSQLidentifier(getCollectionTableName(collectionName)); + String query = "DELETE FROM " + getCollectionTableName(collectionName) - + " WHERE " + recordDefinition.getKeyField().getName() - + " IN (" + getWildcardString(keys.size()) + ")"; + + " WHERE " + recordDefinition.getKeyField().getName() + + " IN (" + getWildcardString(keys.size()) + ")"; - PreparedStatement statement = connection.prepareStatement(query); - for (int i = 0; i < keys.size(); ++i) { - try { + try (PreparedStatement statement = connection.prepareStatement(query)) { + for (int i = 0; i < keys.size(); ++i) { statement.setObject(i + 1, keys.get(i)); - } catch (SQLException e) { - throw new SKException("Failed to set statement values", e); } - } - statement.execute(); + statement.execute(); + } catch (SQLException e) { + throw new SKException("Failed to set statement values", e); + } } - public static boolean isValidSQLIdentifier(String identifier) { - return identifier.matches("[a-zA-Z_][a-zA-Z0-9_]*"); + public static void validateSQLidentifier(String identifier) { + if (!identifier.matches("[a-zA-Z_][a-zA-Z0-9_]*")) { + throw new IllegalArgumentException("Invalid SQL identifier: " + identifier); + } } /** @@ -267,6 +311,7 @@ public static class Builder * @param connection the connection * @return the builder */ + @SuppressFBWarnings("EI_EXPOSE_REP2") public Builder withConnection(Connection connection) { this.connection = connection; return this; @@ -278,6 +323,7 @@ public Builder withConnection(Connection connection) { * @return the builder */ public Builder withCollectionsTable(String collectionsTable) { + validateSQLidentifier(collectionsTable); this.collectionsTable = collectionsTable; return this; } @@ -288,6 +334,7 @@ public Builder withCollectionsTable(String collectionsTable) { * @return the builder */ public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { + validateSQLidentifier(prefixForCollectionTables); this.prefixForCollectionTables = prefixForCollectionTables; return this; } @@ -298,7 +345,8 @@ public JDBCVectorStoreDefaultQueryProvider build() { throw new IllegalArgumentException("connection is required"); } - return new JDBCVectorStoreDefaultQueryProvider(connection, collectionsTable, prefixForCollectionTables); + return new JDBCVectorStoreDefaultQueryProvider(connection, collectionsTable, + prefixForCollectionTables); } } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java index 6ecb59ea..580868e9 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; import javax.annotation.Nullable; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index 6104de8b..fd75dac7 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -7,8 +7,6 @@ import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; -import java.sql.ResultSet; -import java.sql.SQLException; import java.util.List; /** @@ -29,10 +27,8 @@ public interface JDBCVectorStoreQueryProvider { /** * Prepares the vector store. * Executes any necessary setup steps for the vector store. - * - * @throws SQLException if an error occurs */ - void prepareVectorStore() throws SQLException; + void prepareVectorStore(); /** * Checks if the types of the record class fields are supported. @@ -47,9 +43,8 @@ public interface JDBCVectorStoreQueryProvider { * * @param collectionName the collection name * @return true if the collection exists, false otherwise - * @throws SQLException if an error occurs */ - boolean collectionExists(String collectionName) throws SQLException; + boolean collectionExists(String collectionName); /** * Creates a collection. @@ -57,37 +52,37 @@ public interface JDBCVectorStoreQueryProvider { * @param collectionName the collection name * @param recordClass the record class * @param recordDefinition the record definition - * @throws SQLException if an error occurs */ - void createCollection(String collectionName, Class recordClass, VectorStoreRecordDefinition recordDefinition) throws SQLException; + void createCollection(String collectionName, Class recordClass, + VectorStoreRecordDefinition recordDefinition); /** * Deletes a collection. * * @param collectionName the collection name - * @throws SQLException if an error occurs */ - void deleteCollection(String collectionName) throws SQLException; + void deleteCollection(String collectionName); /** - * Gets the names of the collections. + * Gets the collection names. * - * @return the result set - * @throws SQLException if an error occurs + * @return the collection names */ - ResultSet getCollectionNames() throws SQLException; + List getCollectionNames(); /** - * Gets the records. + * Gets records. * * @param collectionName the collection name * @param keys the keys * @param recordDefinition the record definition + * @param mapper the mapper * @param options the options - * @return the result set - * @throws SQLException if an error occurs + * @return the records */ - ResultSet getRecords(String collectionName, List keys, VectorStoreRecordDefinition recordDefinition, GetRecordOptions options) throws SQLException; + List getRecords(String collectionName, List keys, + VectorStoreRecordDefinition recordDefinition, JDBCVectorStoreRecordMapper mapper, + GetRecordOptions options); /** * Upserts records. @@ -96,9 +91,9 @@ public interface JDBCVectorStoreQueryProvider { * @param records the records * @param vectorStoreRecordDefinition the record definition * @param options the options - * @throws SQLException if an error occurs */ - void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition vectorStoreRecordDefinition, UpsertRecordOptions options) throws SQLException; + void upsertRecords(String collectionName, List records, + VectorStoreRecordDefinition vectorStoreRecordDefinition, UpsertRecordOptions options); /** * Deletes records. @@ -107,9 +102,9 @@ public interface JDBCVectorStoreQueryProvider { * @param keys the keys * @param recordDefinition the record definition * @param options the options - * @throws SQLException if an error occurs */ - void deleteRecords(String collectionName, List keys, VectorStoreRecordDefinition recordDefinition, DeleteRecordOptions options) throws SQLException; + void deleteRecords(String collectionName, List keys, + VectorStoreRecordDefinition recordDefinition, DeleteRecordOptions options); /** * The builder for the JDBC vector store query provider. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java index 54f0b76d..0096794c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -14,9 +14,6 @@ import javax.annotation.Nonnull; import java.lang.reflect.Field; import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -38,30 +35,32 @@ public class JDBCVectorStoreRecordCollection */ @SuppressFBWarnings("EI_EXPOSE_REP2") public JDBCVectorStoreRecordCollection( - @Nonnull Connection connection, - @Nonnull String collectionName, - @Nonnull JDBCVectorStoreRecordCollectionOptions options) { + @Nonnull Connection connection, + @Nonnull String collectionName, + @Nonnull JDBCVectorStoreRecordCollectionOptions options) { this.collectionName = collectionName; this.options = options; // If record definition is not provided, create one from the record class recordDefinition = options.getRecordDefinition() == null - ? VectorStoreRecordDefinition.fromRecordClass(options.getRecordClass()) - : options.getRecordDefinition(); + ? VectorStoreRecordDefinition.fromRecordClass(options.getRecordClass()) + : options.getRecordDefinition(); // If mapper is not provided, set a default one if (options.getVectorStoreRecordMapper() == null) { vectorStoreRecordMapper = JDBCVectorStoreRecordMapper.builder() - .withRecordClass(options.getRecordClass()) - .withVectorStoreRecordDefinition(recordDefinition) - .build(); + .withRecordClass(options.getRecordClass()) + .withVectorStoreRecordDefinition(recordDefinition) + .build(); } else { vectorStoreRecordMapper = options.getVectorStoreRecordMapper(); } - + // If the query provider is not provided, set a default one if (options.getQueryProvider() == null) { - this.queryProvider = new JDBCVectorStoreDefaultQueryProvider(connection); + this.queryProvider = JDBCVectorStoreDefaultQueryProvider.builder() + .withConnection(connection) + .build(); } else { this.queryProvider = options.getQueryProvider(); } @@ -88,14 +87,8 @@ public String getCollectionName() { @Override public Mono collectionExistsAsync() { return Mono.fromCallable( - () -> { - try { - return queryProvider.collectionExists(this.collectionName); - } catch (SQLException e) { - throw new SKException("Failed to check if collection exists", e); - } - }) - .subscribeOn(Schedulers.boundedElastic()); + () -> queryProvider.collectionExists(this.collectionName)) + .subscribeOn(Schedulers.boundedElastic()); } /** @@ -106,15 +99,10 @@ public Mono collectionExistsAsync() { @Override public Mono createCollectionAsync() { return Mono.fromRunnable( - () -> { - try { - queryProvider.createCollection(this.collectionName, options.getRecordClass(), recordDefinition); - } catch (SQLException e) { - throw new SKException("Failed to create collection", e); - } - }) - .subscribeOn(Schedulers.boundedElastic()) - .then(); + () -> queryProvider.createCollection(this.collectionName, options.getRecordClass(), + recordDefinition)) + .subscribeOn(Schedulers.boundedElastic()) + .then(); } /** @@ -144,11 +132,7 @@ public Mono createCollectionIfNotExistsAsync() { public Mono deleteCollectionAsync() { return Mono.fromRunnable( () -> { - try { - queryProvider.deleteCollection(this.collectionName); - } catch (SQLException e) { - throw new SKException("Failed to delete collection", e); - } + queryProvider.deleteCollection(this.collectionName); }).subscribeOn(Schedulers.boundedElastic()).then(); } @@ -181,24 +165,15 @@ public Mono getAsync(String key, GetRecordOptions options) { public Mono> getBatchAsync(List keys, GetRecordOptions options) { return Mono.fromCallable( () -> { - List records = new ArrayList<>(); - - try { - ResultSet resultSet = queryProvider.getRecords(this.collectionName, keys, recordDefinition, options); - while (resultSet.next()) { - records.add(vectorStoreRecordMapper.mapStorageModeltoRecord(resultSet)); - } - } catch (SQLException e) { - throw new SKException("Failed to get records", e); - } - - return records; + return queryProvider.getRecords(this.collectionName, keys, recordDefinition, + vectorStoreRecordMapper, options); }).subscribeOn(Schedulers.boundedElastic()); } protected String getKeyFromRecord(Record data) { try { - Field keyField = data.getClass().getDeclaredField(recordDefinition.getKeyField().getName()); + Field keyField = data.getClass() + .getDeclaredField(recordDefinition.getKeyField().getName()); keyField.setAccessible(true); return (String) keyField.get(data); } catch (NoSuchFieldException | IllegalAccessException e) { @@ -235,13 +210,8 @@ public Mono upsertAsync(Record data, UpsertRecordOptions options) { public Mono> upsertBatchAsync(List data, UpsertRecordOptions options) { return Mono.fromCallable( () -> { - try { - queryProvider.upsertRecords(this.collectionName, data, recordDefinition, options); - - return data.stream().map(this::getKeyFromRecord).collect(Collectors.toList()); - } catch (SQLException e) { - throw new SKException("Failed to upsert records", e); - } + queryProvider.upsertRecords(this.collectionName, data, recordDefinition, options); + return data.stream().map(this::getKeyFromRecord).collect(Collectors.toList()); }) .subscribeOn(Schedulers.boundedElastic()); } @@ -269,11 +239,7 @@ public Mono deleteAsync(String key, DeleteRecordOptions options) { public Mono deleteBatchAsync(List keys, DeleteRecordOptions options) { return Mono.fromRunnable( () -> { - try { - queryProvider.deleteRecords(this.collectionName, keys, recordDefinition, options); - } catch (SQLException e) { - throw new SKException("Failed to delete records", e); - } + queryProvider.deleteRecords(this.collectionName, keys, recordDefinition, options); }).subscribeOn(Schedulers.boundedElastic()).then(); } @@ -284,20 +250,17 @@ public Mono deleteBatchAsync(List keys, DeleteRecordOptions option */ @Override public Mono prepareAsync() { - return Mono.fromRunnable(() -> { - try { - queryProvider.prepareVectorStore(); - } catch (SQLException e) { - throw new SKException("Failed to prepare vector store record collection", e); - } - }).subscribeOn(Schedulers.boundedElastic()).then(); + return Mono.fromRunnable(queryProvider::prepareVectorStore) + .subscribeOn(Schedulers.boundedElastic()).then(); } - public static class Builder implements SemanticKernelBuilder> { + public static class Builder + implements SemanticKernelBuilder> { private Connection connection; private String collectionName; private JDBCVectorStoreRecordCollectionOptions options; + @SuppressFBWarnings("EI_EXPOSE_REP2") public Builder withConnection(Connection connection) { this.connection = connection; return this; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java index 4d92eb25..8461cd08 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; import java.sql.Connection; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java index 6fe6dbaf..6b860b39 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java @@ -4,20 +4,37 @@ import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; public class JDBCVectorStoreRecordCollectionOptions { + + /** + * The default name for the collections table. + */ + public static final String DEFAULT_COLLECTIONS_TABLE = "SKCollections"; + + /** + * The prefix for collection tables. + */ + public static final String DEFAULT_PREFIX_FOR_COLLECTION_TABLES = "SKCollection_"; + private final Class recordClass; private final JDBCVectorStoreRecordMapper vectorStoreRecordMapper; private final VectorStoreRecordDefinition recordDefinition; private final JDBCVectorStoreQueryProvider queryProvider; + private final String collectionsTableName; + private final String prefixForCollectionTables; - public JDBCVectorStoreRecordCollectionOptions( + private JDBCVectorStoreRecordCollectionOptions( Class recordClass, VectorStoreRecordDefinition recordDefinition, JDBCVectorStoreRecordMapper vectorStoreRecordMapper, - JDBCVectorStoreQueryProvider queryProvider) { + JDBCVectorStoreQueryProvider queryProvider, + String collectionsTableName, + String prefixForCollectionTables) { this.recordClass = recordClass; this.recordDefinition = recordDefinition; this.vectorStoreRecordMapper = vectorStoreRecordMapper; this.queryProvider = queryProvider; + this.collectionsTableName = collectionsTableName; + this.prefixForCollectionTables = prefixForCollectionTables; } /** @@ -53,6 +70,22 @@ public JDBCVectorStoreRecordMapper getVectorStoreRecordMapper() { return vectorStoreRecordMapper; } + /** + * Gets the collections table. + * @return the collections table + */ + public String getCollectionsTableName() { + return collectionsTableName; + } + + /** + * Gets the prefix for collection tables. + * @return the prefix for collection tables + */ + public String getPrefixForCollectionTables() { + return prefixForCollectionTables; + } + /** * Gets the query provider. * @return the query provider @@ -61,11 +94,19 @@ public JDBCVectorStoreQueryProvider getQueryProvider() { return queryProvider; } + public static void validateSQLidentifier(String identifier) { + if (!identifier.matches("[a-zA-Z_][a-zA-Z0-9_]*")) { + throw new IllegalArgumentException("Invalid SQL identifier: " + identifier); + } + } + public static class Builder { private Class recordClass; private VectorStoreRecordDefinition recordDefinition; private JDBCVectorStoreRecordMapper vectorStoreRecordMapper; private JDBCVectorStoreQueryProvider queryProvider; + private String collectionsTableName = DEFAULT_COLLECTIONS_TABLE; + private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; /** * Sets the record class. @@ -108,6 +149,28 @@ public Builder withQueryProvider(JDBCVectorStoreQueryProvider queryProvi return this; } + /** + * Sets the collections table name. + * @param collectionsTableName the collections table name + * @return the builder + */ + public Builder withCollectionsTableName(String collectionsTableName) { + validateSQLidentifier(collectionsTableName); + this.collectionsTableName = collectionsTableName; + return this; + } + + /** + * Sets the prefix for collection tables. + * @param prefixForCollectionTables the prefix for collection tables + * @return the builder + */ + public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { + validateSQLidentifier(prefixForCollectionTables); + this.prefixForCollectionTables = prefixForCollectionTables; + return this; + } + /** * Builds the options. * @return the options @@ -121,8 +184,9 @@ public JDBCVectorStoreRecordCollectionOptions build() { recordClass, recordDefinition, vectorStoreRecordMapper, - queryProvider - ); + queryProvider, + collectionsTableName, + prefixForCollectionTables); } } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/MySQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/MySQLVectorStoreQueryProvider.java index f7211f24..1110cb95 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/MySQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/MySQLVectorStoreQueryProvider.java @@ -17,9 +17,10 @@ import java.util.List; public class MySQLVectorStoreQueryProvider extends - JDBCVectorStoreDefaultQueryProvider implements JDBCVectorStoreQueryProvider { + JDBCVectorStoreDefaultQueryProvider implements JDBCVectorStoreQueryProvider { - public MySQLVectorStoreQueryProvider(Connection connection, String collectionsTable, String prefixForCollectionTables) { + public MySQLVectorStoreQueryProvider(Connection connection, String collectionsTable, + String prefixForCollectionTables) { super(connection, collectionsTable, prefixForCollectionTables); } @@ -31,7 +32,8 @@ public static Builder builder() { return new Builder(); } - private void setStatementValues(PreparedStatement statement, Object record, List fields) { + private void setStatementValues(PreparedStatement statement, Object record, + List fields) { for (int i = 0; i < fields.size(); ++i) { VectorStoreRecordField field = fields.get(i); try { @@ -42,7 +44,8 @@ private void setStatementValues(PreparedStatement statement, Object record, List if (field instanceof VectorStoreRecordKeyField) { statement.setObject(i + 1, (String) value); } else if (field instanceof VectorStoreRecordVectorField) { - Class vectorType = record.getClass().getDeclaredField(field.getName()).getType(); + Class vectorType = record.getClass().getDeclaredField(field.getName()) + .getType(); // If the vector field is other than String, serialize it to JSON if (vectorType.equals(String.class)) { @@ -63,7 +66,10 @@ private void setStatementValues(PreparedStatement statement, Object record, List } @Override - public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) throws SQLException { + public void upsertRecords(String collectionName, List records, + VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { + validateSQLidentifier(getCollectionTableName(collectionName)); + List fields = recordDefinition.getAllFields(); StringBuilder onDuplicateKeyUpdate = new StringBuilder(); @@ -73,22 +79,25 @@ public void upsertRecords(String collectionName, List records, VectorStoreRec onDuplicateKeyUpdate.append(", "); } - onDuplicateKeyUpdate.append(field.getName()).append(" = VALUES(").append(field.getName()).append(")"); + onDuplicateKeyUpdate.append(field.getName()).append(" = VALUES(") + .append(field.getName()).append(")"); } String query = "INSERT INTO " + getCollectionTableName(collectionName) - + " (" + getQueryColumnsFromFields(fields) + ")" - + " VALUES (" + getWildcardString(fields.size()) + ")" - + " ON DUPLICATE KEY UPDATE " + onDuplicateKeyUpdate; - - PreparedStatement statement = connection.prepareStatement(query); + + " (" + getQueryColumnsFromFields(fields) + ")" + + " VALUES (" + getWildcardString(fields.size()) + ")" + + " ON DUPLICATE KEY UPDATE " + onDuplicateKeyUpdate; + + try (PreparedStatement statement = connection.prepareStatement(query)) { + for (Object record : records) { + setStatementValues(statement, record, recordDefinition.getAllFields()); + statement.addBatch(); + } - for (Object record : records) { - setStatementValues(statement, record, recordDefinition.getAllFields()); - statement.addBatch(); + statement.executeBatch(); + } catch (SQLException e) { + throw new SKException("Failed to upsert records", e); } - - statement.executeBatch(); } public static class Builder @@ -98,7 +107,8 @@ public MySQLVectorStoreQueryProvider build() { throw new IllegalArgumentException("connection is required"); } - return new MySQLVectorStoreQueryProvider(connection, collectionsTable, prefixForCollectionTables); + return new MySQLVectorStoreQueryProvider(connection, collectionsTable, + prefixForCollectionTables); } } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java index dd19f787..10e4d2ef 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java @@ -1,10 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; import com.microsoft.semantickernel.data.VectorStore; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import reactor.core.publisher.Mono; -public interface SQLVectorStore> extends VectorStore { +public interface SQLVectorStore> + extends VectorStore { /** * Prepares the vector store. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java index 3583a273..ff12c88b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStoreRecordCollection.java @@ -1,9 +1,11 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import reactor.core.publisher.Mono; -public interface SQLVectorStoreRecordCollection extends VectorStoreRecordCollection { +public interface SQLVectorStoreRecordCollection + extends VectorStoreRecordCollection { /** * Prepares the vector store record collection. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollection.java index e8d9e1db..52d30bf8 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollection.java @@ -82,11 +82,12 @@ public RedisVectorStoreRecordCollection( // Validate supported types VectorStoreRecordDefinition.validateSupportedTypes( - Collections.singletonList(recordDefinition.getKeyDeclaredField(this.options.getRecordClass())), - supportedKeyTypes); + Collections + .singletonList(recordDefinition.getKeyDeclaredField(this.options.getRecordClass())), + supportedKeyTypes); VectorStoreRecordDefinition.validateSupportedTypes( - recordDefinition.getVectorDeclaredFields(this.options.getRecordClass()), - supportedVectorTypes); + recordDefinition.getVectorDeclaredFields(this.options.getRecordClass()), + supportedVectorTypes); // If mapper is not provided, set a default one if (options.getVectorStoreRecordMapper() == null) { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java index 8c1270ba..7a433dbb 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java @@ -44,7 +44,8 @@ public VolatileVectorStoreRecordCollection(String collectionName, // Validate the key type VectorStoreRecordDefinition.validateSupportedTypes( - Collections.singletonList(recordDefinition.getKeyDeclaredField(options.getRecordClass())), + Collections + .singletonList(recordDefinition.getKeyDeclaredField(options.getRecordClass())), supportedKeyTypes); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java index e36a258b..dff27c18 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java @@ -56,7 +56,8 @@ public List getNonVectorFields() { return fields; } - private List getDeclaredFields(Class recordClass, List fields, String fieldType) { + private List getDeclaredFields(Class recordClass, List fields, + String fieldType) { List declaredFields = new ArrayList<>(); for (VectorStoreRecordField field : fields) { try { @@ -64,7 +65,8 @@ private List getDeclaredFields(Class recordClass, List recordClass) { return recordClass.getDeclaredField(keyField.getName()); } catch (NoSuchFieldException e) { throw new IllegalArgumentException( - "Key field not found in record class: " + keyField.getName()); + "Key field not found in record class: " + keyField.getName()); } } public List getDataDeclaredFields(Class recordClass) { return getDeclaredFields( - recordClass, - dataFields.stream().map(f -> (VectorStoreRecordField) f).collect(Collectors.toList()), - "Data"); + recordClass, + dataFields.stream().map(f -> (VectorStoreRecordField) f).collect(Collectors.toList()), + "Data"); } public List getVectorDeclaredFields(Class recordClass) { return getDeclaredFields( - recordClass, - vectorFields.stream().map(f -> (VectorStoreRecordField) f).collect(Collectors.toList()), - "Vector"); + recordClass, + vectorFields.stream().map(f -> (VectorStoreRecordField) f).collect(Collectors.toList()), + "Vector"); } private VectorStoreRecordDefinition( @@ -191,8 +193,8 @@ public static VectorStoreRecordDefinition fromRecordClass(Class recordClass) return checkFields(keyFields, dataFields, vectorFields); } - - public static void validateSupportedTypes(List declaredFields, Set> supportedTypes) { + public static void validateSupportedTypes(List declaredFields, + Set> supportedTypes) { Set> unsupportedTypes = new HashSet<>(); for (Field declaredField : declaredFields) { if (!supportedTypes.contains(declaredField.getType())) { @@ -201,9 +203,10 @@ public static void validateSupportedTypes(List declaredFields, Set Date: Fri, 2 Aug 2024 01:52:17 -0700 Subject: [PATCH 13/28] Add sample and change Connection for DataSource --- .../audio/OpenAiAudioToTextService.java | 3 +- .../audio/OpenAiTextToAudioService.java | 3 +- .../chatcompletion/OpenAIChatCompletion.java | 6 +- .../OpenAITextGenerationService.java | 3 +- api-test/integration-tests/pom.xml | 6 +- .../JDBCVectorStoreRecordCollectionTest.java | 15 +- .../memory/jdbc/JDBCVectorStoreTest.java | 24 ++- .../semantickernel-syntax-examples/pom.xml | 6 + .../memory/JDBC_DataStorage.java | 188 ++++++++++++++++++ .../connectors/data/jdbc/JDBCVectorStore.java | 36 ++-- .../JDBCVectorStoreDefaultQueryProvider.java | 109 +++++----- .../data/jdbc/JDBCVectorStoreOptions.java | 5 + .../jdbc/JDBCVectorStoreQueryProvider.java | 4 +- .../jdbc/JDBCVectorStoreRecordCollection.java | 48 +++-- ...DBCVectorStoreRecordCollectionFactory.java | 3 +- ...DBCVectorStoreRecordCollectionOptions.java | 30 +-- .../jdbc/MySQLVectorStoreQueryProvider.java | 50 ++++- 17 files changed, 399 insertions(+), 140 deletions(-) create mode 100644 samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiAudioToTextService.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiAudioToTextService.java index 7becd67e..631f2cac 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiAudioToTextService.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiAudioToTextService.java @@ -18,7 +18,8 @@ /** * Provides OpenAi implementation of audio to text service. */ -public class OpenAiAudioToTextService extends OpenAiService implements AudioToTextService { +public class OpenAiAudioToTextService extends OpenAiService + implements AudioToTextService { private static final Logger LOGGER = LoggerFactory.getLogger(OpenAiAudioToTextService.class); diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiTextToAudioService.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiTextToAudioService.java index 25071ca9..c698fab3 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiTextToAudioService.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/audio/OpenAiTextToAudioService.java @@ -17,7 +17,8 @@ /** * Provides OpenAi implementation of text to audio service. */ -public class OpenAiTextToAudioService extends OpenAiService implements TextToAudioService { +public class OpenAiTextToAudioService extends OpenAiService + implements TextToAudioService { private static final Logger LOGGER = LoggerFactory.getLogger(OpenAiTextToAudioService.class); diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java index 83b6c9a7..84a8287e 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java @@ -79,7 +79,8 @@ /** * OpenAI chat completion service. */ -public class OpenAIChatCompletion extends OpenAiService implements ChatCompletionService { +public class OpenAIChatCompletion extends OpenAiService + implements ChatCompletionService { private static final Logger LOGGER = LoggerFactory.getLogger(OpenAIChatCompletion.class); @@ -1055,7 +1056,8 @@ static ChatRequestMessage getChatRequestMessage( /** * Builder for creating a new instance of {@link OpenAIChatCompletion}. */ - public static class Builder extends OpenAiServiceBuilder { + public static class Builder + extends OpenAiServiceBuilder { @Override public OpenAIChatCompletion build() { diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAITextGenerationService.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAITextGenerationService.java index 5c418649..13783229 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAITextGenerationService.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/textcompletion/OpenAITextGenerationService.java @@ -30,7 +30,8 @@ /** * An OpenAI implementation of a {@link TextGenerationService}. */ -public class OpenAITextGenerationService extends OpenAiService implements TextGenerationService { +public class OpenAITextGenerationService extends OpenAiService + implements TextGenerationService { private static final Logger LOGGER = LoggerFactory.getLogger(OpenAITextGenerationService.class); diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 126b3741..7e669927 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -68,9 +68,9 @@ 3.44.1.0 - com.mysql - mysql-connector-j - 8.2.0 + mysql + mysql-connector-java + 8.0.33 test diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java index efbe9638..6e80e4ac 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -5,6 +5,7 @@ import com.microsoft.semantickernel.connectors.data.jdbc.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; import com.microsoft.semantickernel.tests.connectors.memory.Hotel; +import com.mysql.cj.jdbc.MysqlDataSource; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.testcontainers.containers.MySQLContainer; @@ -12,6 +13,7 @@ import org.testcontainers.junit.jupiter.Testcontainers; import javax.annotation.Nonnull; +import javax.sql.DataSource; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -29,20 +31,23 @@ public class JDBCVectorStoreRecordCollectionTest { private static final MySQLContainer CONTAINER = new MySQLContainer<>("mysql:5.7.34"); private static final String MYSQL_USER = "test"; private static final String MYSQL_PASSWORD = "test"; - private static Connection connection; + private static MysqlDataSource dataSource; @BeforeAll - static void setup() throws SQLException { - connection = DriverManager.getConnection(CONTAINER.getJdbcUrl(), MYSQL_USER, MYSQL_PASSWORD); + static void setup() { + dataSource = new MysqlDataSource(); + dataSource.setUrl(CONTAINER.getJdbcUrl()); + dataSource.setUser(MYSQL_USER); + dataSource.setPassword(MYSQL_PASSWORD); } private JDBCVectorStoreRecordCollection buildRecordCollection(@Nonnull String collectionName) { JDBCVectorStoreRecordCollection recordCollection = new JDBCVectorStoreRecordCollection<>( - connection, + dataSource, collectionName, JDBCVectorStoreRecordCollectionOptions.builder() .withRecordClass(Hotel.class) .withQueryProvider(MySQLVectorStoreQueryProvider.builder() - .withConnection(connection) + .withDataSource(dataSource) .build()) .build()); diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java index 0ebeed42..eb134dd0 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java @@ -4,6 +4,7 @@ import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions; import com.microsoft.semantickernel.connectors.data.jdbc.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.tests.connectors.memory.Hotel; +import com.mysql.cj.jdbc.MysqlDataSource; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.testcontainers.containers.MySQLContainer; @@ -26,24 +27,29 @@ public class JDBCVectorStoreTest { private static final MySQLContainer CONTAINER = new MySQLContainer<>("mysql:5.7.34"); private static final String MYSQL_USER = "test"; private static final String MYSQL_PASSWORD = "test"; - private static Connection connection; + private static MysqlDataSource dataSource; @BeforeAll - static void setup() throws SQLException { - connection = DriverManager.getConnection(CONTAINER.getJdbcUrl(), MYSQL_USER, MYSQL_PASSWORD); + static void setup() { + dataSource = new MysqlDataSource(); + dataSource.setUrl(CONTAINER.getJdbcUrl()); + dataSource.setUser(MYSQL_USER); + dataSource.setPassword(MYSQL_PASSWORD); } @Test public void getCollectionNamesAsync() { - JDBCVectorStoreOptions options = JDBCVectorStoreOptions.builder() - .withQueryProvider(MySQLVectorStoreQueryProvider.builder() - .withConnection(connection) - .build()) + MySQLVectorStoreQueryProvider queryProvider = MySQLVectorStoreQueryProvider.builder() + .withDataSource(dataSource) .build(); JDBCVectorStore vectorStore = JDBCVectorStore.builder() - .withConnection(connection) - .withOptions(options) + .withDataSource(dataSource) + .withOptions( + JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build() + ) .build(); vectorStore.getCollectionNamesAsync().block(); diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index bea43f73..dcf92999 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -88,6 +88,12 @@ 1.1.0 compile + + + mysql + mysql-connector-java + 8.0.33 + diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java new file mode 100644 index 00000000..2379e572 --- /dev/null +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java @@ -0,0 +1,188 @@ +package com.microsoft.semantickernel.samples.syntaxexamples.memory; + +import com.azure.ai.openai.OpenAIAsyncClient; +import com.azure.ai.openai.OpenAIClientBuilder; +import com.azure.core.credential.AzureKeyCredential; +import com.azure.core.credential.KeyCredential; +import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStore; +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions; +import com.microsoft.semantickernel.connectors.data.jdbc.MySQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.VectorStoreRecordCollection; +import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; +import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; +import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; +import com.mysql.cj.jdbc.MysqlDataSource; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import javax.sql.DataSource; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class JDBC_DataStorage { + + private static final String CLIENT_KEY = System.getenv("CLIENT_KEY"); + private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY"); + + // Only required if AZURE_CLIENT_KEY is set + private static final String CLIENT_ENDPOINT = System.getenv("CLIENT_ENDPOINT"); + private static final String MODEL_ID = System.getenv() + .getOrDefault("EMBEDDING_MODEL_ID", "text-embedding-3-large"); + private static final int EMBEDDING_DIMENSIONS = 1536; + + // Run a MySQL server with: + // docker run -d --name mysql-container -e MYSQL_ROOT_PASSWORD=root -e MYSQL_DATABASE=sk -p 3306:3306 mysql:latest + + static class GitHubFile { + @VectorStoreRecordKeyAttribute() + private final String id; + @VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "embedding") + private final String description; + @VectorStoreRecordDataAttribute + private final String link; + @VectorStoreRecordVectorAttribute(dimensions = EMBEDDING_DIMENSIONS, indexKind = "Hnsw") + private final List embedding; + + public GitHubFile() { + this(null, null, null, Collections.emptyList()); + } + + public GitHubFile( + String id, + String description, + String link, + List embedding) { + this.id = id; + this.description = description; + this.link = link; + this.embedding = embedding; + } + + public String getId() { + return id; + } + + public String getDescription() { + return description; + } + + static String encodeId(String realId) { + byte[] bytes = Base64.getUrlEncoder().encode(realId.getBytes(StandardCharsets.UTF_8)); + return new String(bytes, StandardCharsets.UTF_8); + } + } + + public static void main(String[] args) throws SQLException { + System.out.println("=============================================================="); + System.out.println("========== JDBC Vector Store Example =============="); + System.out.println("=============================================================="); + + OpenAIAsyncClient client; + + if (AZURE_CLIENT_KEY != null) { + client = new OpenAIClientBuilder() + .credential(new AzureKeyCredential(AZURE_CLIENT_KEY)) + .endpoint(CLIENT_ENDPOINT) + .buildAsyncClient(); + + } else { + client = new OpenAIClientBuilder() + .credential(new KeyCredential(CLIENT_KEY)) + .buildAsyncClient(); + } + + var embeddingGeneration = OpenAITextEmbeddingGenerationService.builder() + .withOpenAIAsyncClient(client) + .withModelId(MODEL_ID) + .withDimensions(EMBEDDING_DIMENSIONS) + .build(); + + var dataSource = new MysqlDataSource(); + dataSource.setUrl("jdbc:mysql://localhost:3306/sk"); + dataSource.setPassword("root"); + dataSource.setUser("root"); + + dataStorageWithMySQL(dataSource, embeddingGeneration); + } + + public static void dataStorageWithMySQL( + DataSource dataSource, + OpenAITextEmbeddingGenerationService embeddingGeneration) { + + // Build a query provider + var queryProvider = MySQLVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build(); + + // Create a new vector store + var jdbcVectorStore = JDBCVectorStore.builder() + .withDataSource(dataSource) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); + + String collectionName = "skgithubfiles"; + var collection = jdbcVectorStore.getCollection(collectionName, GitHubFile.class, + null); + + // Create collection if it does not exist and store data + List ids = collection + .createCollectionIfNotExistsAsync() + .then(storeData(collection, embeddingGeneration, sampleData())) + .block(); + + List data = collection.getBatchAsync(ids, null).block(); + + data.forEach(gitHubFile -> System.out.println("Retrieved: " + gitHubFile.getDescription())); + } + + private static Mono> storeData( + VectorStoreRecordCollection recordStore, + OpenAITextEmbeddingGenerationService embeddingGeneration, + Map data) { + + return Flux.fromIterable(data.entrySet()) + .flatMap(entry -> { + System.out.println("Save '" + entry.getKey() + "' to memory."); + + return embeddingGeneration + .generateEmbeddingsAsync(Collections.singletonList(entry.getValue())) + .flatMap(embeddings -> { + GitHubFile gitHubFile = new GitHubFile( + GitHubFile.encodeId(entry.getKey()), + entry.getValue(), + entry.getKey(), + embeddings.get(0).getVector()); + return recordStore.upsertAsync(gitHubFile, null); + }); + }) + .collectList(); + } + + private static Map sampleData() { + return Arrays.stream(new String[][] { + { "https://github.com/microsoft/semantic-kernel/blob/main/README.md", + "README: Installation, getting started with Semantic Kernel, and how to contribute" }, + { "https://github.com/microsoft/semantic-kernel/blob/main/samples/notebooks/dotnet/02-running-prompts-from-file.ipynb", + "Jupyter notebook describing how to pass prompts from a file to a semantic skill or function" }, + { "https://github.com/microsoft/semantic-kernel/tree/main/samples/skills/ChatSkill/ChatGPT", + "Sample demonstrating how to create a chat skill interfacing with ChatGPT" }, + { "https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel/Memory/VolatileMemoryStore.cs", + "C# class that defines a volatile embedding store" }, + { "https://github.com/microsoft/semantic-kernel/blob/main/samples/dotnet/KernelHttpServer/README.md", + "README: How to set up a Semantic Kernel Service API using Azure Function Runtime v4" }, + { "https://github.com/microsoft/semantic-kernel/blob/main/samples/apps/chat-summary-webapp-react/README.md", + "README: README associated with a sample chat summary react-based webapp" }, + }).collect(Collectors.toMap(element -> element[0], element -> element[1])); + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java index 7e02f4a8..5e497176 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java @@ -8,14 +8,14 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.sql.Connection; +import javax.sql.DataSource; import java.util.List; /** * A JDBC vector store. */ public class JDBCVectorStore implements SQLVectorStore> { - private final Connection connection; + private final DataSource dataSource; private final JDBCVectorStoreOptions options; private final JDBCVectorStoreQueryProvider queryProvider; @@ -23,20 +23,20 @@ public class JDBCVectorStore implements SQLVectorStore JDBCVectorStoreRecordCollection getCollection( + public JDBCVectorStoreRecordCollection getCollection( @Nonnull String collectionName, @Nonnull Class recordClass, @Nullable VectorStoreRecordDefinition recordDefinition) { @@ -67,7 +67,7 @@ public JDBCVectorStoreRecordCollection getCollection( if (this.options != null && this.options.getVectorStoreRecordCollectionFactory() != null) { return this.options.getVectorStoreRecordCollectionFactory() .createVectorStoreRecordCollection( - connection, + dataSource, collectionName, JDBCVectorStoreRecordCollectionOptions.builder() .withRecordClass(recordClass) @@ -77,7 +77,7 @@ public JDBCVectorStoreRecordCollection getCollection( } return new JDBCVectorStoreRecordCollection<>( - connection, + dataSource, collectionName, JDBCVectorStoreRecordCollectionOptions.builder() .withRecordClass(recordClass) @@ -110,18 +110,18 @@ public Mono prepareAsync() { * Builder for creating a {@link JDBCVectorStore}. */ public static class Builder { - private Connection connection; + private DataSource dataSource; private JDBCVectorStoreOptions options; /** - * Sets the connection. + * Sets the data source. * - * @param connection the connection + * @param dataSource the data source * @return the builder */ @SuppressFBWarnings("EI_EXPOSE_REP2") - public Builder withConnection(Connection connection) { - this.connection = connection; + public Builder withDataSource(DataSource dataSource) { + this.dataSource = dataSource; return this; } @@ -151,11 +151,11 @@ public JDBCVectorStore build() { * @return the {@link Mono} with the {@link JDBCVectorStore} */ public Mono buildAsync() { - if (connection == null) { - throw new IllegalArgumentException("connection is required"); + if (dataSource == null) { + throw new IllegalArgumentException("dataSource is required"); } - JDBCVectorStore vectorStore = new JDBCVectorStore(connection, options); + JDBCVectorStore vectorStore = new JDBCVectorStore(dataSource, options); return vectorStore.prepareAsync().thenReturn(vectorStore); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java index 9bf75afd..23264bf1 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java @@ -10,6 +10,7 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import javax.annotation.Nonnull; +import javax.sql.DataSource; import java.lang.reflect.Field; import java.sql.Connection; import java.sql.PreparedStatement; @@ -53,16 +54,16 @@ public class JDBCVectorStoreDefaultQueryProvider supportedVectorTypes.put(List.class, "TEXT"); supportedVectorTypes.put(Collection.class, "TEXT"); } - protected final Connection connection; - protected final String collectionsTable; - protected final String prefixForCollectionTables; + private final DataSource dataSource; + private final String collectionsTable; + private final String prefixForCollectionTables; - @SuppressFBWarnings("EI_EXPOSE_REP2") + @SuppressFBWarnings("EI_EXPOSE_REP2") // DataSource is not exposed protected JDBCVectorStoreDefaultQueryProvider( - @Nonnull Connection connection, + @Nonnull DataSource dataSource, @Nonnull String collectionsTable, @Nonnull String prefixForCollectionTables) { - this.connection = connection; + this.dataSource = dataSource; this.collectionsTable = collectionsTable; this.prefixForCollectionTables = prefixForCollectionTables; } @@ -110,15 +111,17 @@ protected String getColumnNamesAndTypes(List fields, Map, String } protected String getCollectionTableName(String collectionName) { - return prefixForCollectionTables + collectionName; + return validateSQLidentifier(prefixForCollectionTables + collectionName); } @Override public void prepareVectorStore() { - String createCollectionsTable = "CREATE TABLE IF NOT EXISTS " + collectionsTable + String createCollectionsTable = "CREATE TABLE IF NOT EXISTS " + + validateSQLidentifier(collectionsTable) + " (collectionId VARCHAR(255) PRIMARY KEY);"; - try (PreparedStatement createTable = connection.prepareStatement(createCollectionsTable)) { + try (Connection connection = dataSource.getConnection(); + PreparedStatement createTable = connection.prepareStatement(createCollectionsTable)) { createTable.execute(); } catch (SQLException e) { throw new SKException("Failed to prepare vector store", e); @@ -139,11 +142,11 @@ public void validateSupportedTypes(Class recordClass, @Override public boolean collectionExists(String collectionName) { - validateSQLidentifier(collectionsTable); - - String query = "SELECT 1 FROM " + collectionsTable + " WHERE collectionId = ?"; + String query = "SELECT 1 FROM " + validateSQLidentifier(collectionsTable) + + " WHERE collectionId = ?"; - try (PreparedStatement statement = connection.prepareStatement(query)) { + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(query)) { statement.setObject(1, collectionName); return statement.executeQuery().next(); @@ -153,10 +156,9 @@ public boolean collectionExists(String collectionName) { } @Override + @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") // SQL query is generated dynamically with valid identifiers public void createCollection(String collectionName, Class recordClass, VectorStoreRecordDefinition recordDefinition) { - validateSQLidentifier(collectionName); - Field keyDeclaredField = recordDefinition.getKeyDeclaredField(recordClass); List dataDeclaredFields = recordDefinition.getDataDeclaredFields(recordClass); List vectorDeclaredFields = recordDefinition.getVectorDeclaredFields(recordClass); @@ -167,16 +169,18 @@ public void createCollection(String collectionName, Class recordClass, + getColumnNamesAndTypes(dataDeclaredFields, supportedDataTypes) + ", " + getColumnNamesAndTypes(vectorDeclaredFields, supportedVectorTypes) + ");"; - String insertCollectionQuery = "INSERT INTO " + collectionsTable + String insertCollectionQuery = "INSERT INTO " + validateSQLidentifier(collectionsTable) + " (collectionId) VALUES (?)"; - try (PreparedStatement createTable = connection.prepareStatement(createStorageTable)) { + try (Connection connection = dataSource.getConnection(); + PreparedStatement createTable = connection.prepareStatement(createStorageTable)) { createTable.execute(); } catch (SQLException e) { throw new SKException("Failed to create collection", e); } - try (PreparedStatement insert = connection.prepareStatement(insertCollectionQuery)) { + try (Connection connection = dataSource.getConnection(); + PreparedStatement insert = connection.prepareStatement(insertCollectionQuery)) { insert.setObject(1, collectionName); insert.execute(); } catch (SQLException e) { @@ -186,22 +190,21 @@ public void createCollection(String collectionName, Class recordClass, @Override public void deleteCollection(String collectionName) { - validateSQLidentifier(collectionsTable); - validateSQLidentifier(getCollectionTableName(collectionName)); - - String deleteCollectionOperation = "DELETE FROM " + collectionsTable + String deleteCollectionOperation = "DELETE FROM " + validateSQLidentifier(collectionsTable) + " WHERE collectionId = ?"; String dropTableOperation = "DROP TABLE " + getCollectionTableName(collectionName); - try (PreparedStatement deleteCollection = connection - .prepareStatement(deleteCollectionOperation)) { + try (Connection connection = dataSource.getConnection(); + PreparedStatement deleteCollection = connection + .prepareStatement(deleteCollectionOperation)) { deleteCollection.setObject(1, collectionName); deleteCollection.execute(); } catch (SQLException e) { throw new SKException("Failed to delete collection", e); } - try (PreparedStatement dropTable = connection.prepareStatement(dropTableOperation)) { + try (Connection connection = dataSource.getConnection(); + PreparedStatement dropTable = connection.prepareStatement(dropTableOperation)) { dropTable.execute(); } catch (SQLException e) { throw new SKException("Failed to drop table", e); @@ -210,11 +213,10 @@ public void deleteCollection(String collectionName) { @Override public List getCollectionNames() { - validateSQLidentifier(collectionsTable); + String query = "SELECT collectionId FROM " + validateSQLidentifier(collectionsTable); - String query = "SELECT collectionId FROM " + collectionsTable; - - try (PreparedStatement statement = connection.prepareStatement(query)) { + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(query)) { List collectionNames = new ArrayList<>(); ResultSet resultSet = statement.executeQuery(); @@ -230,10 +232,8 @@ public List getCollectionNames() { @Override public List getRecords(String collectionName, List keys, - VectorStoreRecordDefinition recordDefinition, JDBCVectorStoreRecordMapper mapper, - GetRecordOptions options) { - validateSQLidentifier(getCollectionTableName(collectionName)); - + VectorStoreRecordDefinition recordDefinition, JDBCVectorStoreRecordMapper mapper, + GetRecordOptions options) { List fields; if (options == null || options.includeVectors()) { fields = recordDefinition.getAllFields(); @@ -246,7 +246,8 @@ public List getRecords(String collectionName, List keys + " WHERE " + recordDefinition.getKeyField().getName() + " IN (" + getWildcardString(keys.size()) + ")"; - try (PreparedStatement statement = connection.prepareStatement(query)) { + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(query)) { for (int i = 0; i < keys.size(); ++i) { statement.setObject(i + 1, keys.get(i)); } @@ -274,13 +275,12 @@ public void upsertRecords(String collectionName, List records, @Override public void deleteRecords(String collectionName, List keys, VectorStoreRecordDefinition recordDefinition, DeleteRecordOptions options) { - validateSQLidentifier(getCollectionTableName(collectionName)); - String query = "DELETE FROM " + getCollectionTableName(collectionName) + " WHERE " + recordDefinition.getKeyField().getName() + " IN (" + getWildcardString(keys.size()) + ")"; - try (PreparedStatement statement = connection.prepareStatement(query)) { + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(query)) { for (int i = 0; i < keys.size(); ++i) { statement.setObject(i + 1, keys.get(i)); } @@ -291,10 +291,11 @@ public void deleteRecords(String collectionName, List keys, } } - public static void validateSQLidentifier(String identifier) { - if (!identifier.matches("[a-zA-Z_][a-zA-Z0-9_]*")) { - throw new IllegalArgumentException("Invalid SQL identifier: " + identifier); + public static String validateSQLidentifier(String identifier) { + if (identifier.matches("[a-zA-Z_][a-zA-Z0-9_]*")) { + return identifier; } + throw new IllegalArgumentException("Invalid SQL identifier: " + identifier); } /** @@ -302,18 +303,18 @@ public static void validateSQLidentifier(String identifier) { */ public static class Builder implements JDBCVectorStoreQueryProvider.Builder { - protected Connection connection; - protected String collectionsTable = DEFAULT_COLLECTIONS_TABLE; - protected String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; + private DataSource dataSource; + private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; + private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; /** - * Sets the connection. - * @param connection the connection + * Sets the data source. + * @param dataSource the data source * @return the builder */ - @SuppressFBWarnings("EI_EXPOSE_REP2") - public Builder withConnection(Connection connection) { - this.connection = connection; + @SuppressFBWarnings("EI_EXPOSE_REP2") // DataSource is not exposed + public Builder withDataSource(DataSource dataSource) { + this.dataSource = dataSource; return this; } @@ -323,8 +324,7 @@ public Builder withConnection(Connection connection) { * @return the builder */ public Builder withCollectionsTable(String collectionsTable) { - validateSQLidentifier(collectionsTable); - this.collectionsTable = collectionsTable; + this.collectionsTable = validateSQLidentifier(collectionsTable); return this; } @@ -334,18 +334,17 @@ public Builder withCollectionsTable(String collectionsTable) { * @return the builder */ public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { - validateSQLidentifier(prefixForCollectionTables); - this.prefixForCollectionTables = prefixForCollectionTables; + this.prefixForCollectionTables = validateSQLidentifier(prefixForCollectionTables); return this; } @Override public JDBCVectorStoreDefaultQueryProvider build() { - if (connection == null) { - throw new IllegalArgumentException("connection is required"); + if (dataSource == null) { + throw new IllegalArgumentException("DataSource is required"); } - return new JDBCVectorStoreDefaultQueryProvider(connection, collectionsTable, + return new JDBCVectorStoreDefaultQueryProvider(dataSource, collectionsTable, prefixForCollectionTables); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java index 580868e9..adb6e13c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreOptions.java @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + import javax.annotation.Nullable; public class JDBCVectorStoreOptions { @@ -14,6 +16,7 @@ public class JDBCVectorStoreOptions { * * @param vectorStoreRecordCollectionFactory The vector store record collection factory. */ + @SuppressFBWarnings("EI_EXPOSE_REP2") // DataSource in queryProvider is not exposed public JDBCVectorStoreOptions( @Nullable JDBCVectorStoreQueryProvider queryProvider, @Nullable JDBCVectorStoreRecordCollectionFactory vectorStoreRecordCollectionFactory) { @@ -34,6 +37,7 @@ public JDBCVectorStoreOptions() { * @return the query provider */ @Nullable + @SuppressFBWarnings("EI_EXPOSE_REP") // DataSource in queryProvider is not exposed public JDBCVectorStoreQueryProvider getQueryProvider() { return queryProvider; } @@ -73,6 +77,7 @@ public static class Builder { * @param queryProvider The query provider. * @return The updated builder instance. */ + @SuppressFBWarnings("EI_EXPOSE_REP2") // DataSource in queryProvider is not exposed public Builder withQueryProvider(JDBCVectorStoreQueryProvider queryProvider) { this.queryProvider = queryProvider; return this; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index fd75dac7..26d976aa 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -81,8 +81,8 @@ void createCollection(String collectionName, Class recordClass, * @return the records */ List getRecords(String collectionName, List keys, - VectorStoreRecordDefinition recordDefinition, JDBCVectorStoreRecordMapper mapper, - GetRecordOptions options); + VectorStoreRecordDefinition recordDefinition, JDBCVectorStoreRecordMapper mapper, + GetRecordOptions options); /** * Upserts records. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java index 0096794c..dac65a4d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -12,8 +12,8 @@ import reactor.core.scheduler.Schedulers; import javax.annotation.Nonnull; +import javax.sql.DataSource; import java.lang.reflect.Field; -import java.sql.Connection; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -27,15 +27,15 @@ public class JDBCVectorStoreRecordCollection private final JDBCVectorStoreQueryProvider queryProvider; /** - * Creates a new instance of the JDBCVectorRecordStore. - * If using this constructor, call {@link #prepareAsync()} before using the record collection. + * Creates a new instance of the {@link JDBCVectorStoreRecordCollection}. * - * @param connection The JDBC connection. - * @param options The options for the store. + * @param dataSource the data source + * @param collectionName the name of the collection + * @param options the options */ - @SuppressFBWarnings("EI_EXPOSE_REP2") + @SuppressFBWarnings("EI_EXPOSE_REP2") // DataSource is not exposed public JDBCVectorStoreRecordCollection( - @Nonnull Connection connection, + @Nonnull DataSource dataSource, @Nonnull String collectionName, @Nonnull JDBCVectorStoreRecordCollectionOptions options) { this.collectionName = collectionName; @@ -59,7 +59,7 @@ public JDBCVectorStoreRecordCollection( // If the query provider is not provided, set a default one if (options.getQueryProvider() == null) { this.queryProvider = JDBCVectorStoreDefaultQueryProvider.builder() - .withConnection(connection) + .withDataSource(dataSource) .build(); } else { this.queryProvider = options.getQueryProvider(); @@ -256,21 +256,39 @@ public Mono prepareAsync() { public static class Builder implements SemanticKernelBuilder> { - private Connection connection; + private DataSource dataSource; private String collectionName; private JDBCVectorStoreRecordCollectionOptions options; - @SuppressFBWarnings("EI_EXPOSE_REP2") - public Builder withConnection(Connection connection) { - this.connection = connection; + /** + * Sets the data source. + * + * @param dataSource the data source + * @return the builder + */ + @SuppressFBWarnings("EI_EXPOSE_REP2") // DataSource is not exposed + public Builder withDataSource(DataSource dataSource) { + this.dataSource = dataSource; return this; } + /** + * Sets the collection name. + * + * @param collectionName the collection name + * @return the builder + */ public Builder withCollectionName(String collectionName) { this.collectionName = collectionName; return this; } + /** + * Sets the options. + * + * @param options the options + * @return the builder + */ public Builder withOptions(JDBCVectorStoreRecordCollectionOptions options) { this.options = options; return this; @@ -278,8 +296,8 @@ public Builder withOptions(JDBCVectorStoreRecordCollectionOptions build() { - if (connection == null) { - throw new IllegalArgumentException("connection is required"); + if (dataSource == null) { + throw new IllegalArgumentException("dataSource is required"); } if (collectionName == null) { throw new IllegalArgumentException("collectionName is required"); @@ -288,7 +306,7 @@ public JDBCVectorStoreRecordCollection build() { throw new IllegalArgumentException("options is required"); } - return new JDBCVectorStoreRecordCollection<>(connection, collectionName, options); + return new JDBCVectorStoreRecordCollection<>(dataSource, collectionName, options); } } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java index 8461cd08..70b62a7e 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; +import javax.sql.DataSource; import java.sql.Connection; /** @@ -14,7 +15,7 @@ public interface JDBCVectorStoreRecordCollectionFactory { * @return The new JDBC vector store record collection. */ JDBCVectorStoreRecordCollection createVectorStoreRecordCollection( - Connection connection, + DataSource dataSource, String collectionName, JDBCVectorStoreRecordCollectionOptions options); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java index 6b860b39..af1ec49e 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java @@ -2,19 +2,13 @@ package com.microsoft.semantickernel.connectors.data.jdbc; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -public class JDBCVectorStoreRecordCollectionOptions { - - /** - * The default name for the collections table. - */ - public static final String DEFAULT_COLLECTIONS_TABLE = "SKCollections"; - - /** - * The prefix for collection tables. - */ - public static final String DEFAULT_PREFIX_FOR_COLLECTION_TABLES = "SKCollection_"; +import static com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreDefaultQueryProvider.validateSQLidentifier; +import static com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider.DEFAULT_COLLECTIONS_TABLE; +import static com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider.DEFAULT_PREFIX_FOR_COLLECTION_TABLES; +public class JDBCVectorStoreRecordCollectionOptions { private final Class recordClass; private final JDBCVectorStoreRecordMapper vectorStoreRecordMapper; private final VectorStoreRecordDefinition recordDefinition; @@ -90,16 +84,11 @@ public String getPrefixForCollectionTables() { * Gets the query provider. * @return the query provider */ + @SuppressFBWarnings("EI_EXPOSE_REP") // DataSource in queryProvider is not exposed public JDBCVectorStoreQueryProvider getQueryProvider() { return queryProvider; } - public static void validateSQLidentifier(String identifier) { - if (!identifier.matches("[a-zA-Z_][a-zA-Z0-9_]*")) { - throw new IllegalArgumentException("Invalid SQL identifier: " + identifier); - } - } - public static class Builder { private Class recordClass; private VectorStoreRecordDefinition recordDefinition; @@ -144,6 +133,7 @@ public Builder withVectorStoreRecordMapper( * @param queryProvider the query provider * @return the builder */ + @SuppressFBWarnings("EI_EXPOSE_REP2") // DataSource in queryProvider is not exposed public Builder withQueryProvider(JDBCVectorStoreQueryProvider queryProvider) { this.queryProvider = queryProvider; return this; @@ -155,8 +145,7 @@ public Builder withQueryProvider(JDBCVectorStoreQueryProvider queryProvi * @return the builder */ public Builder withCollectionsTableName(String collectionsTableName) { - validateSQLidentifier(collectionsTableName); - this.collectionsTableName = collectionsTableName; + this.collectionsTableName = validateSQLidentifier(collectionsTableName); return this; } @@ -166,8 +155,7 @@ public Builder withCollectionsTableName(String collectionsTableName) { * @return the builder */ public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { - validateSQLidentifier(prefixForCollectionTables); - this.prefixForCollectionTables = prefixForCollectionTables; + this.prefixForCollectionTables = validateSQLidentifier(prefixForCollectionTables); return this; } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/MySQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/MySQLVectorStoreQueryProvider.java index 1110cb95..fbf2afce 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/MySQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/MySQLVectorStoreQueryProvider.java @@ -9,7 +9,9 @@ import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import javax.sql.DataSource; import java.lang.reflect.Field; import java.sql.Connection; import java.sql.PreparedStatement; @@ -19,9 +21,13 @@ public class MySQLVectorStoreQueryProvider extends JDBCVectorStoreDefaultQueryProvider implements JDBCVectorStoreQueryProvider { - public MySQLVectorStoreQueryProvider(Connection connection, String collectionsTable, + private final DataSource dataSource; + + @SuppressFBWarnings("EI_EXPOSE_REP2") + private MySQLVectorStoreQueryProvider(DataSource dataSource, String collectionsTable, String prefixForCollectionTables) { - super(connection, collectionsTable, prefixForCollectionTables); + super(dataSource, collectionsTable, prefixForCollectionTables); + this.dataSource = dataSource; } /** @@ -66,6 +72,7 @@ private void setStatementValues(PreparedStatement statement, Object record, } @Override + @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") // SQL query is generated dynamically with valid identifiers public void upsertRecords(String collectionName, List records, VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { validateSQLidentifier(getCollectionTableName(collectionName)); @@ -88,7 +95,8 @@ public void upsertRecords(String collectionName, List records, + " VALUES (" + getWildcardString(fields.size()) + ")" + " ON DUPLICATE KEY UPDATE " + onDuplicateKeyUpdate; - try (PreparedStatement statement = connection.prepareStatement(query)) { + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(query)) { for (Object record : records) { setStatementValues(statement, record, recordDefinition.getAllFields()); statement.addBatch(); @@ -102,12 +110,42 @@ public void upsertRecords(String collectionName, List records, public static class Builder extends JDBCVectorStoreDefaultQueryProvider.Builder { + private DataSource dataSource; + private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; + private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; + + @SuppressFBWarnings("EI_EXPOSE_REP2") + public Builder withDataSource(DataSource dataSource) { + this.dataSource = dataSource; + return this; + } + + /** + * Sets the collections table name. + * @param collectionsTable the collections table name + * @return the builder + */ + public Builder withCollectionsTable(String collectionsTable) { + this.collectionsTable = validateSQLidentifier(collectionsTable); + return this; + } + + /** + * Sets the prefix for collection tables. + * @param prefixForCollectionTables the prefix for collection tables + * @return the builder + */ + public Builder withPrefixForCollectionTables(String prefixForCollectionTables) { + this.prefixForCollectionTables = validateSQLidentifier(prefixForCollectionTables); + return this; + } + public MySQLVectorStoreQueryProvider build() { - if (connection == null) { - throw new IllegalArgumentException("connection is required"); + if (dataSource == null) { + throw new SKException("DataSource is required"); } - return new MySQLVectorStoreQueryProvider(connection, collectionsTable, + return new MySQLVectorStoreQueryProvider(dataSource, collectionsTable, prefixForCollectionTables); } } From 473e9d189f3193c7d7eb3c36dfa920e3624932c2 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Mon, 5 Aug 2024 16:24:52 -0700 Subject: [PATCH 14/28] Add suggestions and update docs --- .../JDBCVectorStoreRecordCollectionTest.java | 2 +- .../JDBCVectorStoreDefaultQueryProvider.java | 68 +++++++++++++++++++ .../jdbc/JDBCVectorStoreRecordCollection.java | 11 +++ .../jdbc/MySQLVectorStoreQueryProvider.java | 8 +++ .../VectorStoreRecordDefinition.java | 10 ++- 5 files changed, 95 insertions(+), 4 deletions(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java index 6e80e4ac..5e676b1c 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -28,7 +28,7 @@ @Testcontainers public class JDBCVectorStoreRecordCollectionTest { @Container - private static final MySQLContainer CONTAINER = new MySQLContainer<>("mysql:5.7.34"); + private static final MySQLContainer CONTAINER = new MySQLContainer<>("mysql:latest"); private static final String MYSQL_USER = "test"; private static final String MYSQL_PASSWORD = "test"; private static MysqlDataSource dataSource; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java index 23264bf1..096f240a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java @@ -114,6 +114,12 @@ protected String getCollectionTableName(String collectionName) { return validateSQLidentifier(prefixForCollectionTables + collectionName); } + /** + * Prepares the vector store. + * Executes any necessary setup steps for the vector store. + * + * @throws SKException if an error occurs while preparing the vector store + */ @Override public void prepareVectorStore() { String createCollectionsTable = "CREATE TABLE IF NOT EXISTS " @@ -128,6 +134,13 @@ public void prepareVectorStore() { } } + /** + * Checks if the types of the record class fields are supported. + * + * @param recordClass the record class + * @param recordDefinition the record definition + * @throws IllegalArgumentException if the types are not supported + */ @Override public void validateSupportedTypes(Class recordClass, VectorStoreRecordDefinition recordDefinition) { @@ -140,6 +153,13 @@ public void validateSupportedTypes(Class recordClass, recordDefinition.getVectorDeclaredFields(recordClass), supportedVectorTypes.keySet()); } + /** + * Checks if a collection exists. + * + * @param collectionName the collection name + * @return true if the collection exists, false otherwise + * @throws SKException if an error occurs while checking if the collection exists + */ @Override public boolean collectionExists(String collectionName) { String query = "SELECT 1 FROM " + validateSQLidentifier(collectionsTable) @@ -155,6 +175,14 @@ public boolean collectionExists(String collectionName) { } } + /** + * Creates a collection. + * + * @param collectionName the collection name + * @param recordClass the record class + * @param recordDefinition the record definition + * @throws SKException if an error occurs while creating the collection + */ @Override @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") // SQL query is generated dynamically with valid identifiers public void createCollection(String collectionName, Class recordClass, @@ -188,6 +216,12 @@ public void createCollection(String collectionName, Class recordClass, } } + /** + * Deletes a collection. + * + * @param collectionName the collection name + * @throws SKException if an error occurs while deleting the collection + */ @Override public void deleteCollection(String collectionName) { String deleteCollectionOperation = "DELETE FROM " + validateSQLidentifier(collectionsTable) @@ -211,6 +245,12 @@ public void deleteCollection(String collectionName) { } } + /** + * Gets the collection names. + * + * @return the collection names + * @throws SKException if an error occurs while getting the collection names + */ @Override public List getCollectionNames() { String query = "SELECT collectionId FROM " + validateSQLidentifier(collectionsTable); @@ -230,6 +270,18 @@ public List getCollectionNames() { } } + /** + * Gets a list of records from the store. + * + * @param collectionName the collection name + * @param keys the keys + * @param recordDefinition the record definition + * @param mapper the mapper + * @param options the options + * @return the records + * @param the record type + * @throws SKException if an error occurs while getting the records + */ @Override public List getRecords(String collectionName, List keys, VectorStoreRecordDefinition recordDefinition, JDBCVectorStoreRecordMapper mapper, @@ -272,6 +324,15 @@ public void upsertRecords(String collectionName, List records, "Upsert is not supported. Try with a specific query provider."); } + /** + * Deletes records. + * + * @param collectionName the collection name + * @param keys the keys + * @param recordDefinition the record definition + * @param options the options + * @throws SKException if an error occurs while deleting the records + */ @Override public void deleteRecords(String collectionName, List keys, VectorStoreRecordDefinition recordDefinition, DeleteRecordOptions options) { @@ -291,6 +352,13 @@ public void deleteRecords(String collectionName, List keys, } } + /** + * Validates an SQL identifier. + * + * @param identifier the identifier + * @return the identifier if it is valid + * @throws IllegalArgumentException if the identifier is invalid + */ public static String validateSQLidentifier(String identifier) { if (identifier.matches("[a-zA-Z_][a-zA-Z0-9_]*")) { return identifier; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java index dac65a4d..b9c0bd3c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -83,6 +83,7 @@ public String getCollectionName() { * Checks if the collection exists in the store. * * @return A Mono emitting a boolean indicating if the collection exists. + * @throws SKException if the operation fails */ @Override public Mono collectionExistsAsync() { @@ -95,6 +96,7 @@ public Mono collectionExistsAsync() { * Creates the collection in the store. * * @return A Mono representing the completion of the creation operation. + * @throws SKException if the operation fails */ @Override public Mono createCollectionAsync() { @@ -109,6 +111,7 @@ public Mono createCollectionAsync() { * Creates the collection in the store if it does not exist. * * @return A Mono representing the completion of the creation operation. + * @throws SKException if the operation fails */ @Override public Mono createCollectionIfNotExistsAsync() { @@ -127,6 +130,7 @@ public Mono createCollectionIfNotExistsAsync() { * Deletes the collection from the store. * * @return A Mono representing the completion of the deletion operation. + * @throws SKException if the operation fails */ @Override public Mono deleteCollectionAsync() { @@ -142,6 +146,7 @@ public Mono deleteCollectionAsync() { * @param key The key of the record to get. * @param options The options for getting the record. * @return A Mono emitting the record. + * @throws SKException if the operation fails */ @Override public Mono getAsync(String key, GetRecordOptions options) { @@ -160,6 +165,7 @@ public Mono getAsync(String key, GetRecordOptions options) { * @param keys The keys of the records to get. * @param options The options for getting the records. * @return A Mono emitting a collection of records. + * @throws SKException if the operation fails */ @Override public Mono> getBatchAsync(List keys, GetRecordOptions options) { @@ -187,6 +193,7 @@ protected String getKeyFromRecord(Record data) { * @param data The record to upsert. * @param options The options for upserting the record. * @return A Mono emitting the key of the upserted record. + * @throws SKException if the operation fails */ @Override public Mono upsertAsync(Record data, UpsertRecordOptions options) { @@ -205,6 +212,7 @@ public Mono upsertAsync(Record data, UpsertRecordOptions options) { * @param data The records to upsert. * @param options The options for upserting the records. * @return A Mono emitting a collection of keys of the upserted records. + * @throws SKException if the operation fails */ @Override public Mono> upsertBatchAsync(List data, UpsertRecordOptions options) { @@ -222,6 +230,7 @@ public Mono> upsertBatchAsync(List data, UpsertRecordOption * @param key The key of the record to delete. * @param options The options for deleting the record. * @return A Mono representing the completion of the deletion operation. + * @throws SKException if the operation fails */ @Override public Mono deleteAsync(String key, DeleteRecordOptions options) { @@ -234,6 +243,7 @@ public Mono deleteAsync(String key, DeleteRecordOptions options) { * @param keys The keys of the records to delete. * @param options The options for deleting the records. * @return A Mono representing the completion of the deletion operation. + * @throws SKException if the operation fails */ @Override public Mono deleteBatchAsync(List keys, DeleteRecordOptions options) { @@ -247,6 +257,7 @@ public Mono deleteBatchAsync(List keys, DeleteRecordOptions option * Prepares the collection for use. * * @return A Mono representing the completion of the preparation operation. + * @throws SKException if the operation fails */ @Override public Mono prepareAsync() { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/MySQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/MySQLVectorStoreQueryProvider.java index fbf2afce..72ecd87e 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/MySQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/MySQLVectorStoreQueryProvider.java @@ -71,6 +71,14 @@ private void setStatementValues(PreparedStatement statement, Object record, } } + /** + * Upserts records into the collection. + * @param collectionName the collection name + * @param records the records to upsert + * @param recordDefinition the record definition + * @param options the upsert options + * @throws SKException if the upsert fails + */ @Override @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") // SQL query is generated dynamically with valid identifiers public void upsertRecords(String collectionName, List records, diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java index dff27c18..39e04a3f 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/recorddefinition/VectorStoreRecordDefinition.java @@ -56,8 +56,12 @@ public List getNonVectorFields() { return fields; } + private enum DeclaredFieldType { + KEY, DATA, VECTOR + } + private List getDeclaredFields(Class recordClass, List fields, - String fieldType) { + DeclaredFieldType fieldType) { List declaredFields = new ArrayList<>(); for (VectorStoreRecordField field : fields) { try { @@ -85,14 +89,14 @@ public List getDataDeclaredFields(Class recordClass) { return getDeclaredFields( recordClass, dataFields.stream().map(f -> (VectorStoreRecordField) f).collect(Collectors.toList()), - "Data"); + DeclaredFieldType.DATA); } public List getVectorDeclaredFields(Class recordClass) { return getDeclaredFields( recordClass, vectorFields.stream().map(f -> (VectorStoreRecordField) f).collect(Collectors.toList()), - "Vector"); + DeclaredFieldType.VECTOR); } private VectorStoreRecordDefinition( From c2ac6757251788628372d92b70a5795ee6f5ea87 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Mon, 5 Aug 2024 16:49:04 -0700 Subject: [PATCH 15/28] Fix --- .../memory/jdbc/JDBCVectorStoreRecordCollectionTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java index 5e676b1c..6e80e4ac 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -28,7 +28,7 @@ @Testcontainers public class JDBCVectorStoreRecordCollectionTest { @Container - private static final MySQLContainer CONTAINER = new MySQLContainer<>("mysql:latest"); + private static final MySQLContainer CONTAINER = new MySQLContainer<>("mysql:5.7.34"); private static final String MYSQL_USER = "test"; private static final String MYSQL_PASSWORD = "test"; private static MysqlDataSource dataSource; From 78911cf268452df1804355bc5138743ac973a13c Mon Sep 17 00:00:00 2001 From: David Grieve Date: Tue, 6 Aug 2024 16:51:53 -0400 Subject: [PATCH 16/28] make ChatHistory thread safe --- .../chatcompletion/OpenAIChatCompletion.java | 27 +++++++++++-------- .../OpenAIChatMessageContent.java | 2 +- .../services/chatcompletion/ChatHistory.java | 16 ++++++----- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java index 6bdb4f1c..f4a57f00 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java @@ -183,7 +183,7 @@ private static class ChatMessages { private final List newMessages; private final List allMessages; - private final List newChatMessageContent; + private final List> newChatMessageContent; public ChatMessages(List allMessages) { this.allMessages = Collections.unmodifiableList(allMessages); @@ -194,7 +194,7 @@ public ChatMessages(List allMessages) { private ChatMessages( List allMessages, List newMessages, - List newChatMessageContent) { + List> newChatMessageContent) { this.allMessages = Collections.unmodifiableList(allMessages); this.newMessages = Collections.unmodifiableList(newMessages); this.newChatMessageContent = Collections.unmodifiableList(newChatMessageContent); @@ -218,8 +218,8 @@ public ChatMessages add(ChatRequestMessage requestMessage) { } @CheckReturnValue - public ChatMessages addChatMessage(List chatMessageContent) { - ArrayList tmpChatMessageContent = new ArrayList<>( + public ChatMessages addChatMessage(List> chatMessageContent) { + ArrayList> tmpChatMessageContent = new ArrayList<>( newChatMessageContent); tmpChatMessageContent.addAll(chatMessageContent); @@ -580,7 +580,7 @@ private OpenAIFunctionToolCall extractOpenAIFunctionToolCall( arguments); } - private Mono> getChatMessageContentsAsync( + private Mono>> getChatMessageContentsAsync( ChatCompletions completions) { FunctionResultMetadata completionMetadata = FunctionResultMetadata.build( completions.getId(), @@ -594,22 +594,27 @@ private Mono> getChatMessageContentsAsync( .filter(Objects::nonNull) .collect(Collectors.toList()); - return Flux.fromIterable(responseMessages) - .flatMap(response -> { + List> chatMessageContent = + responseMessages + .stream() + .map(response -> { try { - return Mono.just(new OpenAIChatMessageContent( + return new OpenAIChatMessageContent<>( AuthorRole.ASSISTANT, response.getContent(), this.getModelId(), null, null, completionMetadata, - formOpenAiToolCalls(response))); + formOpenAiToolCalls(response)); } catch (Exception e) { - return Mono.error(e); + return null; } }) - .collectList(); + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + return Mono.just(chatMessageContent); } private List> toOpenAIChatMessageContent( diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatMessageContent.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatMessageContent.java index f2cbf858..89f45014 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatMessageContent.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatMessageContent.java @@ -36,7 +36,7 @@ public OpenAIChatMessageContent( @Nullable String modelId, @Nullable T innerContent, @Nullable Charset encoding, - @Nullable FunctionResultMetadata metadata, + @Nullable FunctionResultMetadata metadata, @Nullable List toolCall) { super(authorRole, content, modelId, innerContent, encoding, metadata); diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java index d2f391ff..170aac40 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java @@ -5,11 +5,13 @@ import com.microsoft.semantickernel.services.chatcompletion.message.ChatMessageTextContent; import java.nio.charset.Charset; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.Spliterator; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.function.Consumer; import javax.annotation.Nullable; @@ -18,7 +20,7 @@ */ public class ChatHistory implements Iterable> { - private final List> chatMessageContents; + private final Collection> chatMessageContents; /** * The default constructor @@ -33,7 +35,7 @@ public ChatHistory() { * @param instructions The instructions to add to the chat history */ public ChatHistory(@Nullable String instructions) { - this.chatMessageContents = new ArrayList<>(); + this.chatMessageContents = new ConcurrentLinkedQueue<>(); if (instructions != null) { this.chatMessageContents.add( ChatMessageTextContent.systemMessage(instructions)); @@ -45,8 +47,8 @@ public ChatHistory(@Nullable String instructions) { * * @param chatMessageContents The chat message contents to add to the chat history */ - public ChatHistory(List chatMessageContents) { - this.chatMessageContents = new ArrayList(chatMessageContents); + public ChatHistory(List> chatMessageContents) { + this.chatMessageContents = new ConcurrentLinkedQueue<>(chatMessageContents); } /** @@ -55,7 +57,7 @@ public ChatHistory(List chatMessageContents) { * @return List of messages in the chat */ public List> getMessages() { - return Collections.unmodifiableList(chatMessageContents); + return Collections.unmodifiableList(new ArrayList<>(chatMessageContents)); } /** @@ -67,7 +69,7 @@ public Optional> getLastMessage() { if (chatMessageContents.isEmpty()) { return Optional.empty(); } - return Optional.of(chatMessageContents.get(chatMessageContents.size() - 1)); + return Optional.of(((ConcurrentLinkedQueue>)chatMessageContents).peek()); } /** @@ -114,7 +116,7 @@ public Spliterator> spliterator() { * @param metadata The metadata of the message */ public void addMessage(AuthorRole authorRole, String content, Charset encoding, - FunctionResultMetadata metadata) { + FunctionResultMetadata metadata) { chatMessageContents.add( ChatMessageTextContent.builder() .withAuthorRole(authorRole) From 1257fee6250582d36aa48e3d24ac30a6042bf10d Mon Sep 17 00:00:00 2001 From: David Grieve Date: Wed, 7 Aug 2024 11:09:51 -0400 Subject: [PATCH 17/28] Log message if formOpenAiToolCalls throws exception --- .../openai/chatcompletion/OpenAIChatCompletion.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java index 4bd424dd..c02c8ef0 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java @@ -619,7 +619,8 @@ private Mono>> getChatMessageContentsAsync( null, completionMetadata, formOpenAiToolCalls(response)); - } catch (Exception e) { + } catch (SKCheckedException e) { + LOGGER.warn("Failed to form chat message content", e); return null; } }) @@ -936,7 +937,7 @@ private static boolean hasToolCallBeenExecuted(List chatRequ } private static List getChatRequestMessages( - List messages) { + List> messages) { if (messages == null || messages.isEmpty()) { return new ArrayList<>(); } From 8d8d5df3b3de90571defe0ec8661ce3206278a62 Mon Sep 17 00:00:00 2001 From: David Grieve Date: Wed, 7 Aug 2024 12:24:46 -0400 Subject: [PATCH 18/28] Remove Mono from private getChatMessageContentsAsync --- .../chatcompletion/OpenAIChatCompletion.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java index 4bd424dd..ca8eb5b3 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java @@ -357,19 +357,16 @@ private Mono internalChatMessageContentsAsync( // If we don't want to attempt to invoke any functions // Or if we are auto-invoking, but we somehow end up with other than 1 choice even though only 1 was requested if (autoInvokeAttempts == 0 || responseMessages.size() != 1) { - return getChatMessageContentsAsync(completions) - .flatMap(m -> { - return Mono.just(messages.addChatMessage(m)); - }); + List> chatMessageContents = getChatMessageContentsAsync(completions); + return Mono.just(messages.addChatMessage(chatMessageContents)); } // Or if there are no tool calls to be done ChatResponseMessage response = responseMessages.get(0); List toolCalls = response.getToolCalls(); if (toolCalls == null || toolCalls.isEmpty()) { - return getChatMessageContentsAsync(completions) - .flatMap(m -> { - return Mono.just(messages.addChatMessage(m)); - }); + List> chatMessageContents = getChatMessageContentsAsync( + completions); + return Mono.just(messages.addChatMessage(chatMessageContents)); } ChatRequestAssistantMessage requestMessage = new ChatRequestAssistantMessage( @@ -592,7 +589,7 @@ private OpenAIFunctionToolCall extractOpenAIFunctionToolCall( arguments); } - private Mono>> getChatMessageContentsAsync( + private List> getChatMessageContentsAsync( ChatCompletions completions) { FunctionResultMetadata completionMetadata = FunctionResultMetadata.build( completions.getId(), @@ -619,14 +616,15 @@ private Mono>> getChatMessageContentsAsync( null, completionMetadata, formOpenAiToolCalls(response)); - } catch (Exception e) { + } catch (SKCheckedException e) { + LOGGER.warn("Failed to form chat message content", e); return null; } }) .filter(Objects::nonNull) .collect(Collectors.toList()); - return Mono.just(chatMessageContent); + return chatMessageContent; } private List> toOpenAIChatMessageContent( @@ -936,7 +934,7 @@ private static boolean hasToolCallBeenExecuted(List chatRequ } private static List getChatRequestMessages( - List messages) { + List> messages) { if (messages == null || messages.isEmpty()) { return new ArrayList<>(); } From 8b792f5dc02e8fb13f5fdc9e595cc1c2d554b919 Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Thu, 8 Aug 2024 01:16:00 -0700 Subject: [PATCH 19/28] Add PostgreSQL query provider and mapper --- api-test/integration-tests/pom.xml | 30 ++ .../tests/connectors/memory/Hotel.java | 6 +- .../JDBCVectorStoreRecordCollectionTest.java | 193 +++++++---- .../memory/jdbc/JDBCVectorStoreTest.java | 78 +++-- .../memory/JDBC_DataStorage.java | 4 +- semantickernel-experimental/pom.xml | 5 + .../JDBCVectorStoreDefaultQueryProvider.java | 56 ++-- .../jdbc/JDBCVectorStoreQueryProvider.java | 5 +- .../jdbc/JDBCVectorStoreRecordCollection.java | 42 ++- ...DBCVectorStoreRecordCollectionOptions.java | 13 +- .../MySQLVectorStoreQueryProvider.java | 4 +- .../PostgreSQLVectorStoreQueryProvider.java | 302 ++++++++++++++++++ .../PostgreSQLVectorStoreRecordMapper.java | 146 +++++++++ 13 files changed, 752 insertions(+), 132 deletions(-) rename semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/{jdbc => mysql}/MySQLVectorStoreQueryProvider.java (96%) create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java create mode 100644 semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 7e669927..89eb0d14 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -73,6 +73,11 @@ 8.0.33 test + + org.postgresql + postgresql + 42.7.2 + org.testcontainers @@ -139,6 +144,31 @@ + + + io.fabric8 + docker-maven-plugin + 0.45.0 + + + build-image + package + + build + + + + + + + pgvector:latest + + ${project.basedir}/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/jdbc/pgvector/Dockerfile + + + + + diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java index e43842fc..ad10ad64 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/Hotel.java @@ -18,7 +18,7 @@ public class Hotel { @VectorStoreRecordVectorAttribute(dimensions = 3) private final List descriptionEmbedding; @VectorStoreRecordDataAttribute - private final double rating; + private double rating; public Hotel() { this(null, null, 0, null, null, 0.0); @@ -56,4 +56,8 @@ public List getDescriptionEmbedding() { public double getRating() { return rating; } + + public void setRating(double rating) { + this.rating = rating; + } } diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java index 5e676b1c..6466dfe8 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -1,22 +1,24 @@ package com.microsoft.semantickernel.tests.connectors.memory.jdbc; +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; -import com.microsoft.semantickernel.connectors.data.jdbc.MySQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; import com.microsoft.semantickernel.tests.connectors.memory.Hotel; import com.mysql.cj.jdbc.MysqlDataSource; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.postgresql.ds.PGSimpleDataSource; import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; import javax.annotation.Nonnull; import javax.sql.DataSource; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -27,28 +29,55 @@ @Testcontainers public class JDBCVectorStoreRecordCollectionTest { + + @Container + private static final MySQLContainer MYSQL_CONTAINER = new MySQLContainer<>("mysql:5.7.34"); + + private static final DockerImageName PGVECTOR = DockerImageName.parse("pgvector/pgvector:pg16").asCompatibleSubstituteFor("postgres"); @Container - private static final MySQLContainer CONTAINER = new MySQLContainer<>("mysql:latest"); - private static final String MYSQL_USER = "test"; - private static final String MYSQL_PASSWORD = "test"; - private static MysqlDataSource dataSource; - @BeforeAll - static void setup() { - dataSource = new MysqlDataSource(); - dataSource.setUrl(CONTAINER.getJdbcUrl()); - dataSource.setUser(MYSQL_USER); - dataSource.setPassword(MYSQL_PASSWORD); + private static final PostgreSQLContainer POSTGRESQL_CONTAINER = new PostgreSQLContainer<>(PGVECTOR); + + public enum QueryProvider { + MySQL, + PostgreSQL } - private JDBCVectorStoreRecordCollection buildRecordCollection(@Nonnull String collectionName) { + private JDBCVectorStoreRecordCollection buildRecordCollection(QueryProvider provider, @Nonnull String collectionName) { + JDBCVectorStoreQueryProvider queryProvider; + DataSource dataSource; + + switch (provider) { + case MySQL: + MysqlDataSource mysqlDataSource = new MysqlDataSource(); + mysqlDataSource.setUrl(MYSQL_CONTAINER.getJdbcUrl()); + mysqlDataSource.setUser(MYSQL_CONTAINER.getUsername()); + mysqlDataSource.setPassword(MYSQL_CONTAINER.getPassword()); + dataSource = mysqlDataSource; + queryProvider = MySQLVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build(); + break; + case PostgreSQL: + PGSimpleDataSource pgSimpleDataSource = new PGSimpleDataSource(); + pgSimpleDataSource.setUrl(POSTGRESQL_CONTAINER.getJdbcUrl()); + pgSimpleDataSource.setUser(POSTGRESQL_CONTAINER.getUsername()); + pgSimpleDataSource.setPassword(POSTGRESQL_CONTAINER.getPassword()); + dataSource = pgSimpleDataSource; + queryProvider = PostgreSQLVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build(); + break; + default: + throw new IllegalArgumentException("Unknown query provider: " + provider); + } + + JDBCVectorStoreRecordCollection recordCollection = new JDBCVectorStoreRecordCollection<>( dataSource, collectionName, JDBCVectorStoreRecordCollectionOptions.builder() .withRecordClass(Hotel.class) - .withQueryProvider(MySQLVectorStoreQueryProvider.builder() - .withDataSource(dataSource) - .build()) + .withQueryProvider(queryProvider) .build()); recordCollection.prepareAsync().block(); @@ -56,9 +85,10 @@ private JDBCVectorStoreRecordCollection buildRecordCollection(@Nonnull St return recordCollection; } - @Test - public void buildRecordCollection() { - assertNotNull(buildRecordCollection("buildTest")); + @ParameterizedTest + @EnumSource(QueryProvider.class) + public void buildRecordCollection(QueryProvider provider) { + assertNotNull(buildRecordCollection(provider, "buildTest")); } private List getHotels() { @@ -71,31 +101,50 @@ private List getHotels() { ); } - @Test - public void upsertAndGetRecordAsync() { + @ParameterizedTest + @EnumSource(QueryProvider.class) + public void upsertAndGetRecordAsync(QueryProvider provider) { String collectionName = "upsertAndGetRecordAsync"; - JDBCVectorStoreRecordCollection recordStore = buildRecordCollection(collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); List hotels = getHotels(); for (Hotel hotel : hotels) { - recordStore.upsertAsync(hotel, null).block(); + recordCollection.upsertAsync(hotel, null).block(); + } + + // Upsert the first time + for (Hotel hotel : hotels) { + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block(); + assertNotNull(retrievedHotel); + assertEquals(hotel.getId(), retrievedHotel.getId()); + assertEquals(hotel.getRating(), retrievedHotel.getRating()); + + // Update the rating + hotel.setRating(1.0); + } + + // Upsert the second time with updated rating + for (Hotel hotel : hotels) { + recordCollection.upsertAsync(hotel, null).block(); } for (Hotel hotel : hotels) { - Hotel retrievedHotel = recordStore.getAsync(hotel.getId(), null).block(); + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block(); assertNotNull(retrievedHotel); assertEquals(hotel.getId(), retrievedHotel.getId()); + assertEquals(1.0, retrievedHotel.getRating()); } } - @Test - public void getBatchAsync() { + @ParameterizedTest + @EnumSource(QueryProvider.class) + public void getBatchAsync(QueryProvider provider) { String collectionName = "getBatchAsync"; - JDBCVectorStoreRecordCollection recordStore = buildRecordCollection(collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); List hotels = getHotels(); for (Hotel hotel : hotels) { - recordStore.upsertAsync(hotel, null).block(); + recordCollection.upsertAsync(hotel, null).block(); } List keys = new ArrayList<>(); @@ -103,99 +152,104 @@ public void getBatchAsync() { keys.add(hotel.getId()); } - List retrievedHotels = recordStore.getBatchAsync(keys, null).block(); + List retrievedHotels = recordCollection.getBatchAsync(keys, null).block(); assertNotNull(retrievedHotels); assertEquals(hotels.size(), retrievedHotels.size()); } - @Test - public void upsertBatchAndGetBatchAsync() { + @ParameterizedTest + @EnumSource(QueryProvider.class) + public void upsertBatchAndGetBatchAsync(QueryProvider provider) { String collectionName = "upsertBatchAndGetBatchAsync"; - JDBCVectorStoreRecordCollection recordStore = buildRecordCollection(collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); List hotels = getHotels(); - recordStore.upsertBatchAsync(hotels, null).block(); + recordCollection.upsertBatchAsync(hotels, null).block(); List keys = new ArrayList<>(); for (Hotel hotel : hotels) { keys.add(hotel.getId()); } - List retrievedHotels = recordStore.getBatchAsync(keys, null).block(); + List retrievedHotels = recordCollection.getBatchAsync(keys, null).block(); assertNotNull(retrievedHotels); assertEquals(hotels.size(), retrievedHotels.size()); } - @Test - public void insertAndReplaceAsync() { + @ParameterizedTest + @EnumSource(QueryProvider.class) + public void insertAndReplaceAsync(QueryProvider provider) { String collectionName = "insertAndReplaceAsync"; - JDBCVectorStoreRecordCollection recordStore = buildRecordCollection(collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); List hotels = getHotels(); - recordStore.upsertBatchAsync(hotels, null).block(); - recordStore.upsertBatchAsync(hotels, null).block(); - recordStore.upsertBatchAsync(hotels, null).block(); + recordCollection.upsertBatchAsync(hotels, null).block(); + recordCollection.upsertBatchAsync(hotels, null).block(); + recordCollection.upsertBatchAsync(hotels, null).block(); List keys = new ArrayList<>(); for (Hotel hotel : hotels) { keys.add(hotel.getId()); } - List retrievedHotels = recordStore.getBatchAsync(keys, null).block(); + List retrievedHotels = recordCollection.getBatchAsync(keys, null).block(); assertNotNull(retrievedHotels); assertEquals(hotels.size(), retrievedHotels.size()); } - @Test - public void deleteRecordAsync() { + @ParameterizedTest + @EnumSource(QueryProvider.class) + public void deleteRecordAsync(QueryProvider provider) { String collectionName = "deleteRecordAsync"; - JDBCVectorStoreRecordCollection recordStore = buildRecordCollection(collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); List hotels = getHotels(); - recordStore.upsertBatchAsync(hotels, null).block(); + recordCollection.upsertBatchAsync(hotels, null).block(); for (Hotel hotel : hotels) { - recordStore.deleteAsync(hotel.getId(), null).block(); - Hotel retrievedHotel = recordStore.getAsync(hotel.getId(), null).block(); + recordCollection.deleteAsync(hotel.getId(), null).block(); + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), null).block(); assertNull(retrievedHotel); } } - @Test - public void deleteBatchAsync() { + @ParameterizedTest + @EnumSource(QueryProvider.class) + public void deleteBatchAsync(QueryProvider provider) { String collectionName = "deleteBatchAsync"; - JDBCVectorStoreRecordCollection recordStore = buildRecordCollection(collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); List hotels = getHotels(); - recordStore.upsertBatchAsync(hotels, null).block(); + recordCollection.upsertBatchAsync(hotels, null).block(); List keys = new ArrayList<>(); for (Hotel hotel : hotels) { keys.add(hotel.getId()); } - recordStore.deleteBatchAsync(keys, null).block(); + recordCollection.deleteBatchAsync(keys, null).block(); for (String key : keys) { - Hotel retrievedHotel = recordStore.getAsync(key, null).block(); + Hotel retrievedHotel = recordCollection.getAsync(key, null).block(); assertNull(retrievedHotel); } } - @Test - public void getWithNoVectors() { + @ParameterizedTest + @EnumSource(QueryProvider.class) + public void getWithNoVectors(QueryProvider provider) { String collectionName = "getWithNoVectors"; - JDBCVectorStoreRecordCollection recordStore = buildRecordCollection(collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); List hotels = getHotels(); - recordStore.upsertBatchAsync(hotels, null).block(); + recordCollection.upsertBatchAsync(hotels, null).block(); GetRecordOptions options = GetRecordOptions.builder() .includeVectors(false) .build(); for (Hotel hotel : hotels) { - Hotel retrievedHotel = recordStore.getAsync(hotel.getId(), options).block(); + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), options).block(); assertNotNull(retrievedHotel); assertEquals(hotel.getId(), retrievedHotel.getId()); assertNull(retrievedHotel.getDescriptionEmbedding()); @@ -206,20 +260,21 @@ public void getWithNoVectors() { .build(); for (Hotel hotel : hotels) { - Hotel retrievedHotel = recordStore.getAsync(hotel.getId(), options).block(); + Hotel retrievedHotel = recordCollection.getAsync(hotel.getId(), options).block(); assertNotNull(retrievedHotel); assertEquals(hotel.getId(), retrievedHotel.getId()); assertNotNull(retrievedHotel.getDescriptionEmbedding()); } } - @Test - public void getBatchWithNoVectors() { + @ParameterizedTest + @EnumSource(QueryProvider.class) + public void getBatchWithNoVectors(QueryProvider provider) { String collectionName = "getBatchWithNoVectors"; - JDBCVectorStoreRecordCollection recordStore = buildRecordCollection(collectionName); + JDBCVectorStoreRecordCollection recordCollection = buildRecordCollection(provider, collectionName); List hotels = getHotels(); - recordStore.upsertBatchAsync(hotels, null).block(); + recordCollection.upsertBatchAsync(hotels, null).block(); GetRecordOptions options = GetRecordOptions.builder() .includeVectors(false) @@ -230,7 +285,7 @@ public void getBatchWithNoVectors() { keys.add(hotel.getId()); } - List retrievedHotels = recordStore.getBatchAsync(keys, options).block(); + List retrievedHotels = recordCollection.getBatchAsync(keys, options).block(); assertNotNull(retrievedHotels); assertEquals(hotels.size(), retrievedHotels.size()); @@ -242,7 +297,7 @@ public void getBatchWithNoVectors() { .includeVectors(true) .build(); - retrievedHotels = recordStore.getBatchAsync(keys, options).block(); + retrievedHotels = recordCollection.getBatchAsync(keys, options).block(); assertNotNull(retrievedHotels); assertEquals(hotels.size(), retrievedHotels.size()); diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java index eb134dd0..8c2fbfd0 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java @@ -2,21 +2,26 @@ import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStore; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions; -import com.microsoft.semantickernel.connectors.data.jdbc.MySQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; import com.microsoft.semantickernel.tests.connectors.memory.Hotel; import com.mysql.cj.jdbc.MysqlDataSource; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.postgresql.ds.PGSimpleDataSource; import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; +import javax.annotation.Nonnull; +import javax.sql.DataSource; import java.util.Arrays; import java.util.List; +import com.microsoft.semantickernel.tests.connectors.memory.jdbc.JDBCVectorStoreRecordCollectionTest.QueryProvider; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -24,24 +29,41 @@ @Testcontainers public class JDBCVectorStoreTest { @Container - private static final MySQLContainer CONTAINER = new MySQLContainer<>("mysql:5.7.34"); - private static final String MYSQL_USER = "test"; - private static final String MYSQL_PASSWORD = "test"; - private static MysqlDataSource dataSource; - - @BeforeAll - static void setup() { - dataSource = new MysqlDataSource(); - dataSource.setUrl(CONTAINER.getJdbcUrl()); - dataSource.setUser(MYSQL_USER); - dataSource.setPassword(MYSQL_PASSWORD); - } + private static final MySQLContainer MYSQL_CONTAINER = new MySQLContainer<>("mysql:5.7.34"); + + private static final DockerImageName PGVECTOR = DockerImageName.parse("pgvector/pgvector:pg16").asCompatibleSubstituteFor("postgres"); + @Container + private static final PostgreSQLContainer POSTGRESQL_CONTAINER = new PostgreSQLContainer<>(PGVECTOR); + + private JDBCVectorStore buildVectorStore(QueryProvider provider) { + JDBCVectorStoreQueryProvider queryProvider; + DataSource dataSource; + + switch (provider) { + case MySQL: + MysqlDataSource mysqlDataSource = new MysqlDataSource(); + mysqlDataSource.setUrl(MYSQL_CONTAINER.getJdbcUrl()); + mysqlDataSource.setUser(MYSQL_CONTAINER.getUsername()); + mysqlDataSource.setPassword(MYSQL_CONTAINER.getPassword()); + dataSource = mysqlDataSource; + queryProvider = MySQLVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build(); + break; + case PostgreSQL: + PGSimpleDataSource pgSimpleDataSource = new PGSimpleDataSource(); + pgSimpleDataSource.setUrl(POSTGRESQL_CONTAINER.getJdbcUrl()); + pgSimpleDataSource.setUser(POSTGRESQL_CONTAINER.getUsername()); + pgSimpleDataSource.setPassword(POSTGRESQL_CONTAINER.getPassword()); + dataSource = pgSimpleDataSource; + queryProvider = PostgreSQLVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build(); + break; + default: + throw new IllegalArgumentException("Unknown query provider: " + provider); + } - @Test - public void getCollectionNamesAsync() { - MySQLVectorStoreQueryProvider queryProvider = MySQLVectorStoreQueryProvider.builder() - .withDataSource(dataSource) - .build(); JDBCVectorStore vectorStore = JDBCVectorStore.builder() .withDataSource(dataSource) @@ -52,6 +74,16 @@ public void getCollectionNamesAsync() { ) .build(); + vectorStore.prepareAsync().block(); + return vectorStore; + } + + + @ParameterizedTest + @EnumSource(QueryProvider.class) + public void getCollectionNamesAsync(QueryProvider provider) { + JDBCVectorStore vectorStore = buildVectorStore(provider); + vectorStore.getCollectionNamesAsync().block(); List collectionNames = Arrays.asList("collection1", "collection2", "collection3"); diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java index 2379e572..1e74585c 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java @@ -7,7 +7,7 @@ import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStore; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions; -import com.microsoft.semantickernel.connectors.data.jdbc.MySQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; @@ -18,8 +18,6 @@ import javax.sql.DataSource; import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.DriverManager; import java.sql.SQLException; import java.util.Arrays; import java.util.Base64; diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 20fa172a..3efd8ab0 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -109,6 +109,11 @@ + + org.postgresql + postgresql + 42.7.2 + diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java index 096f240a..df9b9adb 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; +import com.microsoft.semantickernel.data.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.exceptions.SKException; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; @@ -27,11 +28,36 @@ public class JDBCVectorStoreDefaultQueryProvider implements JDBCVectorStoreQueryProvider { - private static final Map, String> supportedKeyTypes; - private static final Map, String> supportedDataTypes; - private static final Map, String> supportedVectorTypes; + /** + * Map of supported key types to their SQL types. + * Can be modified by subclasses. + */ + protected Map, String> supportedKeyTypes; + + /** + * Map of supported data types to their SQL types. + * Can be modified by subclasses. + */ + protected Map, String> supportedDataTypes; + + /** + * Map of supported vector types to their SQL types. + * Can be modified by subclasses. + */ + protected Map, String> supportedVectorTypes; + private final DataSource dataSource; + private final String collectionsTable; + private final String prefixForCollectionTables; + + @SuppressFBWarnings("EI_EXPOSE_REP2") // DataSource is not exposed + protected JDBCVectorStoreDefaultQueryProvider( + @Nonnull DataSource dataSource, + @Nonnull String collectionsTable, + @Nonnull String prefixForCollectionTables) { + this.dataSource = dataSource; + this.collectionsTable = collectionsTable; + this.prefixForCollectionTables = prefixForCollectionTables; - static { supportedKeyTypes = new HashMap<>(); supportedKeyTypes.put(String.class, "VARCHAR(255)"); @@ -54,19 +80,6 @@ public class JDBCVectorStoreDefaultQueryProvider supportedVectorTypes.put(List.class, "TEXT"); supportedVectorTypes.put(Collection.class, "TEXT"); } - private final DataSource dataSource; - private final String collectionsTable; - private final String prefixForCollectionTables; - - @SuppressFBWarnings("EI_EXPOSE_REP2") // DataSource is not exposed - protected JDBCVectorStoreDefaultQueryProvider( - @Nonnull DataSource dataSource, - @Nonnull String collectionsTable, - @Nonnull String prefixForCollectionTables) { - this.dataSource = dataSource; - this.collectionsTable = collectionsTable; - this.prefixForCollectionTables = prefixForCollectionTables; - } /** * Creates a new builder. @@ -102,6 +115,12 @@ protected String getQueryColumnsFromFields(List fields) .collect(Collectors.joining(", ")); } + /** + * Formats the column names and types for a table. + * @param fields the fields + * @param types the types + * @return the formatted column names and types + */ protected String getColumnNamesAndTypes(List fields, Map, String> types) { List columns = fields.stream() .map(field -> field.getName() + " " + types.get(field.getType())) @@ -284,7 +303,8 @@ public List getCollectionNames() { */ @Override public List getRecords(String collectionName, List keys, - VectorStoreRecordDefinition recordDefinition, JDBCVectorStoreRecordMapper mapper, + VectorStoreRecordDefinition recordDefinition, + VectorStoreRecordMapper mapper, GetRecordOptions options) { List fields; if (options == null || options.includeVectors()) { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index 26d976aa..386ef5b2 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -2,11 +2,13 @@ package com.microsoft.semantickernel.connectors.data.jdbc; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.data.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import java.sql.ResultSet; import java.util.List; /** @@ -81,7 +83,8 @@ void createCollection(String collectionName, Class recordClass, * @return the records */ List getRecords(String collectionName, List keys, - VectorStoreRecordDefinition recordDefinition, JDBCVectorStoreRecordMapper mapper, + VectorStoreRecordDefinition recordDefinition, + VectorStoreRecordMapper mapper, GetRecordOptions options); /** diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java index b9c0bd3c..0dcb9b8a 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -2,6 +2,10 @@ package com.microsoft.semantickernel.connectors.data.jdbc; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider; +import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreRecordMapper; +import com.microsoft.semantickernel.data.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; @@ -14,6 +18,7 @@ import javax.annotation.Nonnull; import javax.sql.DataSource; import java.lang.reflect.Field; +import java.sql.ResultSet; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -22,8 +27,8 @@ public class JDBCVectorStoreRecordCollection implements SQLVectorStoreRecordCollection { private final String collectionName; private final VectorStoreRecordDefinition recordDefinition; + private final VectorStoreRecordMapper vectorStoreRecordMapper; private final JDBCVectorStoreRecordCollectionOptions options; - private final JDBCVectorStoreRecordMapper vectorStoreRecordMapper; private final JDBCVectorStoreQueryProvider queryProvider; /** @@ -46,16 +51,6 @@ public JDBCVectorStoreRecordCollection( ? VectorStoreRecordDefinition.fromRecordClass(options.getRecordClass()) : options.getRecordDefinition(); - // If mapper is not provided, set a default one - if (options.getVectorStoreRecordMapper() == null) { - vectorStoreRecordMapper = JDBCVectorStoreRecordMapper.builder() - .withRecordClass(options.getRecordClass()) - .withVectorStoreRecordDefinition(recordDefinition) - .build(); - } else { - vectorStoreRecordMapper = options.getVectorStoreRecordMapper(); - } - // If the query provider is not provided, set a default one if (options.getQueryProvider() == null) { this.queryProvider = JDBCVectorStoreDefaultQueryProvider.builder() @@ -65,6 +60,31 @@ public JDBCVectorStoreRecordCollection( this.queryProvider = options.getQueryProvider(); } + // If mapper is not provided, set a default one + if (options.getVectorStoreRecordMapper() == null) { + // Default mapper for PostgreSQL + if (this.queryProvider instanceof PostgreSQLVectorStoreQueryProvider) { + vectorStoreRecordMapper = PostgreSQLVectorStoreRecordMapper.builder() + .withRecordClass(options.getRecordClass()) + .withVectorStoreRecordDefinition(recordDefinition) + .build(); + // Default mapper for MySQL + } else if (this.queryProvider instanceof MySQLVectorStoreQueryProvider) { + vectorStoreRecordMapper = JDBCVectorStoreRecordMapper.builder() + .withRecordClass(options.getRecordClass()) + .withVectorStoreRecordDefinition(recordDefinition) + .build(); + // Default mapper for other databases + } else { + vectorStoreRecordMapper = JDBCVectorStoreRecordMapper.builder() + .withRecordClass(options.getRecordClass()) + .withVectorStoreRecordDefinition(recordDefinition) + .build(); + } + } else { + vectorStoreRecordMapper = options.getVectorStoreRecordMapper(); + } + // Check if the types are supported queryProvider.validateSupportedTypes(options.getRecordClass(), recordDefinition); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java index af1ec49e..f6aa871d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionOptions.java @@ -1,16 +1,19 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; +import com.microsoft.semantickernel.data.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.sql.ResultSet; + import static com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreDefaultQueryProvider.validateSQLidentifier; import static com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider.DEFAULT_COLLECTIONS_TABLE; import static com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider.DEFAULT_PREFIX_FOR_COLLECTION_TABLES; public class JDBCVectorStoreRecordCollectionOptions { private final Class recordClass; - private final JDBCVectorStoreRecordMapper vectorStoreRecordMapper; + private final VectorStoreRecordMapper vectorStoreRecordMapper; private final VectorStoreRecordDefinition recordDefinition; private final JDBCVectorStoreQueryProvider queryProvider; private final String collectionsTableName; @@ -19,7 +22,7 @@ public class JDBCVectorStoreRecordCollectionOptions { private JDBCVectorStoreRecordCollectionOptions( Class recordClass, VectorStoreRecordDefinition recordDefinition, - JDBCVectorStoreRecordMapper vectorStoreRecordMapper, + VectorStoreRecordMapper vectorStoreRecordMapper, JDBCVectorStoreQueryProvider queryProvider, String collectionsTableName, String prefixForCollectionTables) { @@ -60,7 +63,7 @@ public VectorStoreRecordDefinition getRecordDefinition() { * Gets the vector store record mapper. * @return the vector store record mapper */ - public JDBCVectorStoreRecordMapper getVectorStoreRecordMapper() { + public VectorStoreRecordMapper getVectorStoreRecordMapper() { return vectorStoreRecordMapper; } @@ -92,7 +95,7 @@ public JDBCVectorStoreQueryProvider getQueryProvider() { public static class Builder { private Class recordClass; private VectorStoreRecordDefinition recordDefinition; - private JDBCVectorStoreRecordMapper vectorStoreRecordMapper; + private VectorStoreRecordMapper vectorStoreRecordMapper; private JDBCVectorStoreQueryProvider queryProvider; private String collectionsTableName = DEFAULT_COLLECTIONS_TABLE; private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; @@ -123,7 +126,7 @@ public Builder withRecordDefinition(VectorStoreRecordDefinition recordDe * @return the builder */ public Builder withVectorStoreRecordMapper( - JDBCVectorStoreRecordMapper vectorStoreRecordMapper) { + VectorStoreRecordMapper vectorStoreRecordMapper) { this.vectorStoreRecordMapper = vectorStoreRecordMapper; return this; } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/MySQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java similarity index 96% rename from semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/MySQLVectorStoreQueryProvider.java rename to semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java index 72ecd87e..ff19017c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/MySQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/mysql/MySQLVectorStoreQueryProvider.java @@ -1,8 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. -package com.microsoft.semantickernel.connectors.data.jdbc; +package com.microsoft.semantickernel.connectors.data.mysql; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreDefaultQueryProvider; +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java new file mode 100644 index 00000000..81e32c2b --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -0,0 +1,302 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.postgres; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreDefaultQueryProvider; +import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreQueryProvider; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.exceptions.SKException; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +import javax.sql.DataSource; +import java.lang.reflect.Field; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +public class PostgreSQLVectorStoreQueryProvider extends + JDBCVectorStoreDefaultQueryProvider implements JDBCVectorStoreQueryProvider { + private final DataSource dataSource; + private final String collectionsTable; + private final String prefixForCollectionTables; + + @SuppressFBWarnings("EI_EXPOSE_REP2") + private PostgreSQLVectorStoreQueryProvider(DataSource dataSource, String collectionsTable, + String prefixForCollectionTables) { + super(dataSource, collectionsTable, prefixForCollectionTables); + this.dataSource = dataSource; + this.collectionsTable = collectionsTable; + this.prefixForCollectionTables = prefixForCollectionTables; + + supportedKeyTypes = new HashMap<>(); + supportedKeyTypes.put(String.class, "VARCHAR(255)"); + + supportedDataTypes = new HashMap<>(); + supportedDataTypes.put(String.class, "TEXT"); + supportedDataTypes.put(Integer.class, "INTEGER"); + supportedDataTypes.put(int.class, "INTEGER"); + supportedDataTypes.put(Long.class, "BIGINT"); + supportedDataTypes.put(long.class, "BIGINT"); + supportedDataTypes.put(Float.class, "REAL"); + supportedDataTypes.put(float.class, "REAL"); + supportedDataTypes.put(Double.class, "DOUBLE PRECISION"); + supportedDataTypes.put(double.class, "DOUBLE PRECISION"); + supportedDataTypes.put(Boolean.class, "BOOLEAN"); + supportedDataTypes.put(boolean.class, "BOOLEAN"); + supportedDataTypes.put(OffsetDateTime.class, "TIMESTAMPTZ"); + + supportedVectorTypes = new HashMap<>(); + supportedDataTypes.put(String.class, "TEXT"); + supportedVectorTypes.put(List.class, "VECTOR(%d)"); + supportedVectorTypes.put(Collection.class, "VECTOR(%d)"); + } + + /** + * Creates a new builder. + * @return the builder + */ + public static PostgreSQLVectorStoreQueryProvider.Builder builder() { + return new PostgreSQLVectorStoreQueryProvider.Builder(); + } + + /** + * Prepares the vector store. + * Executes any necessary setup steps for the vector store. + * + * @throws SKException if an error occurs while preparing the vector store + */ + @Override + public void prepareVectorStore() { + super.prepareVectorStore(); + + // Create the vector extension + String pgVector = "CREATE EXTENSION IF NOT EXISTS vector"; + + try (Connection connection = dataSource.getConnection(); + PreparedStatement createPgVector = connection.prepareStatement(pgVector)) { + createPgVector.execute(); + } catch (SQLException e) { + throw new SKException("Failed to prepare vector store", e); + } + } + + private String getColumnNamesAndTypesForVectorFields(List fields, + Class recordClass) { + StringBuilder columnNames = new StringBuilder(); + for (VectorStoreRecordVectorField field : fields) { + try { + Field declaredField = recordClass.getDeclaredField(field.getName()); + if (columnNames.length() > 0) { + columnNames.append(", "); + } + + if (declaredField.getType().equals(String.class)) { + columnNames.append(field.getName()) + .append(supportedVectorTypes.get(String.class)); + } else { + // Get the vector type and dimensions + String type = String.format(supportedVectorTypes.get(declaredField.getType()), + field.getDimensions()); + columnNames.append(field.getName()).append(" ").append(type); + } + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } + } + + return columnNames.toString(); + } + + /** + * Creates a collection. + * + * @param collectionName the collection name + * @param recordClass the record class + * @param recordDefinition the record definition + * @throws SKException if an error occurs while creating the collection + */ + @Override + @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") // SQL query is generated dynamically with valid identifiers + public void createCollection(String collectionName, Class recordClass, + VectorStoreRecordDefinition recordDefinition) { + Field keyDeclaredField = recordDefinition.getKeyDeclaredField(recordClass); + List dataDeclaredFields = recordDefinition.getDataDeclaredFields(recordClass); + + String createStorageTable = "CREATE TABLE IF NOT EXISTS " + + getCollectionTableName(collectionName) + + " (" + keyDeclaredField.getName() + " VARCHAR(255) PRIMARY KEY, " + + getColumnNamesAndTypes(dataDeclaredFields, supportedDataTypes) + ", " + + getColumnNamesAndTypesForVectorFields(recordDefinition.getVectorFields(), recordClass) + + ");"; + + String insertCollectionQuery = "INSERT INTO " + validateSQLidentifier(collectionsTable) + + " (collectionId) VALUES (?)"; + + try (Connection connection = dataSource.getConnection(); + PreparedStatement createTable = connection.prepareStatement(createStorageTable)) { + createTable.execute(); + } catch (SQLException e) { + throw new SKException("Failed to create collection", e); + } + + try (Connection connection = dataSource.getConnection(); + PreparedStatement insert = connection.prepareStatement(insertCollectionQuery)) { + insert.setObject(1, collectionName); + insert.execute(); + } catch (SQLException e) { + throw new SKException("Failed to insert collection", e); + } + } + + private void setStatementValues(PreparedStatement statement, Object record, + List fields) { + for (int i = 0; i < fields.size(); ++i) { + VectorStoreRecordField field = fields.get(i); + try { + Field recordField = record.getClass().getDeclaredField(field.getName()); + recordField.setAccessible(true); + Object value = recordField.get(record); + + if (field instanceof VectorStoreRecordKeyField) { + statement.setObject(i + 1, (String) value); + } else if (field instanceof VectorStoreRecordVectorField) { + Class vectorType = record.getClass().getDeclaredField(field.getName()) + .getType(); + + // If the vector field is other than String, serialize it to JSON + if (vectorType.equals(String.class)) { + statement.setObject(i + 1, value); + } else { + // Serialize the vector to JSON + statement.setString(i + 1, new ObjectMapper().writeValueAsString(value)); + } + } else { + statement.setObject(i + 1, value); + } + } catch (NoSuchFieldException | IllegalAccessException | SQLException e) { + throw new SKException("Failed to set statement values", e); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + } + + private String getWildcardStringWithCast(List fields) { + StringBuilder wildcardString = new StringBuilder(); + int wildcards = fields.size(); + for (int i = 0; i < wildcards; ++i) { + if (i > 0) { + wildcardString.append(", "); + } + wildcardString.append("?"); + // Add casting for vector fields + if (fields.get(i) instanceof VectorStoreRecordVectorField) { + wildcardString.append("::vector"); + } + } + return wildcardString.toString(); + } + + /** + * Upserts records into the collection. + * @param collectionName the collection name + * @param records the records to upsert + * @param recordDefinition the record definition + * @param options the upsert options + * @throws SKException if the upsert fails + */ + @Override + @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING") // SQL query is generated dynamically with valid identifiers + public void upsertRecords(String collectionName, List records, + VectorStoreRecordDefinition recordDefinition, UpsertRecordOptions options) { + validateSQLidentifier(getCollectionTableName(collectionName)); + List fields = recordDefinition.getAllFields(); + + StringBuilder onDuplicateKeyUpdate = new StringBuilder(); + for (VectorStoreRecordField field : fields) { + if (field instanceof VectorStoreRecordKeyField) { + continue; + } + if (onDuplicateKeyUpdate.length() > 0) { + onDuplicateKeyUpdate.append(", "); + } + onDuplicateKeyUpdate.append(field.getName()) + .append(" = EXCLUDED.") + .append(field.getName()); + } + + String query = "INSERT INTO " + getCollectionTableName(collectionName) + + " (" + getQueryColumnsFromFields(fields) + ")" + + " VALUES (" + getWildcardStringWithCast(fields) + ")" + + " ON CONFLICT (" + recordDefinition.getKeyField().getName() + ") DO UPDATE SET " + + onDuplicateKeyUpdate; + + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(query)) { + for (Object record : records) { + setStatementValues(statement, record, recordDefinition.getAllFields()); + statement.addBatch(); + } + + statement.executeBatch(); + } catch (SQLException e) { + throw new SKException("Failed to upsert records", e); + } + } + + public static class Builder + extends JDBCVectorStoreDefaultQueryProvider.Builder { + private DataSource dataSource; + private String collectionsTable = DEFAULT_COLLECTIONS_TABLE; + private String prefixForCollectionTables = DEFAULT_PREFIX_FOR_COLLECTION_TABLES; + + @SuppressFBWarnings("EI_EXPOSE_REP2") + public PostgreSQLVectorStoreQueryProvider.Builder withDataSource(DataSource dataSource) { + this.dataSource = dataSource; + return this; + } + + /** + * Sets the collections table name. + * @param collectionsTable the collections table name + * @return the builder + */ + public PostgreSQLVectorStoreQueryProvider.Builder withCollectionsTable( + String collectionsTable) { + this.collectionsTable = validateSQLidentifier(collectionsTable); + return this; + } + + /** + * Sets the prefix for collection tables. + * @param prefixForCollectionTables the prefix for collection tables + * @return the builder + */ + public PostgreSQLVectorStoreQueryProvider.Builder withPrefixForCollectionTables( + String prefixForCollectionTables) { + this.prefixForCollectionTables = validateSQLidentifier(prefixForCollectionTables); + return this; + } + + public PostgreSQLVectorStoreQueryProvider build() { + if (dataSource == null) { + throw new SKException("DataSource is required"); + } + + return new PostgreSQLVectorStoreQueryProvider(dataSource, collectionsTable, + prefixForCollectionTables); + } + } +} diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java new file mode 100644 index 00000000..83b821c3 --- /dev/null +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreRecordMapper.java @@ -0,0 +1,146 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.connectors.data.postgres; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.data.VectorStoreRecordMapper; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; +import com.microsoft.semantickernel.exceptions.SKException; +import org.postgresql.util.PGobject; + +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.List; +import java.util.function.Function; + +public class PostgreSQLVectorStoreRecordMapper + extends VectorStoreRecordMapper { + + /** + * Constructs a new instance of the VectorStoreRecordMapper. + * + * @param storageModelToRecordMapper the function to convert a storage model to a record + */ + protected PostgreSQLVectorStoreRecordMapper( + Function storageModelToRecordMapper) { + super(null, storageModelToRecordMapper); + } + + /** + * Creates a new builder. + * + * @param the record type + * @return the builder + */ + public static Builder builder() { + return new Builder<>(); + } + + public static class Builder + implements SemanticKernelBuilder> { + private Class recordClass; + private VectorStoreRecordDefinition vectorStoreRecordDefinition; + + /** + * Sets the record class. + * + * @param recordClass the record class + * @return the builder + */ + public Builder withRecordClass(Class recordClass) { + this.recordClass = recordClass; + return this; + } + + /** + * Sets the vector store record definition. + * + * @param vectorStoreRecordDefinition the vector store record definition + * @return the builder + */ + public Builder withVectorStoreRecordDefinition( + VectorStoreRecordDefinition vectorStoreRecordDefinition) { + this.vectorStoreRecordDefinition = vectorStoreRecordDefinition; + return this; + } + + /** + * Builds the {@link PostgreSQLVectorStoreRecordMapper}. + * + * @return the {@link PostgreSQLVectorStoreRecordMapper} + */ + public PostgreSQLVectorStoreRecordMapper build() { + if (recordClass == null) { + throw new IllegalArgumentException("recordClass is required"); + } + if (vectorStoreRecordDefinition == null) { + throw new IllegalArgumentException("vectorStoreRecordDefinition is required"); + } + + return new PostgreSQLVectorStoreRecordMapper<>( + resultSet -> { + try { + Constructor constructor = recordClass.getDeclaredConstructor(); + constructor.setAccessible(true); + Record record = (Record) constructor.newInstance(); + + // Select fields from the record definition. + // Check if vector fields are present in the result set. + List fields; + ResultSetMetaData metaData = resultSet.getMetaData(); + if (metaData.getColumnCount() == vectorStoreRecordDefinition.getAllFields() + .size()) { + fields = vectorStoreRecordDefinition.getAllFields(); + } else { + fields = vectorStoreRecordDefinition.getNonVectorFields(); + } + + for (VectorStoreRecordField field : fields) { + Object value = resultSet.getObject(field.getName()); + Field recordField = recordClass.getDeclaredField(field.getName()); + recordField.setAccessible(true); + + // If the field is a vector field, deserialize the JSON string + if (field instanceof VectorStoreRecordVectorField) { + Class vectorType = recordField.getType(); + + // If the vector type is a string, set the value directly + if (vectorType.equals(String.class)) { + recordField.set(record, value); + } else { + // Deserialize the pgvector string to the vector type + PGobject pgObject = (PGobject) value; + recordField.set(record, + new ObjectMapper().readValue(pgObject.getValue(), + vectorType)); + } + } else { + recordField.set(record, value); + } + } + + return record; + } catch (NoSuchMethodException e) { + throw new SKException("Default constructor not found.", e); + } catch (InstantiationException | InvocationTargetException e) { + throw new SKException(String.format( + "SK cannot instantiate %s. A custom mapper is required.", + recordClass.getName()), e); + } catch (JsonProcessingException e) { + throw new SKException(String.format( + "SK cannot deserialize %s. A custom mapper is required.", + recordClass.getName()), e); + } catch (SQLException | NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + }); + } + } +} From 016f08fb268b0cba34fae12865899c4a0001416e Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Thu, 8 Aug 2024 19:52:46 +0100 Subject: [PATCH 20/28] Memory updates for sample app --- .../JDBCVectorStoreRecordCollectionTest.java | 59 +++++----- .../memory/jdbc/JDBCVectorStoreTest.java | 18 ++- .../memory/AzureAISearch_DataStorage.java | 15 ++- .../memory/InMemory_DataStorage.java | 24 ++-- .../memory/JDBC_DataStorage.java | 104 +++++++++--------- .../memory/Redis_DataStorage.java | 4 +- .../AzureAISearchVectorStore.java | 39 +++++-- ...rchVectorStoreCollectionCreateMapping.java | 5 +- .../AzureAISearchVectorStoreOptions.java | 3 +- ...reAISearchVectorStoreRecordCollection.java | 34 +++--- ...rchVectorStoreRecordCollectionFactory.java | 5 +- ...rchVectorStoreRecordCollectionOptions.java | 3 +- .../connectors/data/jdbc/JDBCVectorStore.java | 29 ++++- .../jdbc/JDBCVectorStoreRecordCollection.java | 32 +++--- ...DBCVectorStoreRecordCollectionFactory.java | 1 - .../connectors/data/jdbc/SQLVectorStore.java | 5 +- .../data/redis/RedisVectorStore.java | 23 +++- .../RedisVectorStoreRecordCollection.java | 39 +++---- ...disVectorStoreRecordCollectionFactory.java | 7 +- .../redis/RedisVectorStoreRecordMapper.java | 4 +- .../semantickernel/data/VectorStore.java | 17 ++- .../data/VectorStoreRecordCollection.java | 20 ++-- .../data/VectorStoreRecordMapper.java | 5 +- .../data/VolatileVectorStore.java | 31 ++++-- .../VolatileVectorStoreRecordCollection.java | 25 +++-- ...latileVectorStoreRecordCollectionTest.java | 16 ++- .../data/VolatileVectorStoreTest.java | 12 +- 27 files changed, 319 insertions(+), 260 deletions(-) diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java index 6e80e4ac..c873ea5b 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreRecordCollectionTest.java @@ -1,37 +1,34 @@ package com.microsoft.semantickernel.tests.connectors.memory.jdbc; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollection; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions; import com.microsoft.semantickernel.connectors.data.jdbc.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; import com.microsoft.semantickernel.tests.connectors.memory.Hotel; import com.mysql.cj.jdbc.MysqlDataSource; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import javax.annotation.Nonnull; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -import javax.annotation.Nonnull; -import javax.sql.DataSource; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; - @Testcontainers public class JDBCVectorStoreRecordCollectionTest { + @Container private static final MySQLContainer CONTAINER = new MySQLContainer<>("mysql:5.7.34"); private static final String MYSQL_USER = "test"; private static final String MYSQL_PASSWORD = "test"; private static MysqlDataSource dataSource; + @BeforeAll static void setup() { dataSource = new MysqlDataSource(); @@ -40,16 +37,17 @@ static void setup() { dataSource.setPassword(MYSQL_PASSWORD); } - private JDBCVectorStoreRecordCollection buildRecordCollection(@Nonnull String collectionName) { - JDBCVectorStoreRecordCollection recordCollection = new JDBCVectorStoreRecordCollection<>( - dataSource, - collectionName, - JDBCVectorStoreRecordCollectionOptions.builder() - .withRecordClass(Hotel.class) - .withQueryProvider(MySQLVectorStoreQueryProvider.builder() - .withDataSource(dataSource) - .build()) - .build()); + private JDBCVectorStoreRecordCollection buildRecordCollection( + @Nonnull String collectionName) { + JDBCVectorStoreRecordCollection recordCollection = new JDBCVectorStoreRecordCollection<>( + dataSource, + collectionName, + JDBCVectorStoreRecordCollectionOptions.builder() + .withRecordClass(Hotel.class) + .withQueryProvider(MySQLVectorStoreQueryProvider.builder() + .withDataSource(dataSource) + .build()) + .build()); recordCollection.prepareAsync().block(); recordCollection.createCollectionIfNotExistsAsync().block(); @@ -63,11 +61,16 @@ public void buildRecordCollection() { private List getHotels() { return List.of( - new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(1.0f, 2.0f, 3.0f), 4.0), - new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(1.0f, 2.0f, 3.0f), 3.0), - new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(1.0f, 2.0f, 3.0f), 5.0), - new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(1.0f, 2.0f, 3.0f), 4.0), - new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(1.0f, 2.0f, 3.0f), 5.0) + new Hotel("id_1", "Hotel 1", 1, "Hotel 1 description", Arrays.asList(1.0f, 2.0f, 3.0f), + 4.0), + new Hotel("id_2", "Hotel 2", 2, "Hotel 2 description", Arrays.asList(1.0f, 2.0f, 3.0f), + 3.0), + new Hotel("id_3", "Hotel 3", 3, "Hotel 3 description", Arrays.asList(1.0f, 2.0f, 3.0f), + 5.0), + new Hotel("id_4", "Hotel 4", 4, "Hotel 4 description", Arrays.asList(1.0f, 2.0f, 3.0f), + 4.0), + new Hotel("id_5", "Hotel 5", 5, "Hotel 5 description", Arrays.asList(1.0f, 2.0f, 3.0f), + 5.0) ); } diff --git a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java index eb134dd0..e0906a1c 100644 --- a/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java +++ b/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/connectors/memory/jdbc/JDBCVectorStoreTest.java @@ -1,26 +1,22 @@ package com.microsoft.semantickernel.tests.connectors.memory.jdbc; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStore; import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreOptions; import com.microsoft.semantickernel.connectors.data.jdbc.MySQLVectorStoreQueryProvider; import com.microsoft.semantickernel.tests.connectors.memory.Hotel; import com.mysql.cj.jdbc.MysqlDataSource; +import java.util.Arrays; +import java.util.List; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.Arrays; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - @Testcontainers public class JDBCVectorStoreTest { @Container @@ -43,7 +39,7 @@ public void getCollectionNamesAsync() { .withDataSource(dataSource) .build(); - JDBCVectorStore vectorStore = JDBCVectorStore.builder() + JDBCVectorStore vectorStore = JDBCVectorStore.builder() .withDataSource(dataSource) .withOptions( JDBCVectorStoreOptions.builder() diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java index 92316385..03cc93e3 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/AzureAISearch_DataStorage.java @@ -13,13 +13,10 @@ import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStore; import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreOptions; -import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreRecordCollection; +import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Base64; @@ -27,8 +24,11 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; public class AzureAISearch_DataStorage { + private static final String CLIENT_KEY = System.getenv("CLIENT_KEY"); private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY"); @@ -45,6 +45,7 @@ public class AzureAISearch_DataStorage { private static final int EMBEDDING_DIMENSIONS = 1536; static class GitHubFile { + @VectorStoreRecordKeyAttribute() private final String id; @VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "embedding") @@ -120,7 +121,9 @@ public static void dataStorageWithAzureAISearch( .build(); String collectionName = "skgithubfiles"; - var collection = azureAISearchVectorStore.getCollection(collectionName, GitHubFile.class, + var collection = azureAISearchVectorStore.getCollection( + collectionName, + GitHubFile.class, null); // Create collection if it does not exist and store data @@ -140,7 +143,7 @@ public static void dataStorageWithAzureAISearch( } private static Mono> storeData( - AzureAISearchVectorStoreRecordCollection recordStore, + VectorStoreRecordCollection recordStore, OpenAITextEmbeddingGenerationService embeddingGeneration, Map data) { diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java index d92c1e06..c74d16f8 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/InMemory_DataStorage.java @@ -5,32 +5,22 @@ import com.azure.ai.openai.OpenAIClientBuilder; import com.azure.core.credential.AzureKeyCredential; import com.azure.core.credential.KeyCredential; -import com.azure.core.util.ClientOptions; -import com.azure.core.util.MetricsOptions; -import com.azure.core.util.TracingOptions; -import com.azure.search.documents.indexes.SearchIndexAsyncClient; -import com.azure.search.documents.indexes.SearchIndexClientBuilder; import com.microsoft.semantickernel.aiservices.openai.textembedding.OpenAITextEmbeddingGenerationService; -import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStore; -import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreOptions; -import com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStoreRecordCollection; +import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.VolatileVectorStore; -import com.microsoft.semantickernel.data.VolatileVectorStoreRecordCollection; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordDataAttribute; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - -import java.nio.charset.StandardCharsets; import java.util.Arrays; -import java.util.Base64; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; public class InMemory_DataStorage { + private static final String CLIENT_KEY = System.getenv("CLIENT_KEY"); private static final String AZURE_CLIENT_KEY = System.getenv("AZURE_CLIENT_KEY"); @@ -43,6 +33,7 @@ public class InMemory_DataStorage { private static final int EMBEDDING_DIMENSIONS = 1536; static class GitHubFile { + @VectorStoreRecordKeyAttribute() private final String id; @VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "embedding") @@ -72,8 +63,7 @@ public String getDescription() { } static String encodeId(String realId) { - byte[] bytes = Base64.getUrlEncoder().encode(realId.getBytes(StandardCharsets.UTF_8)); - return new String(bytes, StandardCharsets.UTF_8); + return AzureAISearch_DataStorage.GitHubFile.encodeId(realId); } } @@ -126,7 +116,7 @@ public static void inMemoryDataStorage( } private static Mono> storeData( - VolatileVectorStoreRecordCollection recordCollection, + VectorStoreRecordCollection recordCollection, OpenAITextEmbeddingGenerationService embeddingGeneration, Map data) { diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java index 2379e572..8aa4bddb 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/JDBC_DataStorage.java @@ -1,3 +1,4 @@ +// Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.samples.syntaxexamples.memory; import com.azure.ai.openai.OpenAIAsyncClient; @@ -13,13 +14,7 @@ import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordKeyAttribute; import com.microsoft.semantickernel.data.recordattributes.VectorStoreRecordVectorAttribute; import com.mysql.cj.jdbc.MysqlDataSource; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - -import javax.sql.DataSource; import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.DriverManager; import java.sql.SQLException; import java.util.Arrays; import java.util.Base64; @@ -27,6 +22,9 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import javax.sql.DataSource; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; public class JDBC_DataStorage { @@ -36,13 +34,14 @@ public class JDBC_DataStorage { // Only required if AZURE_CLIENT_KEY is set private static final String CLIENT_ENDPOINT = System.getenv("CLIENT_ENDPOINT"); private static final String MODEL_ID = System.getenv() - .getOrDefault("EMBEDDING_MODEL_ID", "text-embedding-3-large"); + .getOrDefault("EMBEDDING_MODEL_ID", "text-embedding-3-large"); private static final int EMBEDDING_DIMENSIONS = 1536; // Run a MySQL server with: // docker run -d --name mysql-container -e MYSQL_ROOT_PASSWORD=root -e MYSQL_DATABASE=sk -p 3306:3306 mysql:latest static class GitHubFile { + @VectorStoreRecordKeyAttribute() private final String id; @VectorStoreRecordDataAttribute(hasEmbedding = true, embeddingFieldName = "embedding") @@ -57,10 +56,10 @@ public GitHubFile() { } public GitHubFile( - String id, - String description, - String link, - List embedding) { + String id, + String description, + String link, + List embedding) { this.id = id; this.description = description; this.link = link; @@ -90,21 +89,21 @@ public static void main(String[] args) throws SQLException { if (AZURE_CLIENT_KEY != null) { client = new OpenAIClientBuilder() - .credential(new AzureKeyCredential(AZURE_CLIENT_KEY)) - .endpoint(CLIENT_ENDPOINT) - .buildAsyncClient(); + .credential(new AzureKeyCredential(AZURE_CLIENT_KEY)) + .endpoint(CLIENT_ENDPOINT) + .buildAsyncClient(); } else { client = new OpenAIClientBuilder() - .credential(new KeyCredential(CLIENT_KEY)) - .buildAsyncClient(); + .credential(new KeyCredential(CLIENT_KEY)) + .buildAsyncClient(); } var embeddingGeneration = OpenAITextEmbeddingGenerationService.builder() - .withOpenAIAsyncClient(client) - .withModelId(MODEL_ID) - .withDimensions(EMBEDDING_DIMENSIONS) - .build(); + .withOpenAIAsyncClient(client) + .withModelId(MODEL_ID) + .withDimensions(EMBEDDING_DIMENSIONS) + .build(); var dataSource = new MysqlDataSource(); dataSource.setUrl("jdbc:mysql://localhost:3306/sk"); @@ -115,31 +114,32 @@ public static void main(String[] args) throws SQLException { } public static void dataStorageWithMySQL( - DataSource dataSource, - OpenAITextEmbeddingGenerationService embeddingGeneration) { + DataSource dataSource, + OpenAITextEmbeddingGenerationService embeddingGeneration) { // Build a query provider var queryProvider = MySQLVectorStoreQueryProvider.builder() - .withDataSource(dataSource) - .build(); + .withDataSource(dataSource) + .build(); // Create a new vector store var jdbcVectorStore = JDBCVectorStore.builder() - .withDataSource(dataSource) - .withOptions(JDBCVectorStoreOptions.builder() - .withQueryProvider(queryProvider) - .build()) - .build(); + .withDataSource(dataSource) + .withOptions(JDBCVectorStoreOptions.builder() + .withQueryProvider(queryProvider) + .build()) + .build(); String collectionName = "skgithubfiles"; - var collection = jdbcVectorStore.getCollection(collectionName, GitHubFile.class, - null); + var collection = jdbcVectorStore.getCollection(collectionName, + GitHubFile.class, + null); // Create collection if it does not exist and store data List ids = collection - .createCollectionIfNotExistsAsync() - .then(storeData(collection, embeddingGeneration, sampleData())) - .block(); + .createCollectionIfNotExistsAsync() + .then(storeData(collection, embeddingGeneration, sampleData())) + .block(); List data = collection.getBatchAsync(ids, null).block(); @@ -147,26 +147,26 @@ public static void dataStorageWithMySQL( } private static Mono> storeData( - VectorStoreRecordCollection recordStore, - OpenAITextEmbeddingGenerationService embeddingGeneration, - Map data) { + VectorStoreRecordCollection recordStore, + OpenAITextEmbeddingGenerationService embeddingGeneration, + Map data) { return Flux.fromIterable(data.entrySet()) - .flatMap(entry -> { - System.out.println("Save '" + entry.getKey() + "' to memory."); - - return embeddingGeneration - .generateEmbeddingsAsync(Collections.singletonList(entry.getValue())) - .flatMap(embeddings -> { - GitHubFile gitHubFile = new GitHubFile( - GitHubFile.encodeId(entry.getKey()), - entry.getValue(), - entry.getKey(), - embeddings.get(0).getVector()); - return recordStore.upsertAsync(gitHubFile, null); - }); - }) - .collectList(); + .flatMap(entry -> { + System.out.println("Save '" + entry.getKey() + "' to memory."); + + return embeddingGeneration + .generateEmbeddingsAsync(Collections.singletonList(entry.getValue())) + .flatMap(embeddings -> { + GitHubFile gitHubFile = new GitHubFile( + GitHubFile.encodeId(entry.getKey()), + entry.getValue(), + entry.getKey(), + embeddings.get(0).getVector()); + return recordStore.upsertAsync(gitHubFile, null); + }); + }) + .collectList(); } private static Map sampleData() { diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java index cde31a3a..6cd1db3d 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/src/main/java/com/microsoft/semantickernel/samples/syntaxexamples/memory/Redis_DataStorage.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import com.microsoft.semantickernel.samples.syntaxexamples.memory.AzureAISearch_DataStorage.GitHubFile; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import redis.clients.jedis.JedisPooled; @@ -77,8 +78,7 @@ public String getDescription() { } static String encodeId(String realId) { - byte[] bytes = Base64.getUrlEncoder().encode(realId.getBytes(StandardCharsets.UTF_8)); - return new String(bytes, StandardCharsets.UTF_8); + return AzureAISearch_DataStorage.GitHubFile.encodeId(realId); } } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java index 39d13f75..a7b7a7c8 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStore.java @@ -4,16 +4,15 @@ import com.azure.search.documents.indexes.SearchIndexAsyncClient; import com.azure.search.documents.indexes.models.SearchIndex; import com.microsoft.semantickernel.data.VectorStore; +import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import reactor.core.publisher.Mono; - +import java.util.List; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.util.List; +import reactor.core.publisher.Mono; -public class AzureAISearchVectorStore - implements VectorStore> { +public class AzureAISearchVectorStore implements VectorStore { private final SearchIndexAsyncClient client; private final AzureAISearchVectorStoreOptions options; @@ -21,7 +20,7 @@ public class AzureAISearchVectorStore /** * Creates a new instance of {@link AzureAISearchVectorStore}. * - * @param client The Azure AI Search client. + * @param client The Azure AI Search client. * @param options The options for the vector store. */ @SuppressFBWarnings("EI_EXPOSE_REP2") @@ -34,17 +33,29 @@ public AzureAISearchVectorStore(@Nonnull SearchIndexAsyncClient client, /** * Gets a new instance of {@link AzureAISearchVectorStoreRecordCollection} * - * @param collectionName The name of the collection. - * @param recordClass The class type of the record. + * @param collectionName The name of the collection. + * @param recordClass The class type of the record. * @param recordDefinition The record definition. * @return The collection. */ @Override - public AzureAISearchVectorStoreRecordCollection getCollection( + public final VectorStoreRecordCollection getCollection( @Nonnull String collectionName, + @Nonnull Class keyClass, @Nonnull Class recordClass, - VectorStoreRecordDefinition recordDefinition) { + @Nullable VectorStoreRecordDefinition recordDefinition) { + if (!keyClass.equals(String.class)) { + throw new IllegalArgumentException("Azure AI Search only supports string keys"); + } + return (VectorStoreRecordCollection) getCollection( + collectionName, recordClass, recordDefinition); + } + + public AzureAISearchVectorStoreRecordCollection getCollection( + @Nonnull String collectionName, + @Nonnull Class recordClass, + @Nullable VectorStoreRecordDefinition recordDefinition) { if (options.getVectorStoreRecordCollectionFactory() != null) { return options.getVectorStoreRecordCollectionFactory() .createVectorStoreRecordCollection( @@ -56,7 +67,9 @@ public AzureAISearchVectorStoreRecordCollection getCollect .build()); } - return new AzureAISearchVectorStoreRecordCollection<>(client, collectionName, + return new AzureAISearchVectorStoreRecordCollection<>( + client, + collectionName, AzureAISearchVectorStoreRecordCollectionOptions.builder() .withRecordClass(recordClass) .withRecordDefinition(recordDefinition) @@ -86,6 +99,7 @@ public static Builder builder() { * Builder for {@link AzureAISearchVectorStore}. */ public static class Builder { + @Nullable private SearchIndexAsyncClient client; @Nullable @@ -109,7 +123,8 @@ public Builder withClient(@Nonnull SearchIndexAsyncClient client) { * @param options The options for the Azure AI Search vector store. * @return The updated builder instance. */ - public Builder withOptions(@Nonnull AzureAISearchVectorStoreOptions options) { + public Builder withOptions( + @Nonnull AzureAISearchVectorStoreOptions options) { this.options = options; return this; } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java index e077a510..c57e6f9d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreCollectionCreateMapping.java @@ -13,12 +13,9 @@ import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.time.OffsetDateTime; import java.util.List; -import java.util.Objects; +import javax.annotation.Nonnull; public class AzureAISearchVectorStoreCollectionCreateMapping { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreOptions.java index d7bb0314..db3b7ab6 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreOptions.java @@ -1,10 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.azureaisearch; -import javax.annotation.Nonnull; import javax.annotation.Nullable; public class AzureAISearchVectorStoreOptions { + @Nullable private final AzureAISearchVectorStoreRecordCollectionFactory vectorStoreRecordCollectionFactory; @@ -49,6 +49,7 @@ public AzureAISearchVectorStoreRecordCollectionFactory getVectorStoreRecordColle * */ public static class Builder { + @Nullable private AzureAISearchVectorStoreRecordCollectionFactory vectorStoreRecordCollectionFactory; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java index 5155299d..5937d928 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollection.java @@ -11,22 +11,18 @@ import com.azure.search.documents.indexes.models.VectorSearchProfile; import com.azure.search.documents.models.IndexDocumentsResult; import com.azure.search.documents.models.IndexingResult; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; -import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; -import com.microsoft.semantickernel.exceptions.SKException; import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDataField; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordField; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordKeyField; +import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordVectorField; import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; +import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - -import javax.annotation.Nonnull; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Arrays; @@ -38,9 +34,12 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; -public class AzureAISearchVectorStoreRecordCollection - implements VectorStoreRecordCollection { +public class AzureAISearchVectorStoreRecordCollection implements + VectorStoreRecordCollection { private static final HashSet> supportedKeyTypes = new HashSet<>( Collections.singletonList( @@ -124,7 +123,7 @@ public Mono collectionExistsAsync() { } @Override - public Mono createCollectionAsync() { + public Mono> createCollectionAsync() { List searchFields = new ArrayList<>(); List algorithms = new ArrayList<>(); List profiles = new ArrayList<>(); @@ -151,18 +150,19 @@ public Mono createCollectionAsync() { .setAlgorithms(algorithms) .setProfiles(profiles)); - return client.createIndex(newIndex).then(); + return client.createIndex(newIndex).then(Mono.just(this)); } @Override - public Mono createCollectionIfNotExistsAsync() { + public Mono> createCollectionIfNotExistsAsync() { return collectionExistsAsync().flatMap( exists -> { if (!exists) { return createCollectionAsync(); } return Mono.empty(); - }); + }) + .then(Mono.just(this)); } @Override @@ -191,11 +191,11 @@ public Mono getAsync( } return client.getDocumentWithResponse(key, this.options.getRecordClass(), selectedFields) - .map(response -> { + .flatMap(response -> { if (response.getStatusCode() == 404) { - throw new SKException("Record not found: " + key); + return Mono.error(new SKException("Record not found: " + key)); } - return response.getValue(); + return Mono.just(response.getValue()); }); } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java index c5041284..ec08ba03 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionFactory.java @@ -5,16 +5,15 @@ /** * Factory for creating Azure AI Search vector store record collections. - * */ public interface AzureAISearchVectorStoreRecordCollectionFactory { /** * Creates a new Azure AI Search vector store record collection. * - * @param client The Azure AI Search client. + * @param client The Azure AI Search client. * @param collectionName The name of the collection. - * @param options The options for the collection. + * @param options The options for the collection. * @return The new Azure AI Search vector store record collection. */ AzureAISearchVectorStoreRecordCollection createVectorStoreRecordCollection( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java index 45fb410c..7275dd43 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/azureaisearch/AzureAISearchVectorStoreRecordCollectionOptions.java @@ -4,7 +4,6 @@ import com.azure.search.documents.SearchDocument; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; - import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -14,6 +13,7 @@ * @param the record type */ public class AzureAISearchVectorStoreRecordCollectionOptions { + private final Class recordClass; @Nullable private final VectorStoreRecordMapper vectorStoreRecordMapper; @@ -75,6 +75,7 @@ private AzureAISearchVectorStoreRecordCollectionOptions( * @param the record type */ public static class Builder { + @Nullable private VectorStoreRecordMapper vectorStoreRecordMapper; @Nullable diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java index 5e497176..66ad995b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStore.java @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.jdbc; +import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreRecordCollection; +import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import reactor.core.publisher.Mono; @@ -14,7 +16,7 @@ /** * A JDBC vector store. */ -public class JDBCVectorStore implements SQLVectorStore> { +public class JDBCVectorStore implements SQLVectorStore { private final DataSource dataSource; private final JDBCVectorStoreOptions options; private final JDBCVectorStoreQueryProvider queryProvider; @@ -59,11 +61,32 @@ public static Builder builder() { * @return The collection. */ @Override - public JDBCVectorStoreRecordCollection getCollection( - @Nonnull String collectionName, + public VectorStoreRecordCollection getCollection( + @Nonnull String collectionName, @Nonnull Class keyClass, @Nonnull Class recordClass, @Nullable VectorStoreRecordDefinition recordDefinition) { + if (keyClass != String.class) { + throw new IllegalArgumentException("Redis only supports string keys"); + } + + return (VectorStoreRecordCollection) getCollection( + collectionName, + recordClass, + recordDefinition); + } + /** + * Gets a collection from the vector store. + * + * @param collectionName The name of the collection. + * @param recordClass The class type of the record. + * @param recordDefinition The record definition. + * @return The collection. + */ + public JDBCVectorStoreRecordCollection getCollection( + @Nonnull String collectionName, + @Nonnull Class recordClass, + @Nullable VectorStoreRecordDefinition recordDefinition) { if (this.options != null && this.options.getVectorStoreRecordCollectionFactory() != null) { return this.options.getVectorStoreRecordCollectionFactory() .createVectorStoreRecordCollection( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java index b9c0bd3c..6135b284 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollection.java @@ -2,24 +2,25 @@ package com.microsoft.semantickernel.connectors.data.jdbc; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import reactor.core.publisher.Mono; -import reactor.core.scheduler.Schedulers; - -import javax.annotation.Nonnull; -import javax.sql.DataSource; import java.lang.reflect.Field; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.sql.DataSource; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; public class JDBCVectorStoreRecordCollection implements SQLVectorStoreRecordCollection { + private final String collectionName; private final VectorStoreRecordDefinition recordDefinition; private final JDBCVectorStoreRecordCollectionOptions options; @@ -29,9 +30,9 @@ public class JDBCVectorStoreRecordCollection /** * Creates a new instance of the {@link JDBCVectorStoreRecordCollection}. * - * @param dataSource the data source + * @param dataSource the data source * @param collectionName the name of the collection - * @param options the options + * @param options the options */ @SuppressFBWarnings("EI_EXPOSE_REP2") // DataSource is not exposed public JDBCVectorStoreRecordCollection( @@ -99,12 +100,12 @@ public Mono collectionExistsAsync() { * @throws SKException if the operation fails */ @Override - public Mono createCollectionAsync() { + public Mono> createCollectionAsync() { return Mono.fromRunnable( () -> queryProvider.createCollection(this.collectionName, options.getRecordClass(), recordDefinition)) .subscribeOn(Schedulers.boundedElastic()) - .then(); + .then(Mono.just(this)); } /** @@ -114,7 +115,7 @@ public Mono createCollectionAsync() { * @throws SKException if the operation fails */ @Override - public Mono createCollectionIfNotExistsAsync() { + public Mono> createCollectionIfNotExistsAsync() { return collectionExistsAsync().map( exists -> { if (!exists) { @@ -123,7 +124,7 @@ public Mono createCollectionIfNotExistsAsync() { return Mono.empty(); }) .flatMap(mono -> mono) - .then(); + .then(Mono.just(this)); } /** @@ -143,7 +144,7 @@ public Mono deleteCollectionAsync() { /** * Gets a record from the store. * - * @param key The key of the record to get. + * @param key The key of the record to get. * @param options The options for getting the record. * @return A Mono emitting the record. * @throws SKException if the operation fails @@ -162,7 +163,7 @@ public Mono getAsync(String key, GetRecordOptions options) { /** * Gets a batch of records from the store. * - * @param keys The keys of the records to get. + * @param keys The keys of the records to get. * @param options The options for getting the records. * @return A Mono emitting a collection of records. * @throws SKException if the operation fails @@ -227,7 +228,7 @@ public Mono> upsertBatchAsync(List data, UpsertRecordOption /** * Deletes a record from the store. * - * @param key The key of the record to delete. + * @param key The key of the record to delete. * @param options The options for deleting the record. * @return A Mono representing the completion of the deletion operation. * @throws SKException if the operation fails @@ -240,7 +241,7 @@ public Mono deleteAsync(String key, DeleteRecordOptions options) { /** * Deletes a batch of records from the store. * - * @param keys The keys of the records to delete. + * @param keys The keys of the records to delete. * @param options The options for deleting the records. * @return A Mono representing the completion of the deletion operation. * @throws SKException if the operation fails @@ -267,6 +268,7 @@ public Mono prepareAsync() { public static class Builder implements SemanticKernelBuilder> { + private DataSource dataSource; private String collectionName; private JDBCVectorStoreRecordCollectionOptions options; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java index 70b62a7e..6cfcdcad 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreRecordCollectionFactory.java @@ -2,7 +2,6 @@ package com.microsoft.semantickernel.connectors.data.jdbc; import javax.sql.DataSource; -import java.sql.Connection; /** * Factory for creating JDBC vector store record collections. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java index 10e4d2ef..046f9941 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/SQLVectorStore.java @@ -2,11 +2,10 @@ package com.microsoft.semantickernel.connectors.data.jdbc; import com.microsoft.semantickernel.data.VectorStore; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import reactor.core.publisher.Mono; -public interface SQLVectorStore> - extends VectorStore { +public interface SQLVectorStore + extends VectorStore { /** * Prepares the vector store. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java index 7e561e43..02f728f0 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStore.java @@ -3,6 +3,7 @@ import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.VectorStore; +import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.util.ArrayList; @@ -12,7 +13,7 @@ import reactor.core.publisher.Mono; import redis.clients.jedis.JedisPooled; -public class RedisVectorStore implements VectorStore> { +public class RedisVectorStore implements VectorStore { private final JedisPooled client; private final RedisVectorStoreOptions options; @@ -30,6 +31,22 @@ public RedisVectorStore(@Nonnull JedisPooled client, this.options = options; } + @Override + public VectorStoreRecordCollection getCollection( + @Nonnull String collectionName, + @Nonnull Class keyClass, + @Nonnull Class recordClass, + @Nullable VectorStoreRecordDefinition recordDefinition) { + if (keyClass != String.class) { + throw new IllegalArgumentException("Redis only supports string keys"); + } + + return (VectorStoreRecordCollection) getCollection( + collectionName, + recordClass, + recordDefinition); + } + /** * Gets a collection from the vector store. * @@ -38,8 +55,7 @@ public RedisVectorStore(@Nonnull JedisPooled client, * @param recordDefinition The record definition. * @return The collection. */ - @Override - public RedisVectorStoreRecordCollection getCollection( + public RedisVectorStoreRecordCollection getCollection( @Nonnull String collectionName, @Nonnull Class recordClass, @Nullable VectorStoreRecordDefinition recordDefinition) { @@ -74,7 +90,6 @@ public Mono> getCollectionNamesAsync() { /** * Builder for the Redis vector store. - * */ public static Builder builder() { return new Builder(); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollection.java index 52d30bf8..b0f8858b 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollection.java @@ -12,6 +12,17 @@ import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import java.util.AbstractMap.SimpleEntry; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; import org.json.JSONArray; import org.json.JSONObject; import reactor.core.publisher.Mono; @@ -25,18 +36,6 @@ import redis.clients.jedis.search.IndexOptions; import redis.clients.jedis.search.Schema; -import javax.annotation.Nonnull; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.AbstractMap.SimpleEntry; -import java.util.stream.Collectors; - public class RedisVectorStoreRecordCollection implements VectorStoreRecordCollection { @@ -143,7 +142,7 @@ public Mono collectionExistsAsync() { * @return A Mono representing the completion of the creation operation. */ @Override - public Mono createCollectionAsync() { + public Mono> createCollectionAsync() { return Mono.fromRunnable(() -> { Schema schema = RedisVectorStoreCollectionCreateMapping .mapToSchema(recordDefinition.getAllFields()); @@ -155,17 +154,19 @@ public Mono createCollectionAsync() { collectionName, IndexOptions.defaultOptions().setDefinition(indexDefinition), schema); - }).subscribeOn(Schedulers.boundedElastic()).then(); + }) + .subscribeOn(Schedulers.boundedElastic()) + .then(Mono.just(this)); } @Override - public Mono createCollectionIfNotExistsAsync() { + public Mono> createCollectionIfNotExistsAsync() { return collectionExistsAsync().flatMap(exists -> { if (!exists) { return createCollectionAsync(); } - return Mono.empty(); + return Mono.just(this); }); } @@ -203,7 +204,7 @@ private JsonNode removeRedisPathPrefix(JSONObject object) { /** * Gets a record from the store. * - * @param key The key of the record to get. + * @param key The key of the record to get. * @param options The options for getting the record. * @return A Mono emitting the record. */ @@ -243,7 +244,7 @@ public Mono getAsync(String key, GetRecordOptions options) { /** * Gets a batch of records from the store. * - * @param keys The keys of the records to get. + * @param keys The keys of the records to get. * @param options The options for getting the records. * @return A Mono emitting a list of records. */ @@ -336,7 +337,7 @@ public Mono> upsertBatchAsync(List data, UpsertRecordOption /** * Deletes a record from the store. * - * @param key The key of the record to delete. + * @param key The key of the record to delete. * @param options The options for deleting the record. * @return A Mono representing the completion of the deletion operation. */ diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java index 45417980..df1e7544 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordCollectionFactory.java @@ -1,19 +1,20 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; +import com.azure.search.documents.indexes.SearchIndexAsyncClient; import redis.clients.jedis.JedisPooled; /** * Factory for creating Redis vector store record collections. - * */ public interface RedisVectorStoreRecordCollectionFactory { + /** * Creates a new vector store record collection. * - * @param client The Redis client. + * @param client The Redis client. * @param collectionName The name of the collection. - * @param options The options for the collection. + * @param options The options for the collection. * @return The collection. */ RedisVectorStoreRecordCollection createVectorStoreRecordCollection( diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordMapper.java index a4f5f798..f33a6b1d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/redis/RedisVectorStoreRecordMapper.java @@ -1,17 +1,15 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.connectors.data.redis; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.semantickernel.builders.SemanticKernelBuilder; import com.microsoft.semantickernel.data.VectorStoreRecordMapper; import com.microsoft.semantickernel.exceptions.SKException; - -import javax.annotation.Nullable; import java.util.AbstractMap; import java.util.Map.Entry; import java.util.function.Function; +import javax.annotation.Nullable; public class RedisVectorStoreRecordMapper extends VectorStoreRecordMapper> { diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStore.java index d778829a..a51f044c 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStore.java @@ -1,31 +1,28 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data; -import com.microsoft.semantickernel.data.VectorStoreRecordCollection; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import reactor.core.publisher.Mono; - +import java.util.List; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.util.List; +import reactor.core.publisher.Mono; /** * Represents a vector store. - * - * @param The type of the record collection. */ -public interface VectorStore> { +public interface VectorStore { /** * Gets a collection from the vector store. * - * @param collectionName The name of the collection. - * @param recordClass The class type of the record. + * @param collectionName The name of the collection. + * @param recordClass The class type of the record. * @param recordDefinition The record definition. * @return The collection. */ - RecordCollection getCollection( + VectorStoreRecordCollection getCollection( @Nonnull String collectionName, + @Nonnull Class keyClass, @Nonnull Class recordClass, @Nullable VectorStoreRecordDefinition recordDefinition); diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollection.java index 1466ac35..0b7319c0 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordCollection.java @@ -4,11 +4,11 @@ import com.microsoft.semantickernel.data.recordoptions.DeleteRecordOptions; import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; -import reactor.core.publisher.Mono; - import java.util.List; +import reactor.core.publisher.Mono; public interface VectorStoreRecordCollection { + /** * Gets the name of the collection. * @@ -28,14 +28,14 @@ public interface VectorStoreRecordCollection { * * @return A Mono representing the completion of the creation operation. */ - Mono createCollectionAsync(); + Mono> createCollectionAsync(); /** * Creates the collection in the store if it does not exist. * * @return A Mono representing the completion of the creation operation. */ - Mono createCollectionIfNotExistsAsync(); + Mono> createCollectionIfNotExistsAsync(); /** * Deletes the collection from the store. @@ -47,7 +47,7 @@ public interface VectorStoreRecordCollection { /** * Gets a record from the store. * - * @param key The key of the record to get. + * @param key The key of the record to get. * @param options The options for getting the record. * @return A Mono emitting the record. */ @@ -56,7 +56,7 @@ public interface VectorStoreRecordCollection { /** * Gets a batch of records from the store. * - * @param keys The keys of the records to get. + * @param keys The keys of the records to get. * @param options The options for getting the records. * @return A Mono emitting a list of records. */ @@ -65,7 +65,7 @@ public interface VectorStoreRecordCollection { /** * Inserts or updates a record in the store. * - * @param data The record to upsert. + * @param data The record to upsert. * @param options The options for upserting the record. * @return A Mono emitting the key of the upserted record. */ @@ -74,7 +74,7 @@ public interface VectorStoreRecordCollection { /** * Inserts or updates a batch of records in the store. * - * @param data The records to upsert. + * @param data The records to upsert. * @param options The options for upserting the records. * @return A Mono emitting a list of keys of the upserted records. */ @@ -83,7 +83,7 @@ public interface VectorStoreRecordCollection { /** * Deletes a record from the store. * - * @param key The key of the record to delete. + * @param key The key of the record to delete. * @param options The options for deleting the record. * @return A Mono representing the completion of the deletion operation. */ @@ -92,7 +92,7 @@ public interface VectorStoreRecordCollection { /** * Deletes a batch of records from the store. * - * @param keys The keys of the records to delete. + * @param keys The keys of the records to delete. * @param options The options for deleting the records. * @return A Mono representing the completion of the deletion operation. */ diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordMapper.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordMapper.java index 09420cf8..100c7e6d 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordMapper.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VectorStoreRecordMapper.java @@ -3,6 +3,7 @@ import com.microsoft.semantickernel.builders.SemanticKernelBuilder; +import javax.annotation.Nullable; import java.util.function.Function; /** @@ -12,6 +13,7 @@ * @param the storage model type */ public class VectorStoreRecordMapper { + @Nullable private final Function recordToStorageModelMapper; private final Function storageModelToRecordMapper; @@ -22,7 +24,7 @@ public class VectorStoreRecordMapper { * @param storageModelToRecordMapper the function to convert a storage model to a record */ protected VectorStoreRecordMapper( - Function recordToStorageModelMapper, + @Nullable Function recordToStorageModelMapper, Function storageModelToRecordMapper) { this.recordToStorageModelMapper = recordToStorageModelMapper; this.storageModelToRecordMapper = storageModelToRecordMapper; @@ -33,6 +35,7 @@ protected VectorStoreRecordMapper( * * @return the function to convert a record to a storage model */ + @Nullable public Function getRecordToStorageModelMapper() { return recordToStorageModelMapper; } diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java index ef074247..25e90ad9 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStore.java @@ -2,15 +2,16 @@ package com.microsoft.semantickernel.data; import com.microsoft.semantickernel.data.recorddefinition.VectorStoreRecordDefinition; -import reactor.core.publisher.Mono; - -import javax.annotation.Nonnull; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import reactor.core.publisher.Mono; + +public class VolatileVectorStore implements VectorStore { -public class VolatileVectorStore implements VectorStore> { private final Map> collections; public VolatileVectorStore() { @@ -25,9 +26,25 @@ public VolatileVectorStore() { * @return The collection. */ @Override - public VolatileVectorStoreRecordCollection getCollection( - @Nonnull String collectionName, @Nonnull Class recordClass, - VectorStoreRecordDefinition recordDefinition) { + public VectorStoreRecordCollection getCollection( + @Nonnull String collectionName, + @Nonnull Class keyClass, + @Nonnull Class recordClass, + @Nullable VectorStoreRecordDefinition recordDefinition) { + if (keyClass != String.class) { + throw new IllegalArgumentException("Volatile only supports string keys"); + } + + return (VectorStoreRecordCollection) getCollection( + collectionName, + recordClass, + recordDefinition); + } + + public VectorStoreRecordCollection getCollection( + @Nonnull String collectionName, + @Nonnull Class recordClass, + @Nullable VectorStoreRecordDefinition recordDefinition) { return new VolatileVectorStoreRecordCollection<>( collectionName, collections, diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java index 7a433dbb..07d55cdd 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollection.java @@ -8,18 +8,17 @@ import com.microsoft.semantickernel.data.recordoptions.GetRecordOptions; import com.microsoft.semantickernel.data.recordoptions.UpsertRecordOptions; import com.microsoft.semantickernel.exceptions.SKException; -import reactor.core.publisher.Mono; - import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; +import reactor.core.publisher.Mono; + +public class VolatileVectorStoreRecordCollection implements + VectorStoreRecordCollection { -public class VolatileVectorStoreRecordCollection - implements VectorStoreRecordCollection { private static final HashSet> supportedKeyTypes = new HashSet<>( Collections.singletonList(String.class)); private Map> collections; @@ -82,8 +81,9 @@ public Mono collectionExistsAsync() { * @return A Mono representing the completion of the creation operation. */ @Override - public Mono createCollectionAsync() { - return Mono.fromRunnable(() -> collections.put(collectionName, new ConcurrentHashMap<>())); + public Mono> createCollectionAsync() { + return Mono.fromRunnable(() -> collections.put(collectionName, new ConcurrentHashMap<>())) + .then(Mono.just(this)); } /** @@ -92,9 +92,10 @@ public Mono createCollectionAsync() { * @return A Mono representing the completion of the creation operation. */ @Override - public Mono createCollectionIfNotExistsAsync() { + public Mono> createCollectionIfNotExistsAsync() { return Mono - .fromRunnable(() -> collections.putIfAbsent(collectionName, new ConcurrentHashMap<>())); + .fromRunnable(() -> collections.putIfAbsent(collectionName, new ConcurrentHashMap<>())) + .then(Mono.just(this)); } /** @@ -110,7 +111,7 @@ public Mono deleteCollectionAsync() { /** * Gets a record from the store. * - * @param key The key of the record to get. + * @param key The key of the record to get. * @param options The options for getting the record. * @return A Mono emitting the record. */ @@ -122,7 +123,7 @@ public Mono getAsync(String key, GetRecordOptions options) { /** * Gets a batch of records from the store. * - * @param keys The keys of the records to get. + * @param keys The keys of the records to get. * @param options The options for getting the records. * @return A Mono emitting a list of records. */ @@ -188,7 +189,7 @@ public Mono> upsertBatchAsync(List data, UpsertRecordOption /** * Deletes a record from the store. * - * @param key The key of the record to delete. + * @param key The key of the record to delete. * @param options The options for deleting the record. * @return A Mono representing the completion of the deletion operation. */ diff --git a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java index 9a087adb..915b2166 100644 --- a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java +++ b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreRecordCollectionTest.java @@ -1,20 +1,18 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + public class VolatileVectorStoreRecordCollectionTest { private static VolatileVectorStoreRecordCollection recordCollection; diff --git a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreTest.java b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreTest.java index cfd52757..99c643e5 100644 --- a/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreTest.java +++ b/semantickernel-experimental/src/test/java/com/microsoft/semantickernel/data/VolatileVectorStoreTest.java @@ -1,17 +1,17 @@ // Copyright (c) Microsoft. All rights reserved. package com.microsoft.semantickernel.data; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -import java.util.Arrays; -import java.util.List; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + public class VolatileVectorStoreTest { + private static VolatileVectorStore vectorStore; @BeforeAll From 4d1d9204e5583e65cefeede6623581e7bb21b04f Mon Sep 17 00:00:00 2001 From: Milder Hernandez Cagua Date: Sun, 11 Aug 2024 22:46:08 -0700 Subject: [PATCH 21/28] Add suggestions --- api-test/integration-tests/pom.xml | 25 ------- .../JDBCVectorStoreDefaultQueryProvider.java | 72 +++++++++++-------- .../jdbc/JDBCVectorStoreQueryProvider.java | 22 ++++++ .../PostgreSQLVectorStoreQueryProvider.java | 38 +++++++++- 4 files changed, 102 insertions(+), 55 deletions(-) diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 89eb0d14..cc93c7fe 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -144,31 +144,6 @@ - - - io.fabric8 - docker-maven-plugin - 0.45.0 - - - build-image - package - - build - - - - - - - pgvector:latest - - ${project.basedir}/api-test/integration-tests/src/test/java/com/microsoft/semantickernel/tests/jdbc/pgvector/Dockerfile - - - - - diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java index df9b9adb..f1795083 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreDefaultQueryProvider.java @@ -25,26 +25,14 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import java.util.stream.Stream; public class JDBCVectorStoreDefaultQueryProvider implements JDBCVectorStoreQueryProvider { - /** - * Map of supported key types to their SQL types. - * Can be modified by subclasses. - */ - protected Map, String> supportedKeyTypes; - /** - * Map of supported data types to their SQL types. - * Can be modified by subclasses. - */ - protected Map, String> supportedDataTypes; - - /** - * Map of supported vector types to their SQL types. - * Can be modified by subclasses. - */ - protected Map, String> supportedVectorTypes; + private Map, String> supportedKeyTypes; + private Map, String> supportedDataTypes; + private Map, String> supportedVectorTypes; private final DataSource dataSource; private final String collectionsTable; private final String prefixForCollectionTables; @@ -95,14 +83,9 @@ public static Builder builder() { * @return the formatted wildcard string */ protected String getWildcardString(int wildcards) { - StringBuilder wildcardString = new StringBuilder(); - for (int i = 0; i < wildcards; ++i) { - wildcardString.append("?"); - if (i < wildcards - 1) { - wildcardString.append(", "); - } - } - return wildcardString.toString(); + return Stream.generate(() -> "?") + .limit(wildcards) + .collect(Collectors.joining(", ")); } /** @@ -133,6 +116,36 @@ protected String getCollectionTableName(String collectionName) { return validateSQLidentifier(prefixForCollectionTables + collectionName); } + /** + * Gets the supported key types and their corresponding SQL types. + * + * @return the supported key types + */ + @Override + public Map, String> getSupportedKeyTypes() { + return new HashMap<>(this.supportedKeyTypes); + } + + /** + * Gets the supported data types and their corresponding SQL types. + * + * @return the supported data types + */ + @Override + public Map, String> getSupportedDataTypes() { + return new HashMap<>(this.supportedDataTypes); + } + + /** + * Gets the supported vector types and their corresponding SQL types. + * + * @return the supported vector types + */ + @Override + public Map, String> getSupportedVectorTypes() { + return new HashMap<>(this.supportedVectorTypes); + } + /** * Prepares the vector store. * Executes any necessary setup steps for the vector store. @@ -165,11 +178,12 @@ public void validateSupportedTypes(Class recordClass, VectorStoreRecordDefinition recordDefinition) { VectorStoreRecordDefinition.validateSupportedTypes( Collections.singletonList(recordDefinition.getKeyDeclaredField(recordClass)), - supportedKeyTypes.keySet()); + getSupportedKeyTypes().keySet()); VectorStoreRecordDefinition.validateSupportedTypes( - recordDefinition.getDataDeclaredFields(recordClass), supportedDataTypes.keySet()); + recordDefinition.getDataDeclaredFields(recordClass), getSupportedDataTypes().keySet()); VectorStoreRecordDefinition.validateSupportedTypes( - recordDefinition.getVectorDeclaredFields(recordClass), supportedVectorTypes.keySet()); + recordDefinition.getVectorDeclaredFields(recordClass), + getSupportedVectorTypes().keySet()); } /** @@ -213,8 +227,8 @@ public void createCollection(String collectionName, Class recordClass, String createStorageTable = "CREATE TABLE IF NOT EXISTS " + getCollectionTableName(collectionName) + " (" + keyDeclaredField.getName() + " VARCHAR(255) PRIMARY KEY, " - + getColumnNamesAndTypes(dataDeclaredFields, supportedDataTypes) + ", " - + getColumnNamesAndTypes(vectorDeclaredFields, supportedVectorTypes) + ");"; + + getColumnNamesAndTypes(dataDeclaredFields, getSupportedDataTypes()) + ", " + + getColumnNamesAndTypes(vectorDeclaredFields, getSupportedVectorTypes()) + ");"; String insertCollectionQuery = "INSERT INTO " + validateSQLidentifier(collectionsTable) + " (collectionId) VALUES (?)"; diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java index 386ef5b2..6009b885 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/jdbc/JDBCVectorStoreQueryProvider.java @@ -10,6 +10,7 @@ import java.sql.ResultSet; import java.util.List; +import java.util.Map; /** * The JDBC vector store query provider. @@ -26,6 +27,27 @@ public interface JDBCVectorStoreQueryProvider { */ String DEFAULT_PREFIX_FOR_COLLECTION_TABLES = "SKCollection_"; + /** + * Gets the supported key types and their corresponding SQL types. + * + * @return the supported key types + */ + Map, String> getSupportedKeyTypes(); + + /** + * Gets the supported data types and their corresponding SQL types. + * + * @return the supported data types + */ + Map, String> getSupportedDataTypes(); + + /** + * Gets the supported vector types and their corresponding SQL types. + * + * @return the supported vector types + */ + Map, String> getSupportedVectorTypes(); + /** * Prepares the vector store. * Executes any necessary setup steps for the vector store. diff --git a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java index 81e32c2b..d9d5deff 100644 --- a/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java +++ b/semantickernel-experimental/src/main/java/com/microsoft/semantickernel/connectors/data/postgres/PostgreSQLVectorStoreQueryProvider.java @@ -25,9 +25,15 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; public class PostgreSQLVectorStoreQueryProvider extends JDBCVectorStoreDefaultQueryProvider implements JDBCVectorStoreQueryProvider { + + private Map, String> supportedKeyTypes; + private Map, String> supportedDataTypes; + private Map, String> supportedVectorTypes; + private final DataSource dataSource; private final String collectionsTable; private final String prefixForCollectionTables; @@ -63,6 +69,36 @@ private PostgreSQLVectorStoreQueryProvider(DataSource dataSource, String collect supportedVectorTypes.put(Collection.class, "VECTOR(%d)"); } + /** + * Gets the supported key types and their corresponding SQL types. + * + * @return the supported key types + */ + @Override + public Map, String> getSupportedKeyTypes() { + return new HashMap<>(this.supportedKeyTypes); + } + + /** + * Gets the supported data types and their corresponding SQL types. + * + * @return the supported data types + */ + @Override + public Map, String> getSupportedDataTypes() { + return new HashMap<>(this.supportedDataTypes); + } + + /** + * Gets the supported vector types and their corresponding SQL types. + * + * @return the supported vector types + */ + @Override + public Map, String> getSupportedVectorTypes() { + return new HashMap<>(this.supportedVectorTypes); + } + /** * Creates a new builder. * @return the builder @@ -103,7 +139,7 @@ private String getColumnNamesAndTypesForVectorFields(List Date: Tue, 13 Aug 2024 18:48:01 +0100 Subject: [PATCH 22/28] Allow Handlebars templates to extract fields from Java beans --- .../KernelFunctionFromPrompt.java | 2 + .../PromptTemplateConfig.java | 45 +++++++++++++++---- .../PromptTemplateOption.java | 16 +++++++ .../handlebars/HandlebarsPromptTemplate.java | 16 ++++--- .../PromptTemplateConfigTest.java | 2 + 5 files changed, 67 insertions(+), 14 deletions(-) create mode 100644 semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateOption.java diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java index e3be61e8..5d124d84 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/KernelFunctionFromPrompt.java @@ -20,6 +20,7 @@ import com.microsoft.semantickernel.services.chatcompletion.ChatCompletionService; import com.microsoft.semantickernel.services.textcompletion.TextGenerationService; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -439,6 +440,7 @@ public KernelFunction build() { name, template, templateFormat, + Collections.emptySet(), description, inputVariables, outputVariable, diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateConfig.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateConfig.java index 02adb289..5f69ad46 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateConfig.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateConfig.java @@ -11,9 +11,11 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import javax.annotation.Nullable; /** @@ -43,6 +45,7 @@ public class PromptTemplateConfig { @Nullable private final String template; private final String templateFormat; + private final Set promptTemplateOptions; @Nullable private final String description; private final List inputVariables; @@ -61,6 +64,7 @@ protected PromptTemplateConfig(String template) { DEFAULT_CONFIG_NAME, template, SEMANTIC_KERNEL_TEMPLATE_FORMAT, + Collections.emptySet(), "", Collections.emptyList(), new OutputVariable(String.class.getName(), "out"), @@ -70,14 +74,15 @@ protected PromptTemplateConfig(String template) { /** * Constructor for a prompt template config * - * @param schema Schema version - * @param name Name of the template - * @param template Template string - * @param templateFormat Template format - * @param description Description of the template - * @param inputVariables Input variables - * @param outputVariable Output variable - * @param executionSettings Execution settings + * @param schema Schema version + * @param name Name of the template + * @param template Template string + * @param templateFormat Template format + * @param promptTemplateOptions Prompt template options + * @param description Description of the template + * @param inputVariables Input variables + * @param outputVariable Output variable + * @param executionSettings Execution settings */ @JsonCreator public PromptTemplateConfig( @@ -85,6 +90,7 @@ public PromptTemplateConfig( @Nullable @JsonProperty("name") String name, @Nullable @JsonProperty("template") String template, @Nullable @JsonProperty(value = "template_format", defaultValue = SEMANTIC_KERNEL_TEMPLATE_FORMAT) String templateFormat, + @Nullable @JsonProperty(value = "prompt_template_options") Set promptTemplateOptions, @Nullable @JsonProperty("description") String description, @Nullable @JsonProperty("input_variables") List inputVariables, @Nullable @JsonProperty("output_variable") OutputVariable outputVariable, @@ -96,6 +102,10 @@ public PromptTemplateConfig( templateFormat = SEMANTIC_KERNEL_TEMPLATE_FORMAT; } this.templateFormat = templateFormat; + if (promptTemplateOptions == null) { + promptTemplateOptions = new HashSet<>(); + } + this.promptTemplateOptions = promptTemplateOptions; this.description = description; if (inputVariables == null) { this.inputVariables = new ArrayList<>(); @@ -127,6 +137,7 @@ protected PromptTemplateConfig( @Nullable String name, @Nullable String template, @Nullable String templateFormat, + @Nullable Set promptTemplateOptions, @Nullable String description, @Nullable List inputVariables, @Nullable OutputVariable outputVariable, @@ -136,6 +147,7 @@ protected PromptTemplateConfig( name, template, templateFormat, + promptTemplateOptions, description, inputVariables, outputVariable, @@ -152,6 +164,7 @@ public PromptTemplateConfig(PromptTemplateConfig promptTemplate) { promptTemplate.name, promptTemplate.template, promptTemplate.templateFormat, + promptTemplate.promptTemplateOptions, promptTemplate.description, promptTemplate.inputVariables, promptTemplate.outputVariable, @@ -300,6 +313,15 @@ public int getSchema() { return schema; } + /** + * Get the prompt template options of the prompt template config. + * + * @return The prompt template options of the prompt template config. + */ + public Set getPromptTemplateOptions() { + return Collections.unmodifiableSet(promptTemplateOptions); + } + /** * Create a builder for a prompt template config which is a clone of the current object. * @@ -358,6 +380,7 @@ public static class Builder { @Nullable private String template; private String templateFormat = SEMANTIC_KERNEL_TEMPLATE_FORMAT; + private final Set promptTemplateOptions = new HashSet<>(); @Nullable private String description = null; private List inputVariables = new ArrayList<>(); @@ -433,6 +456,11 @@ public Builder withTemplateFormat(String templateFormat) { return this; } + public Builder addPromptTemplateOption(PromptTemplateOption option) { + promptTemplateOptions.add(option); + return this; + } + /** * Set the inputVariables of the prompt template config. * @@ -477,6 +505,7 @@ public PromptTemplateConfig build() { name, template, templateFormat, + promptTemplateOptions, description, inputVariables, outputVariable, diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateOption.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateOption.java new file mode 100644 index 00000000..5d244613 --- /dev/null +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateOption.java @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft. All rights reserved. +package com.microsoft.semantickernel.semanticfunctions; + +public enum PromptTemplateOption { + /** + * Allow methods on objects provided as arguments to an invocation, to be invoked when rendering + * a template and its return value used. Typically, this would be used to call a getter on an + * object i.e. {@code {{#each users}} {{userName}} {{/each}} } on a handlebars template will + * call the method {@code getUserName()} on each object in {@code users}. + *

+ * WARNING: If this option is used, ensure that your template is trusted, and that objects added + * as arguments to an invocation, do not contain methods that are unsafe to be invoked when + * rendering a template. + */ + ALLOW_CONTEXT_VARIABLE_METHOD_CALLS_UNSAFE +} \ No newline at end of file diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplate.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplate.java index 2e7c260e..ec701ce3 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplate.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/templateengine/handlebars/HandlebarsPromptTemplate.java @@ -9,6 +9,7 @@ import com.github.jknack.handlebars.Helper; import com.github.jknack.handlebars.Options; import com.github.jknack.handlebars.ValueResolver; +import com.github.jknack.handlebars.context.JavaBeanValueResolver; import com.microsoft.semantickernel.Kernel; import com.microsoft.semantickernel.contextvariables.ContextVariable; import com.microsoft.semantickernel.contextvariables.ContextVariableType; @@ -21,6 +22,7 @@ import com.microsoft.semantickernel.semanticfunctions.KernelFunctionArguments; import com.microsoft.semantickernel.semanticfunctions.PromptTemplate; import com.microsoft.semantickernel.semanticfunctions.PromptTemplateConfig; +import com.microsoft.semantickernel.semanticfunctions.PromptTemplateOption; import com.microsoft.semantickernel.services.chatcompletion.ChatMessageContent; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import java.io.IOException; @@ -35,7 +37,6 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import org.apache.commons.text.StringEscapeUtils; import reactor.core.publisher.Mono; /** @@ -168,7 +169,7 @@ public Set> propertySet(Object context) { } } - private static class HandleBarsPromptTemplateHandler { + private class HandleBarsPromptTemplateHandler { private final String template; private final Handlebars handlebars; @@ -181,7 +182,7 @@ public HandleBarsPromptTemplateHandler( this.template = template; this.handlebars = new Handlebars(); this.handlebars - .registerHelper("message", HandleBarsPromptTemplateHandler::handleMessage) + .registerHelper("message", this::handleMessage) .registerHelper("each", handleEach(context)) .with(EscapingStrategy.XML); @@ -190,7 +191,7 @@ public HandleBarsPromptTemplateHandler( // TODO: 1.0 Add more helpers } - private static Helper handleEach(InvocationContext invocationContext) { + private Helper handleEach(InvocationContext invocationContext) { return (context, options) -> { if (context instanceof ContextVariable) { return ((ContextVariable) context) @@ -227,7 +228,7 @@ private static Helper handleEach(InvocationContext invocationContext) { } @Nullable - private static CharSequence handleMessage(Object context, Options options) + private CharSequence handleMessage(Object context, Options options) throws IOException { String role = options.hash("role"); String content = (String) options.fn(context); @@ -258,7 +259,10 @@ public Mono render(KernelFunctionArguments variables) { resolvers.add(new MessageResolver()); resolvers.add(new ContextVariableResolver()); - // resolvers.addAll(ValueResolver.defaultValueResolvers()); + if (promptTemplate.getPromptTemplateOptions() + .contains(PromptTemplateOption.ALLOW_CONTEXT_VARIABLE_METHOD_CALLS_UNSAFE)) { + resolvers.add(JavaBeanValueResolver.INSTANCE); + } Context context = Context .newBuilder(variables) diff --git a/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateConfigTest.java b/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateConfigTest.java index d4141dcf..ea34400a 100644 --- a/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateConfigTest.java +++ b/semantickernel-api/src/test/java/com/microsoft/semantickernel/semanticfunctions/PromptTemplateConfigTest.java @@ -5,6 +5,7 @@ import com.microsoft.semantickernel.orchestration.PromptExecutionSettings; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import org.junit.jupiter.api.Test; @@ -28,6 +29,7 @@ void testInstanceMadeWithBuilderEqualsInstanceMadeWithConstructor() { name, template, "semantic-kernel", + Collections.emptySet(), description, inputVariables, outputVariable, From c65cf68eef308ecb010828d741750855328b760d Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Wed, 14 Aug 2024 13:04:22 +0100 Subject: [PATCH 23/28] Changelog. Code format. Verion bumps --- CHANGELOG.md | 10 +++++++++- .../chatcompletion/OpenAIChatCompletion.java | 8 ++++---- api-test/integration-tests/pom.xml | 14 +++++++------- pom.xml | 13 +++++++++++++ .../semantickernel-syntax-examples/pom.xml | 8 ++++---- .../booking-agent-m365/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 12 ++++++------ .../semantickernel-openapi-plugin/pom.xml | 15 --------------- semantickernel-api/pom.xml | 2 -- .../services/chatcompletion/ChatHistory.java | 3 ++- semantickernel-experimental/pom.xml | 4 +--- 11 files changed, 47 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8e406be..86a40fcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,18 @@ - Add fluent returns to `ChatHistory` `addXMessage` methods. - Add user agent opt-out for OpenAI requests by setting the property `semantic-kernel.useragent-disable` to `true`. - Add several convenience `invokePromptAsync` methods to `Kernel`. +- Allow Handlebars templates to call Javabean getters to extract data from invocation arguments. +- Improve thread safety of `ChatHistory`. + +#### Experimental Changes + +- Add JDBC vector store #### Non-API Changes -- Add custom type Conversion example, CustomTypes_Example +- Add custom type Conversion example, `CustomTypes_Example` +- Dependency updates and pom cleanup +- Documentation updates # 1.2.0 diff --git a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java index ca8eb5b3..5442e51e 100644 --- a/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java +++ b/aiservices/openai/src/main/java/com/microsoft/semantickernel/aiservices/openai/chatcompletion/OpenAIChatCompletion.java @@ -357,7 +357,8 @@ private Mono internalChatMessageContentsAsync( // If we don't want to attempt to invoke any functions // Or if we are auto-invoking, but we somehow end up with other than 1 choice even though only 1 was requested if (autoInvokeAttempts == 0 || responseMessages.size() != 1) { - List> chatMessageContents = getChatMessageContentsAsync(completions); + List> chatMessageContents = getChatMessageContentsAsync( + completions); return Mono.just(messages.addChatMessage(chatMessageContents)); } // Or if there are no tool calls to be done @@ -365,7 +366,7 @@ private Mono internalChatMessageContentsAsync( List toolCalls = response.getToolCalls(); if (toolCalls == null || toolCalls.isEmpty()) { List> chatMessageContents = getChatMessageContentsAsync( - completions); + completions); return Mono.just(messages.addChatMessage(chatMessageContents)); } @@ -603,8 +604,7 @@ private List> getChatMessageContentsAsync( .filter(Objects::nonNull) .collect(Collectors.toList()); - List> chatMessageContent = - responseMessages + List> chatMessageContent = responseMessages .stream() .map(response -> { try { diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index cc93c7fe..b06a664f 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -1,6 +1,7 @@ - + 4.0.0 @@ -65,18 +66,18 @@ org.xerial sqlite-jdbc - 3.44.1.0 + 3.46.0.0 - mysql - mysql-connector-java - 8.0.33 + com.mysql + mysql-connector-j + 9.0.0 test org.postgresql postgresql - 42.7.2 + 42.7.3 @@ -103,7 +104,6 @@ org.wiremock wiremock - 3.3.1 test diff --git a/pom.xml b/pom.xml index a846ccab..d728b645 100644 --- a/pom.xml +++ b/pom.xml @@ -147,6 +147,19 @@ spotbugs-annotations ${spotbugs.version} + + + org.wiremock + wiremock + 3.9.1 + test + + + org.mockito + mockito-junit-jupiter + 5.12.0 + test + diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index dcf92999..28d79878 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -85,14 +85,14 @@ com.google.cloud google-cloud-vertexai - 1.1.0 + 1.6.0 compile - mysql - mysql-connector-java - 8.0.33 + com.mysql + mysql-connector-j + 9.0.0 diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index 0247bef0..621fe1b1 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -52,7 +52,7 @@ com.microsoft.graph microsoft-graph - 6.5.1 + 6.13.0 diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index ac17a0db..401b1f46 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -39,29 +39,29 @@ org.springframework.boot spring-boot-test - 3.2.1 + 3.3.2 test org.assertj assertj-core - 3.25.1 + 3.26.3 test org.springframework.boot spring-boot-autoconfigure - 3.2.1 + 3.3.2 org.springframework.boot spring-boot - 3.2.1 + 3.3.1 org.springframework spring-test - 6.1.2 + 6.1.10 test @@ -72,7 +72,7 @@ org.junit.jupiter junit-jupiter-api - 5.10.1 + 5.10.3 test diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index 5424f254..a7be951b 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -68,21 +68,6 @@ com.microsoft.semantic-kernel semantickernel-aiservices-openai - - org.apache.logging.log4j - log4j-api - test - - - org.apache.logging.log4j - log4j-core - test - - - org.apache.logging.log4j - log4j-slf4j2-impl - test - org.junit.jupiter junit-jupiter-api diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index 6171818b..5c688f75 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -67,7 +67,6 @@ org.wiremock wiremock - 3.3.1 test @@ -80,7 +79,6 @@ org.mockito mockito-junit-jupiter - 5.11.0 test diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java index a8303061..903669b3 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java @@ -69,7 +69,8 @@ public Optional> getLastMessage() { if (chatMessageContents.isEmpty()) { return Optional.empty(); } - return Optional.of(((ConcurrentLinkedQueue>)chatMessageContents).peek()); + return Optional + .of(((ConcurrentLinkedQueue>) chatMessageContents).peek()); } /** diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 3efd8ab0..a16d7a19 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -64,7 +64,6 @@ org.wiremock wiremock - 3.3.1 test @@ -77,7 +76,6 @@ org.mockito mockito-junit-jupiter - 5.11.0 test @@ -112,7 +110,7 @@ org.postgresql postgresql - 42.7.2 + 42.7.3 From b8a65e79e40071aea079a3df5772c4a5bcdc77f2 Mon Sep 17 00:00:00 2001 From: GitHub Date: Wed, 14 Aug 2024 18:39:54 +0100 Subject: [PATCH 24/28] [maven-release-plugin] prepare release java-1.2.1 --- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 5 ++--- api-test/pom.xml | 2 +- pom.xml | 4 ++-- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-bom/pom.xml | 4 ++-- semantickernel-experimental/pom.xml | 2 +- 20 files changed, 23 insertions(+), 24 deletions(-) diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index 0ca6600e..87137409 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.1-SNAPSHOT + 1.2.1 ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index 7743b989..aa59df34 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.1-SNAPSHOT + 1.2.1 ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index dbb1980c..3f1e093b 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.1-SNAPSHOT + 1.2.1 ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index b06a664f..3ce9831f 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -1,13 +1,12 @@ - + 4.0.0 com.microsoft.semantic-kernel api-test - 1.2.1-SNAPSHOT + 1.2.1 ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index 7effdac3..c0bdc949 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.1-SNAPSHOT + 1.2.1 ../pom.xml diff --git a/pom.xml b/pom.xml index d728b645..b6e0d987 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.1-SNAPSHOT + 1.2.1 pom https://www.github.com/microsoft/semantic-kernel @@ -825,6 +825,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - HEAD + java-1.2.1 diff --git a/samples/pom.xml b/samples/pom.xml index a153ef2f..f548f9a0 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.1-SNAPSHOT + 1.2.1 ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index 673abe6d..8b81dcb1 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.1-SNAPSHOT + 1.2.1 ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 28d79878..f5be8c5f 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.2.1-SNAPSHOT + 1.2.1 ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index 621fe1b1..09878c2b 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.2.1-SNAPSHOT + 1.2.1 ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index 586c7969..9aa87c14 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.1-SNAPSHOT + 1.2.1 ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index 401b1f46..65343038 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.2.1-SNAPSHOT + 1.2.1 ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index 40dcdbc0..be828cd6 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.2.1-SNAPSHOT + 1.2.1 ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index e7a1b516..a65f847b 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.1-SNAPSHOT + 1.2.1 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index 1a526043..44715299 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.1-SNAPSHOT + 1.2.1 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index a7be951b..7fe7a594 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.2.1-SNAPSHOT + 1.2.1 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index 952ff6af..4ec391d3 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.2.1-SNAPSHOT + 1.2.1 ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index 5c688f75..1920d5f8 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.1-SNAPSHOT + 1.2.1 ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 180d8da8..3239d819 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.2.1-SNAPSHOT + 1.2.1 pom Semantic Kernel Java BOM @@ -256,6 +256,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - HEAD + java-1.2.1 diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index a16d7a19..92d3269e 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.1-SNAPSHOT + 1.2.1 semantickernel-experimental From cb1a616c48a34f1deb51881aab85e3fb66db21f6 Mon Sep 17 00:00:00 2001 From: GitHub Date: Wed, 14 Aug 2024 18:39:54 +0100 Subject: [PATCH 25/28] [maven-release-plugin] prepare for next development iteration --- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- pom.xml | 4 ++-- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-bom/pom.xml | 4 ++-- semantickernel-experimental/pom.xml | 2 +- 20 files changed, 22 insertions(+), 22 deletions(-) diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index 87137409..3ae86506 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.1 + 1.2.2-SNAPSHOT ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index aa59df34..6b2ce004 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.1 + 1.2.2-SNAPSHOT ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index 3f1e093b..5b460fa3 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.1 + 1.2.2-SNAPSHOT ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index 3ce9831f..b9910a73 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.2.1 + 1.2.2-SNAPSHOT ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index c0bdc949..4349a6ee 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.1 + 1.2.2-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index b6e0d987..8eaf3b2d 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.1 + 1.2.2-SNAPSHOT pom https://www.github.com/microsoft/semantic-kernel @@ -825,6 +825,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.2.1 + HEAD diff --git a/samples/pom.xml b/samples/pom.xml index f548f9a0..a0353407 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.1 + 1.2.2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index 8b81dcb1..d6e45028 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.1 + 1.2.2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index f5be8c5f..1790d76c 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.2.1 + 1.2.2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index 09878c2b..4f7b0c37 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.2.1 + 1.2.2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index 9aa87c14..87a6c2ee 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.1 + 1.2.2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index 65343038..9dfcea2a 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.2.1 + 1.2.2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index be828cd6..5ff0d1b2 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.2.1 + 1.2.2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index a65f847b..0654fa3e 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.1 + 1.2.2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index 44715299..81422168 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.1 + 1.2.2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index 7fe7a594..1bec87db 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.2.1 + 1.2.2-SNAPSHOT ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index 4ec391d3..d45eeb03 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.2.1 + 1.2.2-SNAPSHOT ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index 1920d5f8..5e1d89ec 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.1 + 1.2.2-SNAPSHOT ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 3239d819..438f9908 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.2.1 + 1.2.2-SNAPSHOT pom Semantic Kernel Java BOM @@ -256,6 +256,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - java-1.2.1 + HEAD diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 92d3269e..8c6ffde2 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.1 + 1.2.2-SNAPSHOT semantickernel-experimental From 006360660214378417550f55df7c6aa79fa72691 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Wed, 14 Aug 2024 20:06:37 +0100 Subject: [PATCH 26/28] Fix ChatHistory.getLast --- .../services/chatcompletion/ChatHistory.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java index 903669b3..ea910c01 100644 --- a/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java +++ b/semantickernel-api/src/main/java/com/microsoft/semantickernel/services/chatcompletion/ChatHistory.java @@ -5,13 +5,11 @@ import com.microsoft.semantickernel.services.chatcompletion.message.ChatMessageTextContent; import java.nio.charset.Charset; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.Spliterator; -import java.util.concurrent.ConcurrentLinkedQueue; import java.util.function.Consumer; import javax.annotation.Nullable; @@ -20,7 +18,7 @@ */ public class ChatHistory implements Iterable> { - private final Collection> chatMessageContents; + private final List> chatMessageContents; /** * The default constructor @@ -35,7 +33,7 @@ public ChatHistory() { * @param instructions The instructions to add to the chat history */ public ChatHistory(@Nullable String instructions) { - this.chatMessageContents = new ConcurrentLinkedQueue<>(); + this.chatMessageContents = Collections.synchronizedList(new ArrayList<>()); if (instructions != null) { this.chatMessageContents.add( ChatMessageTextContent.systemMessage(instructions)); @@ -48,7 +46,8 @@ public ChatHistory(@Nullable String instructions) { * @param chatMessageContents The chat message contents to add to the chat history */ public ChatHistory(List> chatMessageContents) { - this.chatMessageContents = new ConcurrentLinkedQueue<>(chatMessageContents); + this.chatMessageContents = Collections + .synchronizedList(new ArrayList<>(chatMessageContents)); } /** @@ -70,7 +69,7 @@ public Optional> getLastMessage() { return Optional.empty(); } return Optional - .of(((ConcurrentLinkedQueue>) chatMessageContents).peek()); + .of(chatMessageContents.get(chatMessageContents.size() - 1)); } /** From 97a132a28cf901cd8e9e34f870c1292e312a00e2 Mon Sep 17 00:00:00 2001 From: John Oliver <1615532+johnoliver@users.noreply.github.com> Date: Wed, 14 Aug 2024 20:14:47 +0100 Subject: [PATCH 27/28] Fix changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86a40fcc..8942e497 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# 1.2.1 +# 1.2.2 - Fix bug in `FunctionInvocation` not using per-invocation type conversion when calling `withResultType`. - Fix bug in Global Hooks not being invoked under certain circumstances. From cdab2eb28c577b9731418b418bfe5701e4622bea Mon Sep 17 00:00:00 2001 From: GitHub Date: Wed, 14 Aug 2024 20:21:06 +0100 Subject: [PATCH 28/28] [maven-release-plugin] prepare release java-1.2.2 --- aiservices/google/pom.xml | 2 +- aiservices/huggingface/pom.xml | 2 +- aiservices/openai/pom.xml | 2 +- api-test/integration-tests/pom.xml | 2 +- api-test/pom.xml | 2 +- pom.xml | 4 ++-- samples/pom.xml | 2 +- samples/semantickernel-concepts/pom.xml | 2 +- .../semantickernel-syntax-examples/pom.xml | 2 +- samples/semantickernel-demos/booking-agent-m365/pom.xml | 2 +- samples/semantickernel-demos/pom.xml | 2 +- .../semantickernel-spring-starter/pom.xml | 2 +- samples/semantickernel-demos/sk-presidio-sample/pom.xml | 2 +- samples/semantickernel-learn-resources/pom.xml | 2 +- samples/semantickernel-sample-plugins/pom.xml | 2 +- .../semantickernel-openapi-plugin/pom.xml | 2 +- .../semantickernel-presidio-plugin/pom.xml | 2 +- semantickernel-api/pom.xml | 2 +- semantickernel-bom/pom.xml | 4 ++-- semantickernel-experimental/pom.xml | 2 +- 20 files changed, 22 insertions(+), 22 deletions(-) diff --git a/aiservices/google/pom.xml b/aiservices/google/pom.xml index 3ae86506..f9ff65a4 100644 --- a/aiservices/google/pom.xml +++ b/aiservices/google/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.2-SNAPSHOT + 1.2.2 ../../pom.xml diff --git a/aiservices/huggingface/pom.xml b/aiservices/huggingface/pom.xml index 6b2ce004..ccfbb689 100644 --- a/aiservices/huggingface/pom.xml +++ b/aiservices/huggingface/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.2-SNAPSHOT + 1.2.2 ../../pom.xml diff --git a/aiservices/openai/pom.xml b/aiservices/openai/pom.xml index 5b460fa3..d6a3762e 100644 --- a/aiservices/openai/pom.xml +++ b/aiservices/openai/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.2-SNAPSHOT + 1.2.2 ../../pom.xml diff --git a/api-test/integration-tests/pom.xml b/api-test/integration-tests/pom.xml index b9910a73..b3ec6563 100644 --- a/api-test/integration-tests/pom.xml +++ b/api-test/integration-tests/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel api-test - 1.2.2-SNAPSHOT + 1.2.2 ../pom.xml diff --git a/api-test/pom.xml b/api-test/pom.xml index 4349a6ee..47d10b04 100644 --- a/api-test/pom.xml +++ b/api-test/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.2-SNAPSHOT + 1.2.2 ../pom.xml diff --git a/pom.xml b/pom.xml index 8eaf3b2d..84e20d0c 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.2-SNAPSHOT + 1.2.2 pom https://www.github.com/microsoft/semantic-kernel @@ -825,6 +825,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - HEAD + java-1.2.2 diff --git a/samples/pom.xml b/samples/pom.xml index a0353407..a62aa7c4 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.2-SNAPSHOT + 1.2.2 ../pom.xml diff --git a/samples/semantickernel-concepts/pom.xml b/samples/semantickernel-concepts/pom.xml index d6e45028..cd4b855c 100644 --- a/samples/semantickernel-concepts/pom.xml +++ b/samples/semantickernel-concepts/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.2-SNAPSHOT + 1.2.2 ../pom.xml diff --git a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml index 1790d76c..27c3420d 100644 --- a/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml +++ b/samples/semantickernel-concepts/semantickernel-syntax-examples/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-concepts - 1.2.2-SNAPSHOT + 1.2.2 ../pom.xml diff --git a/samples/semantickernel-demos/booking-agent-m365/pom.xml b/samples/semantickernel-demos/booking-agent-m365/pom.xml index 4f7b0c37..a0b84de8 100644 --- a/samples/semantickernel-demos/booking-agent-m365/pom.xml +++ b/samples/semantickernel-demos/booking-agent-m365/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.2.2-SNAPSHOT + 1.2.2 ../pom.xml diff --git a/samples/semantickernel-demos/pom.xml b/samples/semantickernel-demos/pom.xml index 87a6c2ee..b5b4fe7b 100644 --- a/samples/semantickernel-demos/pom.xml +++ b/samples/semantickernel-demos/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.2-SNAPSHOT + 1.2.2 ../pom.xml diff --git a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml index 9dfcea2a..e9fab05f 100644 --- a/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml +++ b/samples/semantickernel-demos/semantickernel-spring-starter/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.2.2-SNAPSHOT + 1.2.2 ../pom.xml diff --git a/samples/semantickernel-demos/sk-presidio-sample/pom.xml b/samples/semantickernel-demos/sk-presidio-sample/pom.xml index 5ff0d1b2..8300dcec 100644 --- a/samples/semantickernel-demos/sk-presidio-sample/pom.xml +++ b/samples/semantickernel-demos/sk-presidio-sample/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-demos - 1.2.2-SNAPSHOT + 1.2.2 ../pom.xml diff --git a/samples/semantickernel-learn-resources/pom.xml b/samples/semantickernel-learn-resources/pom.xml index 0654fa3e..c8fb38ba 100644 --- a/samples/semantickernel-learn-resources/pom.xml +++ b/samples/semantickernel-learn-resources/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.2-SNAPSHOT + 1.2.2 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/pom.xml b/samples/semantickernel-sample-plugins/pom.xml index 81422168..71d41a94 100644 --- a/samples/semantickernel-sample-plugins/pom.xml +++ b/samples/semantickernel-sample-plugins/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-samples-parent - 1.2.2-SNAPSHOT + 1.2.2 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml index 1bec87db..e1804b6f 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-openapi-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.2.2-SNAPSHOT + 1.2.2 ../pom.xml diff --git a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml index d45eeb03..e5ce6b0f 100644 --- a/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml +++ b/samples/semantickernel-sample-plugins/semantickernel-presidio-plugin/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-sample-plugins - 1.2.2-SNAPSHOT + 1.2.2 ../pom.xml diff --git a/semantickernel-api/pom.xml b/semantickernel-api/pom.xml index 5e1d89ec..c6bbae4a 100644 --- a/semantickernel-api/pom.xml +++ b/semantickernel-api/pom.xml @@ -6,7 +6,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.2-SNAPSHOT + 1.2.2 ../pom.xml diff --git a/semantickernel-bom/pom.xml b/semantickernel-bom/pom.xml index 438f9908..1322666f 100644 --- a/semantickernel-bom/pom.xml +++ b/semantickernel-bom/pom.xml @@ -5,7 +5,7 @@ com.microsoft.semantic-kernel semantickernel-bom - 1.2.2-SNAPSHOT + 1.2.2 pom Semantic Kernel Java BOM @@ -256,6 +256,6 @@ https://github.com/microsoft/semantic-kernel scm:git:https://github.com/microsoft/semantic-kernel.git scm:git:https://github.com/microsoft/semantic-kernel.git - HEAD + java-1.2.2 diff --git a/semantickernel-experimental/pom.xml b/semantickernel-experimental/pom.xml index 8c6ffde2..44772dc2 100644 --- a/semantickernel-experimental/pom.xml +++ b/semantickernel-experimental/pom.xml @@ -4,7 +4,7 @@ com.microsoft.semantic-kernel semantickernel-parent - 1.2.2-SNAPSHOT + 1.2.2 semantickernel-experimental