diff --git a/ollama-local/output/01-chat-reload-after-unload.txt b/ollama-local/output/01-chat-reload-after-unload.txt index 17eadc2..54f9c1f 100644 --- a/ollama-local/output/01-chat-reload-after-unload.txt +++ b/ollama-local/output/01-chat-reload-after-unload.txt @@ -1,10 +1,10 @@ /api/ps immediately after the unload call: {"models":[]} prompt: "Reply with a single short sentence: why do developers like small local models?" -response: Developers often prefer small local models because they are more efficient, faster, and easier to deploy and train. +response: Developers often appreciate small local models because they are more efficient and can provide quicker, more relevant results for their applications. -total-duration: 901ms -load-duration: 1ms -prompt-eval-count: 44, prompt-eval-duration: 39ms -eval-count: 23, eval-duration: 856ms -26.87 tokens/sec (eval-count / eval-duration) \ No newline at end of file +total-duration: 1076ms +load-duration: 2ms +prompt-eval-count: 44, prompt-eval-duration: 41ms +eval-count: 27, eval-duration: 1028ms +26.25 tokens/sec (eval-count / eval-duration) \ No newline at end of file diff --git a/ollama-local/output/02-chat-back-to-back-call.txt b/ollama-local/output/02-chat-back-to-back-call.txt index 8970e73..a27b401 100644 --- a/ollama-local/output/02-chat-back-to-back-call.txt +++ b/ollama-local/output/02-chat-back-to-back-call.txt @@ -1,8 +1,8 @@ prompt: "Reply with a single short sentence: what is Testcontainers for?" -response: Testcontainers is a popular containerization platform that makes it easy to create and manage application containers for testing, development, and production environments. +response: Testcontainers is a containerization platform that simplifies the process of building and deploying applications quickly and efficiently. -total-duration: 722ms +total-duration: 662ms load-duration: 2ms -prompt-eval-count: 42, prompt-eval-duration: 61ms -eval-count: 17, eval-duration: 653ms -26.02 tokens/sec (eval-count / eval-duration) \ No newline at end of file +prompt-eval-count: 42, prompt-eval-duration: 47ms +eval-count: 16, eval-duration: 607ms +26.33 tokens/sec (eval-count / eval-duration) \ No newline at end of file diff --git a/ollama-local/src/test/java/com/ankurm/ollamalocal/LocalChatAndEmbeddingTest.java b/ollama-local/src/test/java/com/ankurm/ollamalocal/LocalChatAndEmbeddingTest.java index c88c439..33b6fe6 100644 --- a/ollama-local/src/test/java/com/ankurm/ollamalocal/LocalChatAndEmbeddingTest.java +++ b/ollama-local/src/test/java/com/ankurm/ollamalocal/LocalChatAndEmbeddingTest.java @@ -18,8 +18,6 @@ import org.testcontainers.utility.DockerImageName; import org.springframework.ai.chat.client.ChatClient; import org.springframework.ai.chat.metadata.ChatResponseMetadata; -import org.springframework.ai.chat.model.ChatModel; -import org.springframework.ai.chat.prompt.Prompt; import org.springframework.ai.embedding.EmbeddingModel; import org.springframework.ai.ollama.api.OllamaChatOptions; import org.springframework.beans.factory.annotation.Autowired; @@ -61,9 +59,6 @@ class LocalChatAndEmbeddingTest { @Autowired private ChatClient chatClient; - @Autowired - private ChatModel chatModel; - @Autowired private EmbeddingModel embeddingModel; @@ -72,12 +67,13 @@ class LocalChatAndEmbeddingTest { void reloadAfterAConfirmedUnload() throws Exception { // keepAlive("0") tells Ollama to unload the model from memory as soon as this call // finishes -- the same knob you'd reach for in production to free RAM/VRAM between - // bursts of traffic. It's set through ChatModel.call() with an explicit Prompt rather - // than through ChatClient.prompt().options(...): with this module's versions, an option - // set that way never reaches the request Ollama receives, which the going-deeper note - // below covers. - this.chatModel - .call(new Prompt("OK", OllamaChatOptions.builder().model("qwen2.5:0.5b").keepAlive("0").build())); + // bursts of traffic. Passed through ChatClient's per-call options() exactly like any + // other OllamaChatOptions field. + this.chatClient.prompt() + .user("OK") + .options(OllamaChatOptions.builder().model("qwen2.5:0.5b").keepAlive("0")) + .call() + .content(); // Don't take the unload on faith -- ask Ollama's own /api/ps, which lists every // currently loaded model, and confirm the registry is really empty before measuring