From 7fff4ddb9943147778bc3e617505feeb3b0f0626 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 17:18:32 +0000 Subject: [PATCH] Add structured-output module: entity() mapping to records/lists/maps, StructuredOutputValidationAdvisor retries ChatClient.CallResponseSpec.entity() mapping LLM JSON to a record (TicketTriage, with a real enum-constrained Priority field), a List, and a Map -- every case driven by a hand-written ScriptedChatModel with no live LLM anywhere. Key findings, all confirmed by disassembling spring-ai-client-chat-2.0.1.jar and spring-ai-model-2.0.1.jar rather than trusting docs: - StructuredOutputValidationAdvisor lives in org.springframework.ai.chat.client.advisor, in the same spring-ai-client-chat artifact as ToolCallingAdvisor -- unlike the tool-calling module's Tool Search Advisor pieces, it needs no separate Maven Central artifact or version pin. - entity(Class, spec -> spec.validateSchema()) is sugar: DefaultCallResponseSpec.resolveAdvisorChain builds a real StructuredOutputValidationAdvisor from the same JSON schema BeanOutputConverter uses to parse the response, and pushes it onto the advisor chain for that one call. - The schema/format instructions are baked into the user message once, up front, by entity() itself, before the advisor chain runs at all. A validation retry's only contribution is one appended line: "Output JSON validation failed because of: " -- each retry re-augments the ORIGINAL request, not the previous attempt's, so corrections never stack. - Default maxRepeatAttempts is 3 (4 total attempts); default advisorOrder is 2147481647, near Ordered.LOWEST_PRECEDENCE. - Exhausting every retry does NOT throw -- adviseCall's loop just returns the last (still invalid) response to the caller. Plain entity() with no validation, by contrast, throws immediately on the same bad JSON, since BeanOutputConverter.convert() is a separate Jackson deserialization step with no retry loop of its own. Both behaviors are captured from real runs (output/02, output/06). - Spring AI 2.0's JSON stack is Jackson 3 (tools.jackson.databind), not classic com.fasterxml.jackson -- visible directly in every one of this advisor's constructor and field signatures. Companion module for "Structured Output in Spring AI 2.0: Records, JSON Schema and Self-Correcting Responses" on ankurm.com. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01FtpJvZfg4nvLvtzgJTDWpB --- README.md | 1 + structured-output/.gitignore | 1 + structured-output/README.md | 51 ++++++++++++ .../output/01-entity-record-valid.txt | 29 +++++++ .../02-entity-record-invalid-no-retry.txt | 5 ++ .../output/03-entity-list-of-records.txt | 6 ++ structured-output/output/04-entity-map.txt | 3 + ...-validation-advisor-retry-then-success.txt | 74 ++++++++++++++++++ ...06-validation-advisor-exhausts-retries.txt | 7 ++ structured-output/pom.xml | 65 ++++++++++++++++ structured-output/scripts/run-all.sh | 7 ++ .../StructuredOutputApplication.java | 13 ++++ .../config/ChatClientConfig.java | 24 ++++++ .../structuredoutput/domain/ActionItem.java | 10 +++ .../structuredoutput/domain/Priority.java | 13 ++++ .../structuredoutput/domain/TicketTriage.java | 14 ++++ .../src/main/resources/application.yml | 7 ++ .../EntityBindingListTest.java | 52 +++++++++++++ .../EntityBindingMapTest.java | 44 +++++++++++ .../EntityBindingRecordTest.java | 72 +++++++++++++++++ .../ValidationAdvisorExhaustsRetriesTest.java | 62 +++++++++++++++ ...ionAdvisorRetriesOnceThenSucceedsTest.java | 69 ++++++++++++++++ .../support/ScriptedChatModel.java | 78 +++++++++++++++++++ .../structuredoutput/support/Transcript.java | 33 ++++++++ 24 files changed, 740 insertions(+) create mode 100644 structured-output/.gitignore create mode 100644 structured-output/README.md create mode 100644 structured-output/output/01-entity-record-valid.txt create mode 100644 structured-output/output/02-entity-record-invalid-no-retry.txt create mode 100644 structured-output/output/03-entity-list-of-records.txt create mode 100644 structured-output/output/04-entity-map.txt create mode 100644 structured-output/output/05-validation-advisor-retry-then-success.txt create mode 100644 structured-output/output/06-validation-advisor-exhausts-retries.txt create mode 100644 structured-output/pom.xml create mode 100755 structured-output/scripts/run-all.sh create mode 100644 structured-output/src/main/java/com/ankurm/structuredoutput/StructuredOutputApplication.java create mode 100644 structured-output/src/main/java/com/ankurm/structuredoutput/config/ChatClientConfig.java create mode 100644 structured-output/src/main/java/com/ankurm/structuredoutput/domain/ActionItem.java create mode 100644 structured-output/src/main/java/com/ankurm/structuredoutput/domain/Priority.java create mode 100644 structured-output/src/main/java/com/ankurm/structuredoutput/domain/TicketTriage.java create mode 100644 structured-output/src/main/resources/application.yml create mode 100644 structured-output/src/test/java/com/ankurm/structuredoutput/EntityBindingListTest.java create mode 100644 structured-output/src/test/java/com/ankurm/structuredoutput/EntityBindingMapTest.java create mode 100644 structured-output/src/test/java/com/ankurm/structuredoutput/EntityBindingRecordTest.java create mode 100644 structured-output/src/test/java/com/ankurm/structuredoutput/ValidationAdvisorExhaustsRetriesTest.java create mode 100644 structured-output/src/test/java/com/ankurm/structuredoutput/ValidationAdvisorRetriesOnceThenSucceedsTest.java create mode 100644 structured-output/src/test/java/com/ankurm/structuredoutput/support/ScriptedChatModel.java create mode 100644 structured-output/src/test/java/com/ankurm/structuredoutput/support/Transcript.java diff --git a/README.md b/README.md index c2d1dc4..9a4cc0c 100644 --- a/README.md +++ b/README.md @@ -10,5 +10,6 @@ Runnable companion code for the Spring AI articles on [ankurm.com](https://ankur | [`mcp-client/`](mcp-client) | `ChatClient` calling tools from two real external MCP servers (filesystem, git) over stdio via `defaultToolCallbacks(ToolCallbackProvider...)`, contrasted with a local `@Tool` method, with every call logged through one Micrometer `ObservationHandler`. Spring Boot 4.1.1, Spring AI 2.0.1, Java 25. | [Spring AI MCP Client: Calling External MCP Servers from ChatClient](https://ankurm.com/spring-ai-2-0-mcp-client/) | | [`mcp-secure/`](mcp-secure) | The mcp-server article's order-lookup tools behind a real OAuth2 resource server: JWT validation, one scope per tool via `@PreAuthorize`, unauthenticated tool discovery rejected outright, and every call audit-logged through MDC -- denials included. Spring Boot 4.1.1, Spring AI 2.0.1, Spring Security 7.1.1, Java 25. | [Securing an MCP Server with Spring Security 7](https://ankurm.com/spring-ai-2-0-mcp-server-security/) | | [`tool-calling/`](tool-calling) | `@Tool` methods, `ToolCallingAdvisor` (the advisor-layer replacement for Spring AI 1.x's per-model tool loop), `returnDirect`, `ToolContext`, and `ToolSearchToolCallingAdvisor` for progressive disclosure across a 230-tool synthetic library -- every test driven by a hand-written `ScriptedChatModel`, no live model anywhere. Spring Boot 4.1.1, Spring AI 2.0.1, Java 25. | [Tool Calling in Spring AI 2.0](https://ankurm.com/spring-ai-2-0-tool-calling/) | +| [`structured-output/`](structured-output) | `ChatClient.entity()` mapping LLM responses to Java records, lists and maps; `StructuredOutputValidationAdvisor` retrying non-conforming JSON with a real enum-constrained schema, including a captured run that exhausts every retry without throwing. Spring Boot 4.1.1, Spring AI 2.0.1, Java 25. | [Structured Output in Spring AI 2.0](https://ankurm.com/spring-ai-2-0-structured-output/) | Upgrading from Spring AI 1.x: [migration guide](https://ankurm.com/spring-ai-1-to-2-migration-guide/). diff --git a/structured-output/.gitignore b/structured-output/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/structured-output/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/structured-output/README.md b/structured-output/README.md new file mode 100644 index 0000000..1b02b2d --- /dev/null +++ b/structured-output/README.md @@ -0,0 +1,51 @@ +# structured-output + +Companion code for [Structured Output in Spring AI 2.0: Records, JSON Schema and Self-Correcting Responses](https://ankurm.com/spring-ai-2-0-structured-output/), part of the [Spring AI series](../README.md) on ankurm.com. + +Every test drives the real `ChatClient.entity()` machinery and the real `StructuredOutputValidationAdvisor` against a hand-written `ScriptedChatModel` (see [`support/ScriptedChatModel.java`](src/test/java/com/ankurm/structuredoutput/support/ScriptedChatModel.java)) that queues pre-programmed responses instead of calling a live provider. No test in this module calls a real LLM. + +## Versions + +| Component | Version | +|---|---| +| Spring Boot | 4.1.1 | +| Spring AI | 2.0.1 | +| Java | 25 (LTS) | + +`StructuredOutputValidationAdvisor` lives in `org.springframework.ai.chat.client.advisor`, inside the same `spring-ai-client-chat` artifact as `ToolCallingAdvisor` -- unlike the Tool Search Advisor pieces in this series' `tool-calling` module, it needs no separate Maven Central artifact or version pin. + +## Quickstart + +```bash +./scripts/run-all.sh +``` + +Runs the full test suite and regenerates every file under `output/`. + +## What's here + +| File | What it shows | +|---|---| +| [`domain/TicketTriage.java`](src/main/java/com/ankurm/structuredoutput/domain/TicketTriage.java) | The record used for the single-record examples; its `Priority` enum field is what gives the generated JSON schema a real `enum` constraint | +| [`domain/Priority.java`](src/main/java/com/ankurm/structuredoutput/domain/Priority.java) | A closed four-value enum | +| [`domain/ActionItem.java`](src/main/java/com/ankurm/structuredoutput/domain/ActionItem.java) | The record used for the `List` example | +| [`EntityBindingRecordTest.java`](src/test/java/com/ankurm/structuredoutput/EntityBindingRecordTest.java) | `entity(Class)` on valid JSON, and the exception it throws with zero retries on invalid JSON | +| [`EntityBindingListTest.java`](src/test/java/com/ankurm/structuredoutput/EntityBindingListTest.java) | `entity(ParameterizedTypeReference>)` | +| [`EntityBindingMapTest.java`](src/test/java/com/ankurm/structuredoutput/EntityBindingMapTest.java) | `entity(ParameterizedTypeReference>)` | +| [`ValidationAdvisorRetriesOnceThenSucceedsTest.java`](src/test/java/com/ankurm/structuredoutput/ValidationAdvisorRetriesOnceThenSucceedsTest.java) | `entity(Class, spec -> spec.validateSchema())`: one bad response, one corrective retry, one successful mapping | +| [`ValidationAdvisorExhaustsRetriesTest.java`](src/test/java/com/ankurm/structuredoutput/ValidationAdvisorExhaustsRetriesTest.java) | `StructuredOutputValidationAdvisor` wired directly with `maxRepeatAttempts(2)`; every attempt fails and the advisor returns the last invalid response without throwing | + +## Output files + +| File | Captured from | +|---|---| +| `output/01-entity-record-valid.txt` | `EntityBindingRecordTest.mapsValidJsonToARecord` | +| `output/02-entity-record-invalid-no-retry.txt` | `EntityBindingRecordTest.plainEntityThrowsImmediatelyOnAnInvalidEnumValue_noRetry` | +| `output/03-entity-list-of-records.txt` | `EntityBindingListTest` | +| `output/04-entity-map.txt` | `EntityBindingMapTest` | +| `output/05-validation-advisor-retry-then-success.txt` | `ValidationAdvisorRetriesOnceThenSucceedsTest` | +| `output/06-validation-advisor-exhausts-retries.txt` | `ValidationAdvisorExhaustsRetriesTest` | + +## Requirements + +JDK 25, Maven. No API key needed -- `application.yml` supplies a placeholder that the autoconfigured `OpenAiChatModel` bean would use, but no test ever constructs that bean. diff --git a/structured-output/output/01-entity-record-valid.txt b/structured-output/output/01-entity-record-valid.txt new file mode 100644 index 0000000..ea879a0 --- /dev/null +++ b/structured-output/output/01-entity-record-valid.txt @@ -0,0 +1,29 @@ +model call count: 1 + +mapped record: TicketTriage[category=billing, priority=HIGH, requiresEscalation=true, suggestedActions=[Refund the duplicate charge, Reply within 4 hours]] + +JSON schema BeanOutputConverter generated for TicketTriage: +{ + "$schema" : "https://json-schema.org/draft/2020-12/schema", + "type" : "object", + "properties" : { + "category" : { + "type" : "string" + }, + "priority" : { + "type" : "string", + "enum" : [ "LOW", "MEDIUM", "HIGH", "CRITICAL" ] + }, + "requiresEscalation" : { + "type" : "boolean" + }, + "suggestedActions" : { + "type" : "array", + "items" : { + "type" : "string" + } + } + }, + "required" : [ "category", "priority", "requiresEscalation", "suggestedActions" ], + "additionalProperties" : false +} \ No newline at end of file diff --git a/structured-output/output/02-entity-record-invalid-no-retry.txt b/structured-output/output/02-entity-record-invalid-no-retry.txt new file mode 100644 index 0000000..431c1a5 --- /dev/null +++ b/structured-output/output/02-entity-record-invalid-no-retry.txt @@ -0,0 +1,5 @@ +model call count: 1 + +exception thrown to the caller (plain entity() never retries): +tools.jackson.databind.exc.InvalidFormatException: Cannot deserialize value of type `com.ankurm.structuredoutput.domain.Priority` from String "URGENT": not one of the values accepted for Enum class: [HIGH, LOW, MEDIUM, CRITICAL] + at [Source: REDACTED (`StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION` disabled); byte offset: #UNKNOWN] (through reference chain: com.ankurm.structuredoutput.domain.TicketTriage["priority"]) \ No newline at end of file diff --git a/structured-output/output/03-entity-list-of-records.txt b/structured-output/output/03-entity-list-of-records.txt new file mode 100644 index 0000000..49d0cbc --- /dev/null +++ b/structured-output/output/03-entity-list-of-records.txt @@ -0,0 +1,6 @@ +model call count: 1 + +mapped list (3 items): + ActionItem[owner=Priya, task=Send the revised contract, dueDate=2026-09-25] + ActionItem[owner=Marcus, task=Confirm the vendor's SLA numbers, dueDate=2026-09-26] + ActionItem[owner=Priya, task=Book the kickoff call, dueDate=2026-09-24] diff --git a/structured-output/output/04-entity-map.txt b/structured-output/output/04-entity-map.txt new file mode 100644 index 0000000..5c83611 --- /dev/null +++ b/structured-output/output/04-entity-map.txt @@ -0,0 +1,3 @@ +model call count: 1 + +mapped map: {darkModeEnabled=true, maxUploadSizeMb=25, betaFeatures=[new-dashboard, ai-search]} \ No newline at end of file diff --git a/structured-output/output/05-validation-advisor-retry-then-success.txt b/structured-output/output/05-validation-advisor-retry-then-success.txt new file mode 100644 index 0000000..b3a4143 --- /dev/null +++ b/structured-output/output/05-validation-advisor-retry-then-success.txt @@ -0,0 +1,74 @@ +model call count: 2 + +attempt 1 -- model sent priority "URGENT", not one of Priority's four enum values + +first prompt's user message (entity() already bakes the schema in, before any retry): +Customer was charged twice for the same order and wants a refund today. +Your response should be in JSON format. +Do not include any explanations, only provide a RFC8259 compliant JSON response following this format without deviation. +Do not include markdown code blocks in your response. +Remove the ```json markdown from the output. +Here is the JSON Schema instance your output must adhere to: +```{ + "$schema" : "https://json-schema.org/draft/2020-12/schema", + "type" : "object", + "properties" : { + "category" : { + "type" : "string" + }, + "priority" : { + "type" : "string", + "enum" : [ "LOW", "MEDIUM", "HIGH", "CRITICAL" ] + }, + "requiresEscalation" : { + "type" : "boolean" + }, + "suggestedActions" : { + "type" : "array", + "items" : { + "type" : "string" + } + } + }, + "required" : [ "category", "priority", "requiresEscalation", "suggestedActions" ], + "additionalProperties" : false +}``` + + +second prompt's user message (the advisor's own contribution is the one appended line): +Customer was charged twice for the same order and wants a refund today. +Output JSON validation failed because of: does not have a value in the enumeration ["LOW", "MEDIUM", "HIGH", "CRITICAL"] +Your response should be in JSON format. +Do not include any explanations, only provide a RFC8259 compliant JSON response following this format without deviation. +Do not include markdown code blocks in your response. +Remove the ```json markdown from the output. +Here is the JSON Schema instance your output must adhere to: +```{ + "$schema" : "https://json-schema.org/draft/2020-12/schema", + "type" : "object", + "properties" : { + "category" : { + "type" : "string" + }, + "priority" : { + "type" : "string", + "enum" : [ "LOW", "MEDIUM", "HIGH", "CRITICAL" ] + }, + "requiresEscalation" : { + "type" : "boolean" + }, + "suggestedActions" : { + "type" : "array", + "items" : { + "type" : "string" + } + } + }, + "required" : [ "category", "priority", "requiresEscalation", "suggestedActions" ], + "additionalProperties" : false +}``` + + +attempt 2 -- model sent priority "HIGH", validation passed + +final mapped record: TicketTriage[category=billing, priority=HIGH, requiresEscalation=true, suggestedActions=[Refund the duplicate charge]] \ No newline at end of file diff --git a/structured-output/output/06-validation-advisor-exhausts-retries.txt b/structured-output/output/06-validation-advisor-exhausts-retries.txt new file mode 100644 index 0000000..76733d5 --- /dev/null +++ b/structured-output/output/06-validation-advisor-exhausts-retries.txt @@ -0,0 +1,7 @@ +model call count: 3 (1 initial attempt + maxRepeatAttempts(2) retries) + +every attempt returned the same invalid "URGENT" priority value + +raw content returned to the caller (still invalid -- the advisor does not throw): +{"category":"billing","priority":"URGENT","requiresEscalation":true, + "suggestedActions":["Refund the duplicate charge"]} diff --git a/structured-output/pom.xml b/structured-output/pom.xml new file mode 100644 index 0000000..f3aa82b --- /dev/null +++ b/structured-output/pom.xml @@ -0,0 +1,65 @@ + + + 4.0.0 + + + org.springframework.boot + spring-boot-starter-parent + 4.1.1 + + + + com.ankurm + structured-output + 1.0.0 + structured-output + Structured output in Spring AI 2.0: mapping LLM responses to Java records, lists and maps with ChatClient.entity(), and StructuredOutputValidationAdvisor retrying non-conforming JSON. + + + 25 + 2.0.1 + + + + + + org.springframework.ai + spring-ai-bom + ${spring-ai.version} + pom + import + + + + + + + org.springframework.ai + spring-ai-starter-model-openai + + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + + -Duser.timezone=UTC -Dstdout.encoding=UTF-8 -Dfile.encoding=UTF-8 + + + + + diff --git a/structured-output/scripts/run-all.sh b/structured-output/scripts/run-all.sh new file mode 100755 index 0000000..8876081 --- /dev/null +++ b/structured-output/scripts/run-all.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Regenerates every file under output/ -- the test suite writes each one itself via the +# Transcript helper, overwriting it in place. No file in this module's output/ is hand-captured. +set -euo pipefail +cd "$(dirname "$0")/.." +rm -rf target +mvn -q -o test diff --git a/structured-output/src/main/java/com/ankurm/structuredoutput/StructuredOutputApplication.java b/structured-output/src/main/java/com/ankurm/structuredoutput/StructuredOutputApplication.java new file mode 100644 index 0000000..801882e --- /dev/null +++ b/structured-output/src/main/java/com/ankurm/structuredoutput/StructuredOutputApplication.java @@ -0,0 +1,13 @@ +package com.ankurm.structuredoutput; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class StructuredOutputApplication { + + public static void main(String[] args) { + SpringApplication.run(StructuredOutputApplication.class, args); + } + +} diff --git a/structured-output/src/main/java/com/ankurm/structuredoutput/config/ChatClientConfig.java b/structured-output/src/main/java/com/ankurm/structuredoutput/config/ChatClientConfig.java new file mode 100644 index 0000000..8d08121 --- /dev/null +++ b/structured-output/src/main/java/com/ankurm/structuredoutput/config/ChatClientConfig.java @@ -0,0 +1,24 @@ +package com.ankurm.structuredoutput.config; + +import org.springframework.ai.chat.client.ChatClient; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Production wiring: one plain {@code ChatClient} bean with no structured-output advisor + * registered by default. Structured output in this module is opted into per call, through + * {@code ChatClient.CallResponseSpec.entity(...)} -- see + * {@code EntityBindingRecordTest}, {@code EntityBindingListTest}, and + * {@code EntityBindingMapTest} for the plain path, and + * {@code ValidationAdvisorRetriesOnceThenSucceedsTest} for the self-correcting one. + */ +@Configuration +public class ChatClientConfig { + + @Bean + ChatClient chatClient(ChatModel chatModel) { + return ChatClient.builder(chatModel).build(); + } + +} diff --git a/structured-output/src/main/java/com/ankurm/structuredoutput/domain/ActionItem.java b/structured-output/src/main/java/com/ankurm/structuredoutput/domain/ActionItem.java new file mode 100644 index 0000000..6dca82b --- /dev/null +++ b/structured-output/src/main/java/com/ankurm/structuredoutput/domain/ActionItem.java @@ -0,0 +1,10 @@ +package com.ankurm.structuredoutput.domain; + +/** + * One row of a meeting's action items. The target type for the {@code List} + * examples in this module -- mapping a whole list of records out of one model response, not just + * a single one. + */ +public record ActionItem(String owner, String task, String dueDate) { + +} diff --git a/structured-output/src/main/java/com/ankurm/structuredoutput/domain/Priority.java b/structured-output/src/main/java/com/ankurm/structuredoutput/domain/Priority.java new file mode 100644 index 0000000..29df7c8 --- /dev/null +++ b/structured-output/src/main/java/com/ankurm/structuredoutput/domain/Priority.java @@ -0,0 +1,13 @@ +package com.ankurm.structuredoutput.domain; + +/** + * A closed set of triage priorities. Because this is a Java enum, the JSON schema + * {@code BeanOutputConverter} generates for {@link TicketTriage} constrains this field to + * exactly these four values with a real {@code "enum": [...]} entry -- not a comment telling the + * model what's allowed, a schema constraint the validator actually checks a response against. + */ +public enum Priority { + + LOW, MEDIUM, HIGH, CRITICAL + +} diff --git a/structured-output/src/main/java/com/ankurm/structuredoutput/domain/TicketTriage.java b/structured-output/src/main/java/com/ankurm/structuredoutput/domain/TicketTriage.java new file mode 100644 index 0000000..4d00cf3 --- /dev/null +++ b/structured-output/src/main/java/com/ankurm/structuredoutput/domain/TicketTriage.java @@ -0,0 +1,14 @@ +package com.ankurm.structuredoutput.domain; + +import java.util.List; + +/** + * What a support ticket's free-text description gets mapped to. The target type for the + * "single record" examples in this module -- and, because {@link Priority} is a closed enum, the + * type whose generated JSON schema is strict enough to genuinely reject a bad response instead of + * accepting whatever string the model feels like sending. + */ +public record TicketTriage(String category, Priority priority, boolean requiresEscalation, + List suggestedActions) { + +} diff --git a/structured-output/src/main/resources/application.yml b/structured-output/src/main/resources/application.yml new file mode 100644 index 0000000..4635782 --- /dev/null +++ b/structured-output/src/main/resources/application.yml @@ -0,0 +1,7 @@ +spring: + ai: + openai: + # Every test in this module drives a hand-written ScriptedChatModel (see + # src/test/java/com/ankurm/structuredoutput/support/ScriptedChatModel.java) -- the + # autoconfigured OpenAiChatModel bean this key would back is never constructed in a test. + api-key: ${OPENAI_API_KEY:demo-key-not-used-by-tests} diff --git a/structured-output/src/test/java/com/ankurm/structuredoutput/EntityBindingListTest.java b/structured-output/src/test/java/com/ankurm/structuredoutput/EntityBindingListTest.java new file mode 100644 index 0000000..f280ed9 --- /dev/null +++ b/structured-output/src/test/java/com/ankurm/structuredoutput/EntityBindingListTest.java @@ -0,0 +1,52 @@ +package com.ankurm.structuredoutput; + +import java.util.List; + +import com.ankurm.structuredoutput.domain.ActionItem; +import com.ankurm.structuredoutput.support.ScriptedChatModel; +import com.ankurm.structuredoutput.support.Transcript; +import org.junit.jupiter.api.Test; + +import org.springframework.ai.chat.client.ChatClient; +import org.springframework.core.ParameterizedTypeReference; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * {@code entity()} isn't limited to a single record -- a {@link ParameterizedTypeReference} + * maps a whole JSON array to a {@code List} in one call, no manual iteration over a + * parsed {@code JsonNode}. + */ +class EntityBindingListTest { + + @Test + void mapsAJsonArrayToAListOfRecords() { + ScriptedChatModel model = ScriptedChatModel.builder() + .thenRespond(""" + [ + {"owner":"Priya","task":"Send the revised contract","dueDate":"2026-09-25"}, + {"owner":"Marcus","task":"Confirm the vendor's SLA numbers","dueDate":"2026-09-26"}, + {"owner":"Priya","task":"Book the kickoff call","dueDate":"2026-09-24"} + ] + """) + .build(); + ChatClient client = ChatClient.builder(model).build(); + + List actionItems = client.prompt() + .user("Extract the action items from this meeting note: Priya will send the revised " + + "contract by Friday. Marcus needs to confirm the vendor's SLA numbers. Priya " + + "should also book the kickoff call for tomorrow.") + .call() + .entity(new ParameterizedTypeReference>() { + }); + + assertThat(actionItems).hasSize(3); + assertThat(actionItems).extracting(ActionItem::owner).containsExactly("Priya", "Marcus", "Priya"); + assertThat(model.callCount()).isEqualTo(1); + + Transcript.write("03-entity-list-of-records", + "model call count: " + model.callCount() + "\n\n" + "mapped list (" + actionItems.size() + + " items):\n" + actionItems.stream().map(Object::toString).reduce("", (a, b) -> a + " " + b + "\n")); + } + +} diff --git a/structured-output/src/test/java/com/ankurm/structuredoutput/EntityBindingMapTest.java b/structured-output/src/test/java/com/ankurm/structuredoutput/EntityBindingMapTest.java new file mode 100644 index 0000000..cc80dad --- /dev/null +++ b/structured-output/src/test/java/com/ankurm/structuredoutput/EntityBindingMapTest.java @@ -0,0 +1,44 @@ +package com.ankurm.structuredoutput; + +import java.util.Map; + +import com.ankurm.structuredoutput.support.ScriptedChatModel; +import com.ankurm.structuredoutput.support.Transcript; +import org.junit.jupiter.api.Test; + +import org.springframework.ai.chat.client.ChatClient; +import org.springframework.core.ParameterizedTypeReference; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Sometimes there's no record worth declaring -- a one-off extraction, a shape that varies call + * to call. {@code entity()} maps straight to a {@code Map} for that case, the + * same converter machinery as the record and list cases, just with a looser target type. + */ +class EntityBindingMapTest { + + @Test + void mapsAJsonObjectToAMapWhenNoRecordIsWorthDeclaring() { + ScriptedChatModel model = ScriptedChatModel.builder() + .thenRespond(""" + {"darkModeEnabled":true,"maxUploadSizeMb":25,"betaFeatures":["new-dashboard","ai-search"]} + """) + .build(); + ChatClient client = ChatClient.builder(model).build(); + + Map flags = client.prompt() + .user("Extract the feature flags mentioned here: dark mode is on, max upload size is 25MB, " + + "and the beta features enabled are new-dashboard and ai-search.") + .call() + .entity(new ParameterizedTypeReference>() { + }); + + assertThat(flags).containsEntry("darkModeEnabled", true).containsEntry("maxUploadSizeMb", 25); + assertThat(model.callCount()).isEqualTo(1); + + Transcript.write("04-entity-map", + "model call count: " + model.callCount() + "\n\n" + "mapped map: " + flags); + } + +} diff --git a/structured-output/src/test/java/com/ankurm/structuredoutput/EntityBindingRecordTest.java b/structured-output/src/test/java/com/ankurm/structuredoutput/EntityBindingRecordTest.java new file mode 100644 index 0000000..ceba5ba --- /dev/null +++ b/structured-output/src/test/java/com/ankurm/structuredoutput/EntityBindingRecordTest.java @@ -0,0 +1,72 @@ +package com.ankurm.structuredoutput; + +import com.ankurm.structuredoutput.domain.Priority; +import com.ankurm.structuredoutput.domain.TicketTriage; +import com.ankurm.structuredoutput.support.ScriptedChatModel; +import com.ankurm.structuredoutput.support.Transcript; +import org.junit.jupiter.api.Test; + +import org.springframework.ai.chat.client.ChatClient; +import org.springframework.ai.converter.BeanOutputConverter; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The plain path: {@code ChatClient.CallResponseSpec.entity(Class)}, no validation, no retries. + * When the model's JSON matches the record's shape, this is all mapping a response to a record + * takes. When it doesn't -- one bad enum value here -- {@code entity()} throws immediately, + * which is the motivating problem the rest of this module's tests solve. + */ +class EntityBindingRecordTest { + + @Test + void mapsValidJsonToARecord() { + ScriptedChatModel model = ScriptedChatModel.builder() + .thenRespond(""" + {"category":"billing","priority":"HIGH","requiresEscalation":true, + "suggestedActions":["Refund the duplicate charge","Reply within 4 hours"]} + """) + .build(); + ChatClient client = ChatClient.builder(model).build(); + + TicketTriage triage = client.prompt() + .user("Customer was charged twice for the same order and wants a refund today.") + .call() + .entity(TicketTriage.class); + + assertThat(triage.category()).isEqualTo("billing"); + assertThat(triage.priority()).isEqualTo(Priority.HIGH); + assertThat(triage.requiresEscalation()).isTrue(); + assertThat(triage.suggestedActions()).containsExactly("Refund the duplicate charge", "Reply within 4 hours"); + assertThat(model.callCount()).isEqualTo(1); + + BeanOutputConverter converter = new BeanOutputConverter<>(TicketTriage.class); + Transcript.write("01-entity-record-valid", + "model call count: " + model.callCount() + "\n\n" + "mapped record: " + triage + "\n\n" + + "JSON schema BeanOutputConverter generated for TicketTriage:\n" + converter.getJsonSchema()); + } + + @Test + void plainEntityThrowsImmediatelyOnAnInvalidEnumValue_noRetry() { + // "URGENT" is not one of Priority's four constants -- Jackson rejects it during + // deserialization, and plain entity() has no retry loop to catch that. + ScriptedChatModel model = ScriptedChatModel.builder() + .thenRespond(""" + {"category":"billing","priority":"URGENT","requiresEscalation":true, + "suggestedActions":["Refund the duplicate charge"]} + """) + .build(); + ChatClient client = ChatClient.builder(model).build(); + + Throwable thrown = org.assertj.core.api.Assertions + .catchThrowable(() -> client.prompt().user("Customer was charged twice.").call().entity(TicketTriage.class)); + + assertThat(thrown).isNotNull(); + assertThat(model.callCount()).isEqualTo(1); + + Transcript.write("02-entity-record-invalid-no-retry", "model call count: " + model.callCount() + "\n\n" + + "exception thrown to the caller (plain entity() never retries):\n" + thrown.getClass().getName() + + ": " + thrown.getMessage()); + } + +} diff --git a/structured-output/src/test/java/com/ankurm/structuredoutput/ValidationAdvisorExhaustsRetriesTest.java b/structured-output/src/test/java/com/ankurm/structuredoutput/ValidationAdvisorExhaustsRetriesTest.java new file mode 100644 index 0000000..1b5cce7 --- /dev/null +++ b/structured-output/src/test/java/com/ankurm/structuredoutput/ValidationAdvisorExhaustsRetriesTest.java @@ -0,0 +1,62 @@ +package com.ankurm.structuredoutput; + +import com.ankurm.structuredoutput.domain.TicketTriage; +import com.ankurm.structuredoutput.support.ScriptedChatModel; +import com.ankurm.structuredoutput.support.Transcript; +import org.junit.jupiter.api.Test; + +import org.springframework.ai.chat.client.ChatClient; +import org.springframework.ai.chat.client.advisor.StructuredOutputValidationAdvisor; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * What the defaults do not do: when every attempt keeps failing validation, disassembling + * {@code StructuredOutputValidationAdvisor.adviseCall} shows the loop just runs out of attempts + * and returns the caller the last (still-invalid) response -- it never throws. This module wires + * the advisor directly with {@code .advisors(...)} instead of the {@code entity()} convenience, + * which is also how you'd reuse one validating advisor across many calls instead of opting in + * per call, and sets {@code maxRepeatAttempts(2)} explicitly so the test doesn't depend on the + * advisor's own default of 3. + */ +class ValidationAdvisorExhaustsRetriesTest { + + @Test + void exhaustsAllRetriesAndReturnsTheLastInvalidResponseWithoutThrowing() { + String stillInvalid = """ + {"category":"billing","priority":"URGENT","requiresEscalation":true, + "suggestedActions":["Refund the duplicate charge"]} + """; + // maxRepeatAttempts(2) means 1 initial attempt + 2 retries = 3 total calls -- three + // scripted responses, all still invalid, so every retry is genuinely exhausted. + ScriptedChatModel model = ScriptedChatModel.builder() + .thenRespond(stillInvalid) + .thenRespond(stillInvalid) + .thenRespond(stillInvalid) + .build(); + StructuredOutputValidationAdvisor advisor = StructuredOutputValidationAdvisor.builder() + .outputType(TicketTriage.class) + .maxRepeatAttempts(2) + .build(); + ChatClient client = ChatClient.builder(model).build(); + + // content(), not entity() -- this isolates the advisor's own behavior from + // BeanOutputConverter.convert(), which is a separate step that would throw on this same + // bad JSON if it were asked to parse it into a TicketTriage. + String rawContent = client.prompt() + .user("Customer was charged twice for the same order and wants a refund today.") + .advisors(advisor) + .call() + .content(); + + assertThat(model.callCount()).isEqualTo(3); + assertThat(rawContent).contains("\"priority\":\"URGENT\""); + + Transcript.write("06-validation-advisor-exhausts-retries", + "model call count: " + model.callCount() + " (1 initial attempt + maxRepeatAttempts(2) retries)\n\n" + + "every attempt returned the same invalid \"URGENT\" priority value\n\n" + + "raw content returned to the caller (still invalid -- the advisor does not throw):\n" + + rawContent); + } + +} diff --git a/structured-output/src/test/java/com/ankurm/structuredoutput/ValidationAdvisorRetriesOnceThenSucceedsTest.java b/structured-output/src/test/java/com/ankurm/structuredoutput/ValidationAdvisorRetriesOnceThenSucceedsTest.java new file mode 100644 index 0000000..2294b3d --- /dev/null +++ b/structured-output/src/test/java/com/ankurm/structuredoutput/ValidationAdvisorRetriesOnceThenSucceedsTest.java @@ -0,0 +1,69 @@ +package com.ankurm.structuredoutput; + +import com.ankurm.structuredoutput.domain.Priority; +import com.ankurm.structuredoutput.domain.TicketTriage; +import com.ankurm.structuredoutput.support.ScriptedChatModel; +import com.ankurm.structuredoutput.support.Transcript; +import org.junit.jupiter.api.Test; + +import org.springframework.ai.chat.client.ChatClient; +import org.springframework.ai.chat.prompt.Prompt; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The self-correcting path: {@code entity(Class, spec -> spec.validateSchema())}. Disassembling + * {@code DefaultChatClient$DefaultCallResponseSpec.resolveAdvisorChain} shows exactly what that + * one flag does -- it builds a real {@code StructuredOutputValidationAdvisor} from the same JSON + * schema {@code BeanOutputConverter} would use to parse the response, and pushes it onto the + * advisor chain for this call only. The model's first response here has the same bad "URGENT" + * enum value {@code EntityBindingRecordTest} showed blowing up plain {@code entity()}; this time + * the advisor catches it, tells the model what was wrong, and the second response succeeds. + */ +class ValidationAdvisorRetriesOnceThenSucceedsTest { + + @Test + void retriesOnceAfterASchemaValidationFailureThenSucceeds() { + ScriptedChatModel model = ScriptedChatModel.builder() + .thenRespond(""" + {"category":"billing","priority":"URGENT","requiresEscalation":true, + "suggestedActions":["Refund the duplicate charge"]} + """) + .thenRespond(""" + {"category":"billing","priority":"HIGH","requiresEscalation":true, + "suggestedActions":["Refund the duplicate charge"]} + """) + .build(); + ChatClient client = ChatClient.builder(model).build(); + + TicketTriage triage = client.prompt() + .user("Customer was charged twice for the same order and wants a refund today.") + .call() + .entity(TicketTriage.class, spec -> spec.validateSchema()); + + assertThat(triage.priority()).isEqualTo(Priority.HIGH); + assertThat(model.callCount()).isEqualTo(2); + + Prompt firstPrompt = model.capturedPrompts().get(0); + Prompt secondPrompt = model.capturedPrompts().get(1); + String firstUserText = firstPrompt.getUserMessage().getText(); + String secondUserText = secondPrompt.getUserMessage().getText(); + + // The schema/format instructions are already in the FIRST prompt -- entity() bakes them + // into the user message before the advisor chain ever runs. The advisor's own + // contribution on a retry is exactly one appended line. + assertThat(firstUserText).contains("Here is the JSON Schema instance your output must adhere to:"); + assertThat(secondUserText).contains("Output JSON validation failed because of:"); + assertThat(secondUserText).contains("Customer was charged twice for the same order and wants a refund today."); + + Transcript.write("05-validation-advisor-retry-then-success", + "model call count: " + model.callCount() + "\n\n" + + "attempt 1 -- model sent priority \"URGENT\", not one of Priority's four enum values\n\n" + + "first prompt's user message (entity() already bakes the schema in, before any retry):\n" + + firstUserText + "\n\n" + + "second prompt's user message (the advisor's own contribution is the one appended line):\n" + + secondUserText + "\n\n" + "attempt 2 -- model sent priority \"HIGH\", validation passed\n\n" + + "final mapped record: " + triage); + } + +} diff --git a/structured-output/src/test/java/com/ankurm/structuredoutput/support/ScriptedChatModel.java b/structured-output/src/test/java/com/ankurm/structuredoutput/support/ScriptedChatModel.java new file mode 100644 index 0000000..9420d54 --- /dev/null +++ b/structured-output/src/test/java/com/ankurm/structuredoutput/support/ScriptedChatModel.java @@ -0,0 +1,78 @@ +package com.ankurm.structuredoutput.support; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.CopyOnWriteArrayList; + +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.ChatOptions; +import org.springframework.ai.chat.prompt.Prompt; + +/** + * A hand-written {@link ChatModel} that returns a queue of pre-scripted responses instead of + * calling a real provider. {@code ChatModel} has exactly one abstract method, + * {@code call(Prompt)} -- confirmed with {@code javap} on spring-ai-model-2.0.1.jar, the same + * finding this series' tool-calling article made -- so this class is a fully legitimate + * {@code ChatModel} from {@code StructuredOutputValidationAdvisor}'s point of view, not a mock + * standing in for one. Every retry this module's tests observe is the real advisor calling this + * model again, not a simulated retry. + */ +public class ScriptedChatModel implements ChatModel { + + private final Queue script; + + private final List capturedPrompts = new CopyOnWriteArrayList<>(); + + private ScriptedChatModel(Deque script) { + this.script = script; + } + + @Override + public ChatResponse call(Prompt prompt) { + this.capturedPrompts.add(prompt); + ChatResponse next = this.script.poll(); + if (next == null) { + throw new IllegalStateException("ScriptedChatModel ran out of queued responses after " + + this.capturedPrompts.size() + " calls. Prompts so far: " + this.capturedPrompts); + } + return next; + } + + @Override + public ChatOptions getOptions() { + return ChatOptions.builder().build(); + } + + public List capturedPrompts() { + return List.copyOf(this.capturedPrompts); + } + + public int callCount() { + return this.capturedPrompts.size(); + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + + private final Deque script = new ArrayDeque<>(); + + public Builder thenRespond(String text) { + this.script.add(new ChatResponse(List.of(new Generation(new AssistantMessage(text))))); + return this; + } + + public ScriptedChatModel build() { + return new ScriptedChatModel(new ArrayDeque<>(this.script)); + } + + } + +} diff --git a/structured-output/src/test/java/com/ankurm/structuredoutput/support/Transcript.java b/structured-output/src/test/java/com/ankurm/structuredoutput/support/Transcript.java new file mode 100644 index 0000000..12c7d6e --- /dev/null +++ b/structured-output/src/test/java/com/ankurm/structuredoutput/support/Transcript.java @@ -0,0 +1,33 @@ +package com.ankurm.structuredoutput.support; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Writes a test's captured, real output to {@code output/.txt} and echoes it to stdout. + * Every console block quoted in the companion article comes from a file this class wrote -- + * nothing in the article is retyped or tidied up by hand. + */ +public final class Transcript { + + private Transcript() { + } + + public static void write(String name, String content) { + try { + Path dir = Path.of("output"); + Files.createDirectories(dir); + Path file = dir.resolve(name + ".txt"); + Files.writeString(file, content, StandardCharsets.UTF_8); + System.out.println("--- " + name + " ---"); + System.out.println(content); + } + catch (IOException ex) { + throw new UncheckedIOException(ex); + } + } + +}