Add evaluation module: RelevancyEvaluator and FactCheckingEvaluator, golden dataset with a pass-rate gate, deterministic CI judge and simulated judge noise

Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01JXVi2GMQ7bR5EmbUFdDj7N

evaluation/src/main/java/com/ankurm/evaluation/CaseResult.java

@@ -0,0 +1,9 @@
+package com.ankurm.evaluation;
+
+/** The outcome of running one golden case through the evaluators. */
+public record CaseResult(String id, String answer, boolean relevant, boolean grounded, boolean hasFacts) {
+
+    public boolean passed() {
+        return relevant && grounded && hasFacts;
+    }
+}

evaluation/src/main/java/com/ankurm/evaluation/CompositeEvaluator.java

@@ -0,0 +1,37 @@
+package com.ankurm.evaluation;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.springframework.ai.evaluation.EvaluationRequest;
+import org.springframework.ai.evaluation.EvaluationResponse;
+import org.springframework.ai.evaluation.Evaluator;
+
+/** Runs several named evaluators on one request. It passes only if all of them pass; the score is their mean; per-evaluator verdicts go in metadata. */
+public class CompositeEvaluator implements Evaluator {
+
+    private final Map<String, Evaluator> parts;
+
+    public CompositeEvaluator(Map<String, Evaluator> parts) {
+        this.parts = parts;
+    }
+
+    @Override
+    public EvaluationResponse evaluate(EvaluationRequest request) {
+        Map<String, Object> verdicts = new LinkedHashMap<>();
+        boolean pass = true;
+        float total = 0;
+        StringBuilder feedback = new StringBuilder();
+        for (var e : parts.entrySet()) {
+            EvaluationResponse r = e.getValue().evaluate(request);
+            verdicts.put(e.getKey(), r.isPass());
+            pass &= r.isPass();
+            total += r.getScore();
+            if (!r.isPass()) {
+                feedback.append(e.getKey()).append(" failed").append(r.getFeedback().isEmpty() ? "; " : " (" + r.getFeedback() + "); ");
+            }
+        }
+        return new EvaluationResponse(pass, total / parts.size(), feedback.toString().strip(), verdicts);
+    }
+}

evaluation/src/main/java/com/ankurm/evaluation/ContainsFactsEvaluator.java

@@ -0,0 +1,30 @@
+package com.ankurm.evaluation;
+
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import org.springframework.ai.evaluation.EvaluationRequest;
+import org.springframework.ai.evaluation.EvaluationResponse;
+import org.springframework.ai.evaluation.Evaluator;
+
+/**
+ * A plain-code {@link Evaluator}: the answer must contain every expected fact as a substring.
+ * No model call, no cost, no noise, and it fills in the feedback the built-in evaluators leave empty.
+ */
+public class ContainsFactsEvaluator implements Evaluator {
+
+    private final List<String> facts;
+
+    public ContainsFactsEvaluator(List<String> facts) {
+        this.facts = facts;
+    }
+
+    @Override
+    public EvaluationResponse evaluate(EvaluationRequest request) {
+        String answer = request.getResponseContent().toLowerCase(Locale.ROOT);
+        List<String> missing = facts.stream().filter(f -> !answer.contains(f.toLowerCase(Locale.ROOT))).toList();
+        float score = facts.isEmpty() ? 1f : (facts.size() - missing.size()) / (float) facts.size();
+        return new EvaluationResponse(missing.isEmpty(), score, missing.isEmpty() ? "all facts present" : "missing: " + missing, Map.of());
+    }
+}

evaluation/src/main/java/com/ankurm/evaluation/EvalReport.java

@@ -0,0 +1,22 @@
+package com.ankurm.evaluation;
+
+import java.util.List;
+
+/** All results of one run over the golden dataset, with the pass rate a build can gate on. */
+public record EvalReport(List<CaseResult> results) {
+
+    public double passRate() {
+        return results.isEmpty() ? 0 : results.stream().filter(CaseResult::passed).count() / (double) results.size();
+    }
+
+    public List<String> failedIds() {
+        return results.stream().filter(r -> !r.passed()).map(CaseResult::id).toList();
+    }
+
+    /** Fails the build (with an {@link AssertionError} naming the failing cases) when the pass rate is below the threshold. */
+    public void requirePassRate(double threshold) {
+        if (passRate() < threshold) {
+            throw new AssertionError("pass rate %.2f is below the %.2f threshold; failing cases: %s".formatted(passRate(), threshold, failedIds()));
+        }
+    }
+}

evaluation/src/main/java/com/ankurm/evaluation/EvalRunner.java

@@ -0,0 +1,38 @@
+package com.ankurm.evaluation;
+
+import java.util.List;
+
+import org.springframework.ai.chat.client.ChatClient;
+import org.springframework.ai.chat.evaluation.FactCheckingEvaluator;
+import org.springframework.ai.chat.evaluation.RelevancyEvaluator;
+import org.springframework.ai.chat.model.ChatModel;
+import org.springframework.ai.evaluation.EvaluationRequest;
+
+/** Runs every golden case through the assistant, then through the relevancy, fact-checking and contains-facts evaluators. */
+public class EvalRunner {
+
+    private final SupportAssistant assistant;
+
+    private final RelevancyEvaluator relevancy;
+
+    private final FactCheckingEvaluator factChecking;
+
+    public EvalRunner(SupportAssistant assistant, ChatModel judge) {
+        this.assistant = assistant;
+        this.relevancy = new RelevancyEvaluator(ChatClient.builder(judge));
+        this.factChecking = FactCheckingEvaluator.builder(ChatClient.builder(judge)).build();
+    }
+
+    public EvalReport run(List<GoldenCase> cases) {
+        return new EvalReport(cases.stream().map(this::runOne).toList());
+    }
+
+    private CaseResult runOne(GoldenCase c) {
+        String answer = assistant.answer(c.question(), c.documents());
+        EvaluationRequest request = new EvaluationRequest(c.question(), c.documents(), answer);
+        boolean relevant = relevancy.evaluate(request).isPass();
+        boolean grounded = factChecking.evaluate(request).isPass();
+        boolean facts = new ContainsFactsEvaluator(c.expectedFacts()).evaluate(request).isPass();
+        return new CaseResult(c.id(), answer, relevant, grounded, facts);
+    }
+}

evaluation/src/main/java/com/ankurm/evaluation/GoldenCase.java

@@ -0,0 +1,13 @@
+package com.ankurm.evaluation;
+
+import java.util.List;
+
+import org.springframework.ai.document.Document;
+
+/** One row of the golden dataset: a question, the documents retrieval is expected to supply, and facts a good answer must contain. */
+public record GoldenCase(String id, String question, List<String> context, List<String> expectedFacts) {
+
+    public List<Document> documents() {
+        return context.stream().map(t -> Document.builder().text(t).build()).toList();
+    }
+}

evaluation/src/main/java/com/ankurm/evaluation/GoldenDataset.java

@@ -0,0 +1,28 @@
+package com.ankurm.evaluation;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+
+import tools.jackson.core.type.TypeReference;
+import tools.jackson.databind.json.JsonMapper;
+
+/** Loads the golden dataset from a JSON file on the classpath. */
+public final class GoldenDataset {
+
+    private GoldenDataset() {
+    }
+
+    public static List<GoldenCase> load(String resource) {
+        try (InputStream in = GoldenDataset.class.getResourceAsStream(resource)) {
+            if (in == null) {
+                throw new IllegalArgumentException("missing resource " + resource);
+            }
+            return JsonMapper.builder().build().readValue(in, new TypeReference<List<GoldenCase>>() {
+            });
+        }
+        catch (IOException e) {
+            throw new IllegalStateException(e);
+        }
+    }
+}

evaluation/src/main/java/com/ankurm/evaluation/GradedEvaluator.java

@@ -0,0 +1,53 @@
+package com.ankurm.evaluation;
+
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.springframework.ai.chat.client.ChatClient;
+import org.springframework.ai.evaluation.EvaluationRequest;
+import org.springframework.ai.evaluation.EvaluationResponse;
+import org.springframework.ai.evaluation.Evaluator;
+
+/**
+ * An LLM-as-judge that grades 1-5 instead of yes/no, because the built-in evaluators can only
+ * return a score of 0 or 1. It pulls the first digit 1-5 out of the reply, so "4", "4/5" and
+ * "Score: 4 - mostly right" all work; a reply with no grade fails with the raw text as feedback.
+ */
+public class GradedEvaluator implements Evaluator {
+
+    static final String PROMPT = """
+            Grade how well the answer addresses the question, using only the context.
+            Reply with a single integer from 1 (useless) to 5 (complete and correct).
+
+            Question: %s
+
+            Answer: %s
+
+            Context: %s
+            """;
+
+    private static final Pattern GRADE = Pattern.compile("\\b([1-5])\\b");
+
+    private final ChatClient.Builder builder;
+
+    private final int minimumGrade;
+
+    public GradedEvaluator(ChatClient.Builder builder, int minimumGrade) {
+        this.builder = builder;
+        this.minimumGrade = minimumGrade;
+    }
+
+    @Override
+    public EvaluationResponse evaluate(EvaluationRequest request) {
+        String reply = builder.build().prompt()
+                .user(PROMPT.formatted(request.getUserText(), request.getResponseContent(), doGetSupportingData(request)))
+                .call().content();
+        Matcher m = GRADE.matcher(reply == null ? "" : reply);
+        if (!m.find()) {
+            return new EvaluationResponse(false, 0f, "judge did not return a grade: " + reply, Map.of());
+        }
+        int grade = Integer.parseInt(m.group(1));
+        return new EvaluationResponse(grade >= minimumGrade, grade / 5f, "grade " + grade + " of 5", Map.of("grade", grade));
+    }
+}

evaluation/src/main/java/com/ankurm/evaluation/RuleBasedJudge.java

@@ -0,0 +1,77 @@
+package com.ankurm.evaluation;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import org.springframework.ai.chat.messages.AssistantMessage;
+import org.springframework.ai.chat.model.ChatModel;
+import org.springframework.ai.chat.model.ChatResponse;
+import org.springframework.ai.chat.model.Generation;
+import org.springframework.ai.chat.prompt.Prompt;
+
+/**
+ * A deterministic stand-in for a judge model, for continuous integration. It is a
+ * {@link ChatModel}, so the real {@code RelevancyEvaluator} and {@code FactCheckingEvaluator}
+ * run unchanged against it, prompt templates and all. It does not understand anything: it
+ * answers "yes" when enough of the claim's content words (and numbers) also occur in the context.
+ * That catches a swapped number or an off-topic answer and misses a negation -- see
+ * {@code JudgeLimitsTest}.
+ */
+public class RuleBasedJudge implements ChatModel {
+
+    private static final Set<String> STOP = Set.of("the", "and", "that", "this", "with", "from", "have", "your", "you", "are", "for", "can", "will", "not", "does", "what", "how", "long", "which");
+
+    private static final Pattern RELEVANCY = Pattern.compile("Response:\\s*(.*?)\\s*Context:\\s*(.*?)\\s*Answer:", Pattern.DOTALL);
+
+    private static final Pattern FACT = Pattern.compile("Document:\\s*(.*?)\\s*Claim:\\s*(.*)", Pattern.DOTALL);
+
+    private final double minimumOverlap;
+
+    public RuleBasedJudge(double minimumOverlap) {
+        this.minimumOverlap = minimumOverlap;
+    }
+
+    @Override
+    public ChatResponse call(Prompt prompt) {
+        String text = prompt.getInstructions().getLast().getText();
+        String claim;
+        String evidence;
+        Matcher r = RELEVANCY.matcher(text);
+        Matcher f = FACT.matcher(text);
+        if (r.find()) {
+            claim = r.group(1);
+            evidence = r.group(2);
+        }
+        else if (f.find()) {
+            evidence = f.group(1);
+            claim = f.group(2);
+        }
+        else {
+            throw new IllegalArgumentException("RuleBasedJudge does not recognise this prompt: " + text);
+        }
+        return new ChatResponse(List.of(new Generation(new AssistantMessage(overlap(claim, evidence) >= minimumOverlap ? "yes" : "no"))));
+    }
+
+    /** Share of the claim's content words that also appear in the evidence. */
+    public static double overlap(String claim, String evidence) {
+        Set<String> have = words(evidence);
+        Set<String> need = words(claim);
+        if (need.isEmpty()) {
+            return 0;
+        }
+        long hit = need.stream().filter(have::contains).count();
+        return hit / (double) need.size();
+    }
+
+    private static Set<String> words(String s) {
+        return Arrays.stream(s.toLowerCase(Locale.ROOT).split("[^a-z0-9]+"))
+                .filter(w -> w.length() > 3 || w.matches("\\d+"))
+                .filter(w -> !STOP.contains(w))
+                .collect(Collectors.toSet());
+    }
+}

evaluation/src/main/java/com/ankurm/evaluation/SupportAssistant.java

@@ -0,0 +1,33 @@
+package com.ankurm.evaluation;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.springframework.ai.chat.client.ChatClient;
+import org.springframework.ai.document.Document;
+
+/**
+ * The application under test: a tiny retrieval-grounded support assistant. It receives the
+ * retrieved documents as an argument so a test controls exactly what the model is shown; in a real
+ * application a {@code QuestionAnswerAdvisor} would put them there.
+ */
+public class SupportAssistant {
+
+    static final String SYSTEM = "Answer using only the context. If the context does not contain the answer, say you do not know.";
+
+    private final ChatClient client;
+
+    public SupportAssistant(ChatClient.Builder builder) {
+        this.client = builder.defaultSystem(SYSTEM).build();
+    }
+
+    public String answer(String question, List<Document> context) {
+        String joined = context.stream().map(Document::getText).collect(Collectors.joining("\n"));
+        return client.prompt()
+                .user(u -> u.text("Context:\n{context}\n\nQuestion: {question}")
+                        .param("context", joined)
+                        .param("question", question))
+                .call()
+                .content();
+    }
+}

evaluation/src/main/java/com/ankurm/evaluation/VerdictNormalizingModel.java

@@ -0,0 +1,33 @@
+package com.ankurm.evaluation;
+
+import java.util.List;
+import java.util.Locale;
+
+import org.springframework.ai.chat.messages.AssistantMessage;
+import org.springframework.ai.chat.model.ChatModel;
+import org.springframework.ai.chat.model.ChatResponse;
+import org.springframework.ai.chat.model.Generation;
+import org.springframework.ai.chat.prompt.Prompt;
+
+/**
+ * Wraps a judge model and reduces its reply to the bare word the built-in evaluators compare
+ * against. They require the whole stripped reply to equal "yes" (ignoring case), so "Yes." or "Yes,
+ * the response matches" count as a failure. This keeps the first word, drops punctuation, and
+ * leaves everything else alone.
+ */
+public class VerdictNormalizingModel implements ChatModel {
+
+    private final ChatModel delegate;
+
+    public VerdictNormalizingModel(ChatModel delegate) {
+        this.delegate = delegate;
+    }
+
+    @Override
+    public ChatResponse call(Prompt prompt) {
+        String reply = delegate.call(prompt).getResult().getOutput().getText();
+        String first = reply == null ? "" : reply.strip().split("\\s+", 2)[0].replaceAll("[^A-Za-z]", "").toLowerCase(Locale.ROOT);
+        String verdict = first.equals("yes") || first.equals("no") ? first : reply;
+        return new ChatResponse(List.of(new Generation(new AssistantMessage(verdict))));
+    }
+}

evaluation/src/test/java/com/ankurm/evaluation/CiLayersTest.java

@@ -0,0 +1,51 @@
+package com.ankurm.evaluation;
+
+import com.ankurm.evaluation.support.Answers;
+import com.ankurm.evaluation.support.RecordingModel;
+import com.ankurm.evaluation.support.Transcript;
+import org.junit.jupiter.api.Test;
+import org.springframework.ai.chat.client.ChatClient;
+import org.springframework.ai.chat.messages.AssistantMessage;
+import org.springframework.ai.chat.model.ChatResponse;
+import org.springframework.ai.chat.model.Generation;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Counts the model calls each testing layer makes, so "free in CI" is a measured number. Writes output/07. */
+class CiLayersTest {
+
+    @Test
+    void modelCallsPerLayer() {
+        var cases = GoldenDataset.load("/golden/support-golden.json");
+
+        // Layer 1: prompt/plumbing unit test, no evaluator at all.
+        RecordingModel app = Answers.appModel(Answers.good());
+        var assistant = new SupportAssistant(ChatClient.builder(app));
+        assistant.answer(cases.getFirst().question(), cases.getFirst().documents());
+        int unitAppCalls = app.callCount();
+
+        // Layer 2: golden run with the deterministic judge.
+        RecordingModel app2 = Answers.appModel(Answers.good());
+        RuleBasedJudge rule = new RuleBasedJudge(0.6);
+        int[] judgeCalls = { 0 };
+        var countingJudge = new org.springframework.ai.chat.model.ChatModel() {
+            @Override
+            public ChatResponse call(org.springframework.ai.chat.prompt.Prompt prompt) {
+                judgeCalls[0]++;
+                return rule.call(prompt);
+            }
+        };
+        EvalReport report = new EvalRunner(new SupportAssistant(ChatClient.builder(app2)), countingJudge).run(cases);
+
+        try (Transcript t = new Transcript("07-ci-layers.txt", "Model calls per testing layer (stub app model, 12 golden cases)")) {
+            t.line("%-44s %-12s %-12s %s", "layer", "app calls", "judge calls", "network");
+            t.line("%-44s %-12d %-12d %s", "1 unit: prompt and wiring, no evaluator", unitAppCalls, 0, "none");
+            t.line("%-44s %-12d %-12d %s", "2 golden run, RuleBasedJudge (pull request)", app2.callCount(), judgeCalls[0], "none");
+            t.line("%-44s %-12s %-12s %s", "3 golden run, real judge (nightly)", "n/a", "n/a", "needs OPENAI_API_KEY; skipped here");
+            t.line("pass rate at layer 2: %.2f", report.passRate());
+        }
+        assertThat(unitAppCalls).isEqualTo(1);
+        assertThat(app2.callCount()).isEqualTo(12);
+        assertThat(judgeCalls[0]).isEqualTo(24);
+    }
+}

evaluation/src/test/java/com/ankurm/evaluation/EvaluatorAnatomyTest.java

@@ -0,0 +1,62 @@
+package com.ankurm.evaluation;
+
+import java.util.List;
+
+import com.ankurm.evaluation.support.RecordingModel;
+import com.ankurm.evaluation.support.Transcript;
+import org.junit.jupiter.api.Test;
+import org.springframework.ai.chat.client.ChatClient;
+import org.springframework.ai.chat.evaluation.FactCheckingEvaluator;
+import org.springframework.ai.chat.evaluation.RelevancyEvaluator;
+import org.springframework.ai.document.Document;
+import org.springframework.ai.evaluation.EvaluationRequest;
+import org.springframework.ai.evaluation.EvaluationResponse;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** What the two built-in evaluators actually send to the judge and what they hand back. Writes output/01. */
+class EvaluatorAnatomyTest {
+
+    private static final EvaluationRequest REQUEST = new EvaluationRequest(
+            "How long do I have to request a refund on an annual plan?",
+            List.of(Document.builder().text("Annual plans can be refunded within 30 days of purchase.").build(),
+                    Document.builder().text("Monthly plans are not refundable.").build()),
+            "You can request a refund on an annual plan within 30 days of purchase.");
+
+    @Test
+    void whatTheJudgeIsSentAndWhatComesBack() {
+        try (Transcript t = new Transcript("01-evaluator-prompts.txt", "What RelevancyEvaluator and FactCheckingEvaluator send to the judge")) {
+            RecordingModel judge = RecordingModel.replying("yes");
+
+            EvaluationResponse relevancy = new RelevancyEvaluator(ChatClient.builder(judge)).evaluate(REQUEST);
+            String relevancyPrompt = judge.lastUserText();
+            EvaluationResponse fact = FactCheckingEvaluator.builder(ChatClient.builder(judge)).build().evaluate(REQUEST);
+            String factPrompt = judge.lastUserText();
+
+            t.line("--- RelevancyEvaluator: prompt sent to the judge (%d message(s)) ---", judge.prompts().getFirst().getInstructions().size());
+            t.line(relevancyPrompt);
+            t.line("--- RelevancyEvaluator: judge said \"yes\" ---");
+            t.line("pass=%s score=%s feedback='%s' metadata=%s", relevancy.isPass(), relevancy.getScore(), relevancy.getFeedback(), relevancy.getMetadata());
+            t.blank();
+            t.line("--- FactCheckingEvaluator: prompt sent to the judge ---");
+            t.line(factPrompt);
+            t.line("--- FactCheckingEvaluator: judge said \"yes\" ---");
+            t.line("pass=%s score=%s feedback='%s' metadata=%s", fact.isPass(), fact.getScore(), fact.getFeedback(), fact.getMetadata());
+            t.blank();
+
+            EvaluationResponse failing = new RelevancyEvaluator(ChatClient.builder(RecordingModel.replying("no"))).evaluate(REQUEST);
+            t.line("--- RelevancyEvaluator: judge said \"no\" ---");
+            t.line("pass=%s score=%s feedback='%s'", failing.isPass(), failing.getScore(), failing.getFeedback());
+
+            assertThat(relevancy.isPass()).isTrue();
+            assertThat(relevancy.getScore()).isEqualTo(1.0f);
+            assertThat(failing.getScore()).isEqualTo(0.0f);
+            assertThat(failing.getFeedback()).isEmpty();
+            assertThat(relevancyPrompt).contains("\tAnnual plans can be refunded within 30 days of purchase.\n\tMonthly plans are not refundable.");
+            assertThat(factPrompt).contains("Claim:").contains("Document:");
+            // FactCheckingEvaluator builds its response without a score, so a pass still reports 0.0.
+            assertThat(fact.isPass()).isTrue();
+            assertThat(fact.getScore()).isEqualTo(0.0f);
+        }
+    }
+}

evaluation/src/test/java/com/ankurm/evaluation/GoldenDatasetTest.java

@@ -0,0 +1,93 @@
+package com.ankurm.evaluation;
+
+import java.util.List;
+import java.util.Map;
+
+import com.ankurm.evaluation.support.Answers;
+import com.ankurm.evaluation.support.RecordingModel;
+import com.ankurm.evaluation.support.Transcript;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.springframework.ai.chat.client.ChatClient;
+import org.springframework.ai.chat.evaluation.FactCheckingEvaluator;
+import org.springframework.ai.chat.evaluation.RelevancyEvaluator;
+import org.springframework.ai.evaluation.EvaluationRequest;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** A golden dataset run end to end, with a pass-rate gate. A healthy build passes it; a build with three bad answers fails it. Writes output/03. */
+class GoldenDatasetTest {
+
+    static final List<GoldenCase> GOLDEN = GoldenDataset.load("/golden/support-golden.json");
+
+    private static EvalReport run(Map<String, String> answers) {
+        SupportAssistant app = new SupportAssistant(ChatClient.builder(Answers.appModel(answers)));
+        return new EvalRunner(app, new RuleBasedJudge(0.6)).run(GOLDEN);
+    }
+
+    @Test
+    void healthyAndRegressedBuilds() {
+        EvalReport healthy = run(Answers.good());
+        EvalReport regressed = run(Answers.regressed());
+
+        try (Transcript t = new Transcript("03-golden-run.txt", "Golden dataset: %d cases, deterministic judge, pass-rate gate at 0.90".formatted(GOLDEN.size()))) {
+            for (var entry : List.of(Map.entry("healthy build", healthy), Map.entry("regressed build", regressed))) {
+                t.line("== %s ==", entry.getKey());
+                t.line("%-18s %-9s %-9s %-9s %-7s", "case", "relevant", "grounded", "has facts", "passed");
+                for (CaseResult r : entry.getValue().results()) {
+                    t.line("%-18s %-9s %-9s %-9s %-7s", r.id(), r.relevant(), r.grounded(), r.hasFacts(), r.passed());
+                }
+                t.line("pass rate: %.2f   failing: %s", entry.getValue().passRate(), entry.getValue().failedIds());
+                t.blank();
+            }
+            t.line("== the gate ==");
+            healthy.requirePassRate(0.90);
+            t.line("healthy build   : requirePassRate(0.90) returned normally");
+            Throwable thrown = org.assertj.core.api.Assertions.catchThrowable(() -> regressed.requirePassRate(0.90));
+            t.line("regressed build : %s", thrown);
+        }
+
+        assertThat(healthy.passRate()).isEqualTo(1.0);
+        assertThat(regressed.failedIds()).containsExactly("refund-annual", "sso-plan", "extra-seat");
+        assertThatThrownBy(() -> regressed.requirePassRate(0.90)).isInstanceOf(AssertionError.class).hasMessageContaining("sso-plan");
+    }
+
+    @Test
+    void whichEvaluatorCaughtWhat() {
+        EvalReport regressed = run(Answers.regressed());
+        CaseResult hallucinated = regressed.results().stream().filter(r -> r.id().equals("refund-annual")).findFirst().orElseThrow();
+        CaseResult offTopic = regressed.results().stream().filter(r -> r.id().equals("sso-plan")).findFirst().orElseThrow();
+        CaseResult wrongNumber = regressed.results().stream().filter(r -> r.id().equals("extra-seat")).findFirst().orElseThrow();
+
+        assertThat(hallucinated.grounded()).isFalse();
+        assertThat(offTopic.relevant()).isFalse();
+        // A plausible wrong number slips past the judge (5 of 6 content words overlap) and is caught only by the exact-fact check.
+        assertThat(wrongNumber.relevant()).isTrue();
+        assertThat(wrongNumber.grounded()).isTrue();
+        assertThat(wrongNumber.hasFacts()).isFalse();
+    }
+
+    static List<GoldenCase> cases() {
+        return GOLDEN;
+    }
+
+    /** The per-case style: one JUnit invocation per golden row, readable in any test report. Only suitable for a deterministic judge. */
+    @ParameterizedTest(name = "[{index}] {0}")
+    @MethodSource("ids")
+    void healthyBuildPassesEveryCase(String id) {
+        GoldenCase c = GOLDEN.stream().filter(g -> g.id().equals(id)).findFirst().orElseThrow();
+        RecordingModel app = Answers.appModel(Answers.good());
+        String answer = new SupportAssistant(ChatClient.builder(app)).answer(c.question(), c.documents());
+        EvaluationRequest request = new EvaluationRequest(c.question(), c.documents(), answer);
+        RuleBasedJudge judge = new RuleBasedJudge(0.6);
+        assertThat(new RelevancyEvaluator(ChatClient.builder(judge)).evaluate(request).isPass()).as("relevant").isTrue();
+        assertThat(FactCheckingEvaluator.builder(ChatClient.builder(judge)).build().evaluate(request).isPass()).as("grounded").isTrue();
+        assertThat(new ContainsFactsEvaluator(c.expectedFacts()).evaluate(request).isPass()).as("facts").isTrue();
+    }
+
+    static List<String> ids() {
+        return GOLDEN.stream().map(GoldenCase::id).toList();
+    }
+}

evaluation/src/test/java/com/ankurm/evaluation/GradedCompositeTest.java

@@ -0,0 +1,65 @@
+package com.ankurm.evaluation;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.ankurm.evaluation.support.Answers;
+import com.ankurm.evaluation.support.RecordingModel;
+import com.ankurm.evaluation.support.Transcript;
+import org.junit.jupiter.api.Test;
+import org.springframework.ai.chat.client.ChatClient;
+import org.springframework.ai.chat.evaluation.RelevancyEvaluator;
+import org.springframework.ai.document.Document;
+import org.springframework.ai.evaluation.EvaluationRequest;
+import org.springframework.ai.evaluation.EvaluationResponse;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** The two custom evaluators: a 1-5 grade (the built-ins only score 0 or 1) and a composite that restores feedback. Writes output/05. */
+class GradedCompositeTest {
+
+    private static final EvaluationRequest REQUEST = new EvaluationRequest("Q?", List.of(Document.builder().text("ctx").build()), "answer");
+
+    @Test
+    void gradedEvaluatorReadsAGradeOutOfMessyReplies() {
+        Map<String, Boolean> expected = new LinkedHashMap<>();
+        expected.put("5", true);
+        expected.put("4/5", true);
+        expected.put("Score: 4 - mostly right, misses the date", true);
+        expected.put("3", false);
+        expected.put("I would say excellent", false);
+        expected.put("10", false);
+
+        try (Transcript t = new Transcript("05-graded-and-composite.txt", "GradedEvaluator (minimum grade 4) and CompositeEvaluator")) {
+            t.line("%-44s %-6s %-6s %s", "judge reply", "pass", "score", "feedback");
+            for (var e : expected.entrySet()) {
+                EvaluationResponse r = new GradedEvaluator(ChatClient.builder(RecordingModel.replying(e.getKey())), 4).evaluate(REQUEST);
+                t.line("%-44s %-6s %-6s %s", "'" + e.getKey() + "'", r.isPass(), r.getScore(), r.getFeedback());
+                assertThat(r.isPass()).as(e.getKey()).isEqualTo(e.getValue());
+            }
+
+            t.blank();
+            t.line("== CompositeEvaluator on the regressed 'extra-seat' answer (\"12 USD\" instead of \"8 USD\") ==");
+            GoldenCase c = GoldenDatasetTest.GOLDEN.stream().filter(g -> g.id().equals("extra-seat")).findFirst().orElseThrow();
+            String answer = Answers.regressed().get(c.question());
+            EvaluationRequest request = new EvaluationRequest(c.question(), c.documents(), answer);
+            RuleBasedJudge judge = new RuleBasedJudge(0.6);
+            var composite = new CompositeEvaluator(new LinkedHashMap<>(Map.of()) {
+                {
+                    put("relevancy", new RelevancyEvaluator(ChatClient.builder(judge)));
+                    put("contains-facts", new ContainsFactsEvaluator(c.expectedFacts()));
+                }
+            });
+            EvaluationResponse r = composite.evaluate(request);
+            t.line("answer   : %s", answer);
+            t.line("pass     : %s", r.isPass());
+            t.line("score    : %s", r.getScore());
+            t.line("feedback : %s", r.getFeedback());
+            t.line("verdicts : %s", r.getMetadata());
+            assertThat(r.isPass()).isFalse();
+            assertThat(r.getFeedback()).contains("contains-facts failed").contains("8 USD");
+            assertThat(r.getMetadata()).containsEntry("relevancy", true).containsEntry("contains-facts", false);
+        }
+    }
+}

evaluation/src/test/java/com/ankurm/evaluation/JudgeLimitsTest.java

@@ -0,0 +1,45 @@
+package com.ankurm.evaluation;
+
+import java.util.List;
+
+import com.ankurm.evaluation.support.Transcript;
+import org.junit.jupiter.api.Test;
+import org.springframework.ai.chat.client.ChatClient;
+import org.springframework.ai.chat.evaluation.RelevancyEvaluator;
+import org.springframework.ai.document.Document;
+import org.springframework.ai.evaluation.EvaluationRequest;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** What the cheap deterministic judge gets wrong. This is the reason a real judge still runs somewhere (nightly). Writes output/06. */
+class JudgeLimitsTest {
+
+    private record Probe(String label, String answer, boolean actuallyCorrect) {
+    }
+
+    @Test
+    void theRuleBasedJudgeIsFooledInBothDirections() {
+        List<Document> context = List.of(Document.builder().text("Annual plans can be refunded within 30 days of purchase. Monthly plans are not refundable.").build());
+        String question = "Can I get a refund on a monthly plan?";
+        List<Probe> probes = List.of(
+                new Probe("correct, same words", "Monthly plans are not refundable.", true),
+                new Probe("WRONG: negation dropped", "Monthly plans are refundable.", false),
+                new Probe("WRONG: roles swapped", "Annual plans are not refundable. Monthly plans can be refunded within 30 days of purchase.", false),
+                new Probe("correct, paraphrased", "No, you cannot get your money back on month-to-month subscriptions.", true),
+                new Probe("correct, but a refusal", "I do not have that information.", true));
+
+        try (Transcript t = new Transcript("06-judge-limits.txt", "RuleBasedJudge(0.6) on hand-labelled answers to: " + question)) {
+            t.line("%-26s %-9s %-14s %s", "answer type", "correct?", "judge says", "verdict");
+            for (Probe p : probes) {
+                boolean judged = new RelevancyEvaluator(ChatClient.builder(new RuleBasedJudge(0.6)))
+                        .evaluate(new EvaluationRequest(question, context, p.answer())).isPass();
+                t.line("%-26s %-9s %-14s %s", p.label(), p.actuallyCorrect(), judged ? "pass" : "fail", judged == p.actuallyCorrect() ? "right" : "WRONG");
+            }
+        }
+
+        boolean negation = new RelevancyEvaluator(ChatClient.builder(new RuleBasedJudge(0.6))).evaluate(new EvaluationRequest(question, context, "Monthly plans are refundable.")).isPass();
+        boolean paraphrase = new RelevancyEvaluator(ChatClient.builder(new RuleBasedJudge(0.6))).evaluate(new EvaluationRequest(question, context, "No, you cannot get your money back on month-to-month subscriptions.")).isPass();
+        assertThat(negation).as("a dropped negation shares every content word, so the judge accepts it").isTrue();
+        assertThat(paraphrase).as("a correct paraphrase shares almost no words, so the judge rejects it").isFalse();
+    }
+}

evaluation/src/test/java/com/ankurm/evaluation/LiveJudgeTest.java

@@ -0,0 +1,41 @@
+package com.ankurm.evaluation;
+
+import com.ankurm.evaluation.support.Answers;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
+import org.springframework.ai.chat.client.ChatClient;
+import org.springframework.ai.chat.model.ChatModel;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.test.context.SpringBootTest;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * The nightly layer: the same golden dataset and the same evaluators, judged by a real model.
+ * Skipped unless {@code OPENAI_API_KEY} is set, so a pull-request build never spends money. It was
+ * compiled and its skip was recorded for this article, but it was NOT executed against a live
+ * model, so no live-judge result appears anywhere in the post.
+ */
+@Tag("live")
+@EnabledIfEnvironmentVariable(named = "OPENAI_API_KEY", matches = ".+")
+@SpringBootTest(classes = LiveJudgeTest.App.class)
+class LiveJudgeTest {
+
+    @SpringBootApplication
+    static class App {
+    }
+
+    @Autowired
+    ChatModel model;
+
+    @Test
+    void realJudgeAgreesWithTheKnownGoodAndKnownBadBuilds() {
+        var cases = GoldenDataset.load("/golden/support-golden.json");
+        var healthy = new EvalRunner(new SupportAssistant(ChatClient.builder(Answers.appModel(Answers.good()))), model).run(cases);
+        var regressed = new EvalRunner(new SupportAssistant(ChatClient.builder(Answers.appModel(Answers.regressed()))), model).run(cases);
+        assertThat(healthy.passRate()).isGreaterThanOrEqualTo(0.85);
+        assertThat(regressed.passRate()).isLessThan(healthy.passRate());
+    }
+}

evaluation/src/test/java/com/ankurm/evaluation/NoisyJudgeTest.java

@@ -0,0 +1,67 @@
+package com.ankurm.evaluation;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import com.ankurm.evaluation.support.Answers;
+import com.ankurm.evaluation.support.FlakyJudge;
+import com.ankurm.evaluation.support.Transcript;
+import org.junit.jupiter.api.Test;
+import org.springframework.ai.chat.client.ChatClient;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * What a noisy judge does to a test suite. The noise is SIMULATED (a correct judge whose verdict
+ * is flipped 3 or 8 percent of the time, seeded), so the numbers show the effect of noise on the gating
+ * rule and say nothing about how often any real model is wrong. Writes output/04.
+ */
+class NoisyJudgeTest {
+
+    static final List<GoldenCase> GOLDEN = GoldenDataset.load("/golden/support-golden.json");
+
+    static final int RUNS = 200;
+
+    private static double passRate(Map<String, String> answers, double flip, long seed) {
+        SupportAssistant app = new SupportAssistant(ChatClient.builder(Answers.appModel(answers)));
+        return new EvalRunner(app, new FlakyJudge(new RuleBasedJudge(0.6), flip, seed)).run(GOLDEN).passRate();
+    }
+
+    private static long below(double[] rates, double threshold) {
+        return Arrays.stream(rates).filter(r -> r < threshold).count();
+    }
+
+    @Test
+    void strictPerCaseAssertionsVersusAnAggregateThreshold() {
+        double[] flips = { 0.03, 0.08 };
+        double[][] healthy = new double[flips.length][RUNS];
+        double[][] regressed = new double[flips.length][RUNS];
+        for (int f = 0; f < flips.length; f++) {
+            for (int i = 0; i < RUNS; i++) {
+                healthy[f][i] = passRate(Answers.good(), flips[f], i);
+                regressed[f][i] = passRate(Answers.regressed(), flips[f], 10_000 + i);
+            }
+        }
+
+        try (Transcript t = new Transcript("04-noisy-judge.txt", "Simulated judge noise: %d runs of a %d-case suite per build".formatted(RUNS, GOLDEN.size()))) {
+            t.line("noise-free pass rate: healthy build 1.00, regressed build 0.75");
+            for (int f = 0; f < flips.length; f++) {
+                t.blank();
+                t.line("== %.0f%% of judge verdicts flipped (mean pass rate: healthy %.3f, regressed %.3f) ==", flips[f] * 100,
+                        Arrays.stream(healthy[f]).average().orElse(0), Arrays.stream(regressed[f]).average().orElse(0));
+                t.line("%-22s %-28s %s", "gate", "healthy build fails it", "regressed build fails it");
+                for (double threshold : new double[] { 1.0, 0.90, 0.85, 0.80 }) {
+                    String name = threshold == 1.0 ? "every case must pass" : "pass rate >= %.2f".formatted(threshold);
+                    t.line("%-22s %-28s %s", name, "%d of %d runs".formatted(below(healthy[f], threshold), RUNS), "%d of %d runs".formatted(below(regressed[f], threshold), RUNS));
+                }
+            }
+        }
+
+        // At 3 percent noise a 0.80 gate has few false alarms and still catches the regression; "every case must pass" does not.
+        assertThat(below(healthy[0], 1.0)).isGreaterThan(below(healthy[0], 0.80));
+        assertThat(below(regressed[0], 0.80)).isGreaterThan(below(healthy[0], 0.80));
+        // At 8 percent noise even 0.80 raises false alarms: the gate has to be set from measured noise, not guessed.
+        assertThat(below(healthy[1], 0.80)).isGreaterThan(below(healthy[0], 0.80));
+    }
+}

evaluation/src/test/java/com/ankurm/evaluation/VerdictParsingTest.java

@@ -0,0 +1,64 @@
+package com.ankurm.evaluation;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import com.ankurm.evaluation.support.RecordingModel;
+import com.ankurm.evaluation.support.Transcript;
+import org.junit.jupiter.api.Test;
+import org.springframework.ai.chat.client.ChatClient;
+import org.springframework.ai.chat.evaluation.FactCheckingEvaluator;
+import org.springframework.ai.chat.evaluation.RelevancyEvaluator;
+import org.springframework.ai.chat.model.ChatModel;
+import org.springframework.ai.document.Document;
+import org.springframework.ai.evaluation.EvaluationRequest;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** The built-in evaluators pass only on a bare "yes". This shows which judge replies count, and the normalising wrapper that fixes it. Writes output/02. */
+class VerdictParsingTest {
+
+    private static final EvaluationRequest REQUEST = new EvaluationRequest("Q?", List.of(Document.builder().text("ctx").build()), "answer");
+
+    private static boolean relevancy(ChatModel judge) {
+        return new RelevancyEvaluator(ChatClient.builder(judge)).evaluate(REQUEST).isPass();
+    }
+
+    private static boolean fact(ChatModel judge) {
+        return FactCheckingEvaluator.builder(ChatClient.builder(judge)).build().evaluate(REQUEST).isPass();
+    }
+
+    @Test
+    void onlyABareYesPasses() {
+        Map<String, Boolean> expectedRaw = new LinkedHashMap<>();
+        expectedRaw.put("yes", true);
+        expectedRaw.put("YES", true);
+        expectedRaw.put("  yes\n", true);
+        expectedRaw.put("Yes.", false);
+        expectedRaw.put("yes!", false);
+        expectedRaw.put("Yes, the response is in line with the context.", false);
+        expectedRaw.put("**Yes**", false);
+        expectedRaw.put("no", false);
+        expectedRaw.put("No.", false);
+        expectedRaw.put("", false);
+
+        try (Transcript t = new Transcript("02-verdict-parsing.txt", "Which judge replies the built-in evaluators accept")) {
+            t.line("%-52s %-9s %-9s %-9s", "judge reply", "relevancy", "fact-chk", "wrapped");
+            for (var e : expectedRaw.entrySet()) {
+                ChatModel raw = RecordingModel.replying(e.getKey());
+                boolean r = relevancy(raw);
+                boolean f = fact(raw);
+                boolean wrapped = relevancy(new VerdictNormalizingModel(raw));
+                t.line("%-52s %-9s %-9s %-9s", "'" + e.getKey().replace("\n", "\\n") + "'", r, f, wrapped);
+                assertThat(r).as("relevancy for '%s'", e.getKey()).isEqualTo(e.getValue());
+                assertThat(f).as("fact for '%s'", e.getKey()).isEqualTo(e.getValue());
+            }
+            // The wrapper turns the punctuated, bold and sentence-style verdicts into passes, and leaves "no" alone.
+            assertThat(relevancy(new VerdictNormalizingModel(RecordingModel.replying("Yes.")))).isTrue();
+            assertThat(relevancy(new VerdictNormalizingModel(RecordingModel.replying("Yes, the response is in line with the context.")))).isTrue();
+            assertThat(relevancy(new VerdictNormalizingModel(RecordingModel.replying("No.")))).isFalse();
+            assertThat(relevancy(new VerdictNormalizingModel(RecordingModel.replying("**Yes**")))).isTrue();
+        }
+    }
+}

evaluation/src/test/java/com/ankurm/evaluation/support/Answers.java

@@ -0,0 +1,54 @@
+package com.ankurm.evaluation.support;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.springframework.ai.chat.prompt.Prompt;
+
+/**
+ * What the pretend application model says for each golden question. {@link #good()} is a healthy
+ * build; {@link #regressed()} is the same build after a bad prompt change: one hallucinated
+ * answer, one off-topic answer, and one with a plausible but wrong number.
+ */
+public final class Answers {
+
+    private Answers() {
+    }
+
+    public static Map<String, String> good() {
+        Map<String, String> m = new HashMap<>();
+        m.put("How long do I have to request a refund on an annual plan?", "You can request a refund on an annual plan within 30 days of purchase.");
+        m.put("Can I get a refund on a monthly plan?", "No. Monthly plans are not refundable.");
+        m.put("How much storage does the Pro plan include?", "The Pro plan includes 200 GB of storage.");
+        m.put("When is support open?", "Support is available Monday to Friday, 9:00 to 17:00 UTC.");
+        m.put("Which plan has SAML single sign-on?", "SAML single sign-on is available on the Enterprise plan only.");
+        m.put("Can I move my data to the US after signing up?", "No. The region can only be chosen at account creation and cannot be changed later.");
+        m.put("What happens if I exceed the API rate limit?", "Exceeding 600 requests per minute returns HTTP 429.");
+        m.put("How long are Pro backups kept?", "Pro backups are kept for 14 days.");
+        m.put("Is there a macOS agent?", "I do not know. Acme Cloud supports Linux and Windows agents.");
+        m.put("How long is a password reset link valid?", "A password reset link expires after 60 minutes.");
+        m.put("How much does an extra seat cost?", "Extra seats cost 8 USD per month.");
+        m.put("Do I need a credit card for the trial?", "No. The free trial does not require a credit card.");
+        return m;
+    }
+
+    public static Map<String, String> regressed() {
+        Map<String, String> m = good();
+        m.put("How long do I have to request a refund on an annual plan?", "You can request a refund within 60 days, including on monthly plans.");
+        m.put("Which plan has SAML single sign-on?", "Our team is happy to help with any sign-in question you have.");
+        m.put("How much does an extra seat cost?", "Extra seats cost 12 USD per month.");
+        return m;
+    }
+
+    /** The scripted application model: finds the golden question in the user message and returns its answer. */
+    public static RecordingModel appModel(Map<String, String> answers) {
+        return new RecordingModel((Prompt p) -> {
+            String text = p.getInstructions().getLast().getText();
+            return answers.entrySet().stream()
+                    .filter(e -> text.contains("Question: " + e.getKey()))
+                    .map(Map.Entry::getValue)
+                    .findFirst()
+                    .orElse("I do not know.");
+        });
+    }
+}

evaluation/src/test/java/com/ankurm/evaluation/support/FlakyJudge.java

@@ -0,0 +1,39 @@
+package com.ankurm.evaluation.support;
+
+import java.util.List;
+import java.util.Random;
+
+import org.springframework.ai.chat.messages.AssistantMessage;
+import org.springframework.ai.chat.model.ChatModel;
+import org.springframework.ai.chat.model.ChatResponse;
+import org.springframework.ai.chat.model.Generation;
+import org.springframework.ai.chat.prompt.Prompt;
+
+/**
+ * A SIMULATION of judge noise, not a measurement of any real model: it forwards to a correct
+ * judge, then with a fixed probability returns the opposite verdict. The random source is seeded,
+ * so runs are repeatable. The point is to show what noise does to a test suite, not to claim a rate.
+ */
+public class FlakyJudge implements ChatModel {
+
+    private final ChatModel delegate;
+
+    private final double flipRate;
+
+    private final Random random;
+
+    public FlakyJudge(ChatModel delegate, double flipRate, long seed) {
+        this.delegate = delegate;
+        this.flipRate = flipRate;
+        this.random = new Random(seed);
+    }
+
+    @Override
+    public ChatResponse call(Prompt prompt) {
+        String verdict = delegate.call(prompt).getResult().getOutput().getText();
+        if (random.nextDouble() < flipRate) {
+            verdict = verdict.equals("yes") ? "no" : "yes";
+        }
+        return new ChatResponse(List.of(new Generation(new AssistantMessage(verdict))));
+    }
+}

evaluation/src/test/java/com/ankurm/evaluation/support/RecordingModel.java

@@ -0,0 +1,45 @@
+package com.ankurm.evaluation.support;
+
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.function.Function;
+
+import org.springframework.ai.chat.messages.AssistantMessage;
+import org.springframework.ai.chat.model.ChatModel;
+import org.springframework.ai.chat.model.ChatResponse;
+import org.springframework.ai.chat.model.Generation;
+import org.springframework.ai.chat.prompt.Prompt;
+
+/** A scripted {@link ChatModel}: no network, no key. It records every prompt and answers with whatever the replier returns. */
+public class RecordingModel implements ChatModel {
+
+    private final List<Prompt> prompts = new CopyOnWriteArrayList<>();
+
+    private final Function<Prompt, String> replier;
+
+    public RecordingModel(Function<Prompt, String> replier) {
+        this.replier = replier;
+    }
+
+    public static RecordingModel replying(String fixed) {
+        return new RecordingModel(p -> fixed);
+    }
+
+    public List<Prompt> prompts() {
+        return prompts;
+    }
+
+    public int callCount() {
+        return prompts.size();
+    }
+
+    public String lastUserText() {
+        return prompts.getLast().getInstructions().getLast().getText();
+    }
+
+    @Override
+    public ChatResponse call(Prompt prompt) {
+        prompts.add(prompt);
+        return new ChatResponse(List.of(new Generation(new AssistantMessage(replier.apply(prompt)))));
+    }
+}

evaluation/src/test/java/com/ankurm/evaluation/support/Transcript.java

@@ -0,0 +1,47 @@
+package com.ankurm.evaluation.support;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * Writes a numbered transcript under {@code output/} (repository root, not {@code docs/}) and
+ * echoes it to the console. Every console block quoted in the article comes out of one of these
+ * files verbatim.
+ */
+public final class Transcript implements AutoCloseable {
+
+    private final Path path;
+    private final StringWriter buffer = new StringWriter();
+    private final PrintWriter out = new PrintWriter(buffer);
+
+    public Transcript(String fileName, String title) {
+        this.path = Path.of("output", fileName);
+        out.println("# " + title);
+        out.println();
+    }
+
+    public Transcript line(String format, Object... args) {
+        out.println(args.length == 0 ? format : String.format(format, args));
+        return this;
+    }
+
+    public Transcript blank() {
+        out.println();
+        return this;
+    }
+
+    @Override
+    public void close() {
+        out.flush();
+        try {
+            Files.createDirectories(path.getParent());
+            Files.writeString(path, buffer.toString());
+        } catch (IOException e) {
+            throw new IllegalStateException("could not write " + path, e);
+        }
+        System.out.print(buffer);
+    }
+}

evaluation/src/test/resources/golden/support-golden.json

@@ -0,0 +1,26 @@
+[
+  {"id": "refund-annual", "question": "How long do I have to request a refund on an annual plan?",
+   "context": ["Annual plans can be refunded within 30 days of purchase. Monthly plans are not refundable."], "expectedFacts": ["30 days"]},
+  {"id": "refund-monthly", "question": "Can I get a refund on a monthly plan?",
+   "context": ["Annual plans can be refunded within 30 days of purchase. Monthly plans are not refundable."], "expectedFacts": ["not refundable"]},
+  {"id": "storage-pro", "question": "How much storage does the Pro plan include?",
+   "context": ["The Free plan includes 5 GB of storage. The Pro plan includes 200 GB of storage."], "expectedFacts": ["200 GB"]},
+  {"id": "support-hours", "question": "When is support open?",
+   "context": ["Support is available Monday to Friday, 9:00 to 17:00 UTC. Enterprise customers have 24/7 support."], "expectedFacts": ["Monday to Friday"]},
+  {"id": "sso-plan", "question": "Which plan has SAML single sign-on?",
+   "context": ["Single sign-on with SAML is available on the Enterprise plan only."], "expectedFacts": ["Enterprise"]},
+  {"id": "data-region", "question": "Can I move my data to the US after signing up?",
+   "context": ["Data is stored in the EU region by default. US storage can be selected at account creation and cannot be changed later."], "expectedFacts": ["cannot be changed"]},
+  {"id": "api-rate-limit", "question": "What happens if I exceed the API rate limit?",
+   "context": ["The API allows 600 requests per minute per key. Exceeding the limit returns HTTP 429."], "expectedFacts": ["429"]},
+  {"id": "backup-retention", "question": "How long are Pro backups kept?",
+   "context": ["Backups are kept for 14 days on the Pro plan and 35 days on the Enterprise plan."], "expectedFacts": ["14 days"]},
+  {"id": "macos-agent", "question": "Is there a macOS agent?",
+   "context": ["Acme Cloud supports Linux and Windows agents."], "expectedFacts": ["do not know"]},
+  {"id": "reset-link", "question": "How long is a password reset link valid?",
+   "context": ["Password reset links expire after 60 minutes."], "expectedFacts": ["60 minutes"]},
+  {"id": "extra-seat", "question": "How much does an extra seat cost?",
+   "context": ["Each Pro plan includes 5 seats. Extra seats cost 8 USD per month."], "expectedFacts": ["8 USD"]},
+  {"id": "trial-card", "question": "Do I need a credit card for the trial?",
+   "context": ["The free trial lasts 14 days and does not require a credit card."], "expectedFacts": ["does not require"]}
+]