Add evaluation module: RelevancyEvaluator and FactCheckingEvaluator, golden dataset with a pass-rate gate, deterministic CI judge and simulated judge noise
Co-Authored-By: Claude Sonnet 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01JXVi2GMQ7bR5EmbUFdDj7N
This commit is contained in:
Claude
@@ -0,0 +1,51 @@
|
||||
package com.ankurm.evaluation;
|
||||
|
||||
import com.ankurm.evaluation.support.Answers;
|
||||
import com.ankurm.evaluation.support.RecordingModel;
|
||||
import com.ankurm.evaluation.support.Transcript;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/** Counts the model calls each testing layer makes, so "free in CI" is a measured number. Writes output/07. */
|
||||
class CiLayersTest {
|
||||
|
||||
@Test
|
||||
void modelCallsPerLayer() {
|
||||
var cases = GoldenDataset.load("/golden/support-golden.json");
|
||||
|
||||
// Layer 1: prompt/plumbing unit test, no evaluator at all.
|
||||
RecordingModel app = Answers.appModel(Answers.good());
|
||||
var assistant = new SupportAssistant(ChatClient.builder(app));
|
||||
assistant.answer(cases.getFirst().question(), cases.getFirst().documents());
|
||||
int unitAppCalls = app.callCount();
|
||||
|
||||
// Layer 2: golden run with the deterministic judge.
|
||||
RecordingModel app2 = Answers.appModel(Answers.good());
|
||||
RuleBasedJudge rule = new RuleBasedJudge(0.6);
|
||||
int[] judgeCalls = { 0 };
|
||||
var countingJudge = new org.springframework.ai.chat.model.ChatModel() {
|
||||
@Override
|
||||
public ChatResponse call(org.springframework.ai.chat.prompt.Prompt prompt) {
|
||||
judgeCalls[0]++;
|
||||
return rule.call(prompt);
|
||||
}
|
||||
};
|
||||
EvalReport report = new EvalRunner(new SupportAssistant(ChatClient.builder(app2)), countingJudge).run(cases);
|
||||
|
||||
try (Transcript t = new Transcript("07-ci-layers.txt", "Model calls per testing layer (stub app model, 12 golden cases)")) {
|
||||
t.line("%-44s %-12s %-12s %s", "layer", "app calls", "judge calls", "network");
|
||||
t.line("%-44s %-12d %-12d %s", "1 unit: prompt and wiring, no evaluator", unitAppCalls, 0, "none");
|
||||
t.line("%-44s %-12d %-12d %s", "2 golden run, RuleBasedJudge (pull request)", app2.callCount(), judgeCalls[0], "none");
|
||||
t.line("%-44s %-12s %-12s %s", "3 golden run, real judge (nightly)", "n/a", "n/a", "needs OPENAI_API_KEY; skipped here");
|
||||
t.line("pass rate at layer 2: %.2f", report.passRate());
|
||||
}
|
||||
assertThat(unitAppCalls).isEqualTo(1);
|
||||
assertThat(app2.callCount()).isEqualTo(12);
|
||||
assertThat(judgeCalls[0]).isEqualTo(24);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package com.ankurm.evaluation;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.ankurm.evaluation.support.RecordingModel;
|
||||
import com.ankurm.evaluation.support.Transcript;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.chat.evaluation.FactCheckingEvaluator;
|
||||
import org.springframework.ai.chat.evaluation.RelevancyEvaluator;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.evaluation.EvaluationRequest;
|
||||
import org.springframework.ai.evaluation.EvaluationResponse;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/** What the two built-in evaluators actually send to the judge and what they hand back. Writes output/01. */
|
||||
class EvaluatorAnatomyTest {
|
||||
|
||||
private static final EvaluationRequest REQUEST = new EvaluationRequest(
|
||||
"How long do I have to request a refund on an annual plan?",
|
||||
List.of(Document.builder().text("Annual plans can be refunded within 30 days of purchase.").build(),
|
||||
Document.builder().text("Monthly plans are not refundable.").build()),
|
||||
"You can request a refund on an annual plan within 30 days of purchase.");
|
||||
|
||||
@Test
|
||||
void whatTheJudgeIsSentAndWhatComesBack() {
|
||||
try (Transcript t = new Transcript("01-evaluator-prompts.txt", "What RelevancyEvaluator and FactCheckingEvaluator send to the judge")) {
|
||||
RecordingModel judge = RecordingModel.replying("yes");
|
||||
|
||||
EvaluationResponse relevancy = new RelevancyEvaluator(ChatClient.builder(judge)).evaluate(REQUEST);
|
||||
String relevancyPrompt = judge.lastUserText();
|
||||
EvaluationResponse fact = FactCheckingEvaluator.builder(ChatClient.builder(judge)).build().evaluate(REQUEST);
|
||||
String factPrompt = judge.lastUserText();
|
||||
|
||||
t.line("--- RelevancyEvaluator: prompt sent to the judge (%d message(s)) ---", judge.prompts().getFirst().getInstructions().size());
|
||||
t.line(relevancyPrompt);
|
||||
t.line("--- RelevancyEvaluator: judge said \"yes\" ---");
|
||||
t.line("pass=%s score=%s feedback='%s' metadata=%s", relevancy.isPass(), relevancy.getScore(), relevancy.getFeedback(), relevancy.getMetadata());
|
||||
t.blank();
|
||||
t.line("--- FactCheckingEvaluator: prompt sent to the judge ---");
|
||||
t.line(factPrompt);
|
||||
t.line("--- FactCheckingEvaluator: judge said \"yes\" ---");
|
||||
t.line("pass=%s score=%s feedback='%s' metadata=%s", fact.isPass(), fact.getScore(), fact.getFeedback(), fact.getMetadata());
|
||||
t.blank();
|
||||
|
||||
EvaluationResponse failing = new RelevancyEvaluator(ChatClient.builder(RecordingModel.replying("no"))).evaluate(REQUEST);
|
||||
t.line("--- RelevancyEvaluator: judge said \"no\" ---");
|
||||
t.line("pass=%s score=%s feedback='%s'", failing.isPass(), failing.getScore(), failing.getFeedback());
|
||||
|
||||
assertThat(relevancy.isPass()).isTrue();
|
||||
assertThat(relevancy.getScore()).isEqualTo(1.0f);
|
||||
assertThat(failing.getScore()).isEqualTo(0.0f);
|
||||
assertThat(failing.getFeedback()).isEmpty();
|
||||
assertThat(relevancyPrompt).contains("\tAnnual plans can be refunded within 30 days of purchase.\n\tMonthly plans are not refundable.");
|
||||
assertThat(factPrompt).contains("Claim:").contains("Document:");
|
||||
// FactCheckingEvaluator builds its response without a score, so a pass still reports 0.0.
|
||||
assertThat(fact.isPass()).isTrue();
|
||||
assertThat(fact.getScore()).isEqualTo(0.0f);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package com.ankurm.evaluation;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.ankurm.evaluation.support.Answers;
|
||||
import com.ankurm.evaluation.support.RecordingModel;
|
||||
import com.ankurm.evaluation.support.Transcript;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.chat.evaluation.FactCheckingEvaluator;
|
||||
import org.springframework.ai.chat.evaluation.RelevancyEvaluator;
|
||||
import org.springframework.ai.evaluation.EvaluationRequest;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/** A golden dataset run end to end, with a pass-rate gate. A healthy build passes it; a build with three bad answers fails it. Writes output/03. */
|
||||
class GoldenDatasetTest {
|
||||
|
||||
static final List<GoldenCase> GOLDEN = GoldenDataset.load("/golden/support-golden.json");
|
||||
|
||||
private static EvalReport run(Map<String, String> answers) {
|
||||
SupportAssistant app = new SupportAssistant(ChatClient.builder(Answers.appModel(answers)));
|
||||
return new EvalRunner(app, new RuleBasedJudge(0.6)).run(GOLDEN);
|
||||
}
|
||||
|
||||
@Test
|
||||
void healthyAndRegressedBuilds() {
|
||||
EvalReport healthy = run(Answers.good());
|
||||
EvalReport regressed = run(Answers.regressed());
|
||||
|
||||
try (Transcript t = new Transcript("03-golden-run.txt", "Golden dataset: %d cases, deterministic judge, pass-rate gate at 0.90".formatted(GOLDEN.size()))) {
|
||||
for (var entry : List.of(Map.entry("healthy build", healthy), Map.entry("regressed build", regressed))) {
|
||||
t.line("== %s ==", entry.getKey());
|
||||
t.line("%-18s %-9s %-9s %-9s %-7s", "case", "relevant", "grounded", "has facts", "passed");
|
||||
for (CaseResult r : entry.getValue().results()) {
|
||||
t.line("%-18s %-9s %-9s %-9s %-7s", r.id(), r.relevant(), r.grounded(), r.hasFacts(), r.passed());
|
||||
}
|
||||
t.line("pass rate: %.2f failing: %s", entry.getValue().passRate(), entry.getValue().failedIds());
|
||||
t.blank();
|
||||
}
|
||||
t.line("== the gate ==");
|
||||
healthy.requirePassRate(0.90);
|
||||
t.line("healthy build : requirePassRate(0.90) returned normally");
|
||||
Throwable thrown = org.assertj.core.api.Assertions.catchThrowable(() -> regressed.requirePassRate(0.90));
|
||||
t.line("regressed build : %s", thrown);
|
||||
}
|
||||
|
||||
assertThat(healthy.passRate()).isEqualTo(1.0);
|
||||
assertThat(regressed.failedIds()).containsExactly("refund-annual", "sso-plan", "extra-seat");
|
||||
assertThatThrownBy(() -> regressed.requirePassRate(0.90)).isInstanceOf(AssertionError.class).hasMessageContaining("sso-plan");
|
||||
}
|
||||
|
||||
@Test
|
||||
void whichEvaluatorCaughtWhat() {
|
||||
EvalReport regressed = run(Answers.regressed());
|
||||
CaseResult hallucinated = regressed.results().stream().filter(r -> r.id().equals("refund-annual")).findFirst().orElseThrow();
|
||||
CaseResult offTopic = regressed.results().stream().filter(r -> r.id().equals("sso-plan")).findFirst().orElseThrow();
|
||||
CaseResult wrongNumber = regressed.results().stream().filter(r -> r.id().equals("extra-seat")).findFirst().orElseThrow();
|
||||
|
||||
assertThat(hallucinated.grounded()).isFalse();
|
||||
assertThat(offTopic.relevant()).isFalse();
|
||||
// A plausible wrong number slips past the judge (5 of 6 content words overlap) and is caught only by the exact-fact check.
|
||||
assertThat(wrongNumber.relevant()).isTrue();
|
||||
assertThat(wrongNumber.grounded()).isTrue();
|
||||
assertThat(wrongNumber.hasFacts()).isFalse();
|
||||
}
|
||||
|
||||
static List<GoldenCase> cases() {
|
||||
return GOLDEN;
|
||||
}
|
||||
|
||||
/** The per-case style: one JUnit invocation per golden row, readable in any test report. Only suitable for a deterministic judge. */
|
||||
@ParameterizedTest(name = "[{index}] {0}")
|
||||
@MethodSource("ids")
|
||||
void healthyBuildPassesEveryCase(String id) {
|
||||
GoldenCase c = GOLDEN.stream().filter(g -> g.id().equals(id)).findFirst().orElseThrow();
|
||||
RecordingModel app = Answers.appModel(Answers.good());
|
||||
String answer = new SupportAssistant(ChatClient.builder(app)).answer(c.question(), c.documents());
|
||||
EvaluationRequest request = new EvaluationRequest(c.question(), c.documents(), answer);
|
||||
RuleBasedJudge judge = new RuleBasedJudge(0.6);
|
||||
assertThat(new RelevancyEvaluator(ChatClient.builder(judge)).evaluate(request).isPass()).as("relevant").isTrue();
|
||||
assertThat(FactCheckingEvaluator.builder(ChatClient.builder(judge)).build().evaluate(request).isPass()).as("grounded").isTrue();
|
||||
assertThat(new ContainsFactsEvaluator(c.expectedFacts()).evaluate(request).isPass()).as("facts").isTrue();
|
||||
}
|
||||
|
||||
static List<String> ids() {
|
||||
return GOLDEN.stream().map(GoldenCase::id).toList();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package com.ankurm.evaluation;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.ankurm.evaluation.support.Answers;
|
||||
import com.ankurm.evaluation.support.RecordingModel;
|
||||
import com.ankurm.evaluation.support.Transcript;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.chat.evaluation.RelevancyEvaluator;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.evaluation.EvaluationRequest;
|
||||
import org.springframework.ai.evaluation.EvaluationResponse;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/** The two custom evaluators: a 1-5 grade (the built-ins only score 0 or 1) and a composite that restores feedback. Writes output/05. */
|
||||
class GradedCompositeTest {
|
||||
|
||||
private static final EvaluationRequest REQUEST = new EvaluationRequest("Q?", List.of(Document.builder().text("ctx").build()), "answer");
|
||||
|
||||
@Test
|
||||
void gradedEvaluatorReadsAGradeOutOfMessyReplies() {
|
||||
Map<String, Boolean> expected = new LinkedHashMap<>();
|
||||
expected.put("5", true);
|
||||
expected.put("4/5", true);
|
||||
expected.put("Score: 4 - mostly right, misses the date", true);
|
||||
expected.put("3", false);
|
||||
expected.put("I would say excellent", false);
|
||||
expected.put("10", false);
|
||||
|
||||
try (Transcript t = new Transcript("05-graded-and-composite.txt", "GradedEvaluator (minimum grade 4) and CompositeEvaluator")) {
|
||||
t.line("%-44s %-6s %-6s %s", "judge reply", "pass", "score", "feedback");
|
||||
for (var e : expected.entrySet()) {
|
||||
EvaluationResponse r = new GradedEvaluator(ChatClient.builder(RecordingModel.replying(e.getKey())), 4).evaluate(REQUEST);
|
||||
t.line("%-44s %-6s %-6s %s", "'" + e.getKey() + "'", r.isPass(), r.getScore(), r.getFeedback());
|
||||
assertThat(r.isPass()).as(e.getKey()).isEqualTo(e.getValue());
|
||||
}
|
||||
|
||||
t.blank();
|
||||
t.line("== CompositeEvaluator on the regressed 'extra-seat' answer (\"12 USD\" instead of \"8 USD\") ==");
|
||||
GoldenCase c = GoldenDatasetTest.GOLDEN.stream().filter(g -> g.id().equals("extra-seat")).findFirst().orElseThrow();
|
||||
String answer = Answers.regressed().get(c.question());
|
||||
EvaluationRequest request = new EvaluationRequest(c.question(), c.documents(), answer);
|
||||
RuleBasedJudge judge = new RuleBasedJudge(0.6);
|
||||
var composite = new CompositeEvaluator(new LinkedHashMap<>(Map.of()) {
|
||||
{
|
||||
put("relevancy", new RelevancyEvaluator(ChatClient.builder(judge)));
|
||||
put("contains-facts", new ContainsFactsEvaluator(c.expectedFacts()));
|
||||
}
|
||||
});
|
||||
EvaluationResponse r = composite.evaluate(request);
|
||||
t.line("answer : %s", answer);
|
||||
t.line("pass : %s", r.isPass());
|
||||
t.line("score : %s", r.getScore());
|
||||
t.line("feedback : %s", r.getFeedback());
|
||||
t.line("verdicts : %s", r.getMetadata());
|
||||
assertThat(r.isPass()).isFalse();
|
||||
assertThat(r.getFeedback()).contains("contains-facts failed").contains("8 USD");
|
||||
assertThat(r.getMetadata()).containsEntry("relevancy", true).containsEntry("contains-facts", false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package com.ankurm.evaluation;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import com.ankurm.evaluation.support.Transcript;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.chat.evaluation.RelevancyEvaluator;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.evaluation.EvaluationRequest;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/** What the cheap deterministic judge gets wrong. This is the reason a real judge still runs somewhere (nightly). Writes output/06. */
|
||||
class JudgeLimitsTest {
|
||||
|
||||
private record Probe(String label, String answer, boolean actuallyCorrect) {
|
||||
}
|
||||
|
||||
@Test
|
||||
void theRuleBasedJudgeIsFooledInBothDirections() {
|
||||
List<Document> context = List.of(Document.builder().text("Annual plans can be refunded within 30 days of purchase. Monthly plans are not refundable.").build());
|
||||
String question = "Can I get a refund on a monthly plan?";
|
||||
List<Probe> probes = List.of(
|
||||
new Probe("correct, same words", "Monthly plans are not refundable.", true),
|
||||
new Probe("WRONG: negation dropped", "Monthly plans are refundable.", false),
|
||||
new Probe("WRONG: roles swapped", "Annual plans are not refundable. Monthly plans can be refunded within 30 days of purchase.", false),
|
||||
new Probe("correct, paraphrased", "No, you cannot get your money back on month-to-month subscriptions.", true),
|
||||
new Probe("correct, but a refusal", "I do not have that information.", true));
|
||||
|
||||
try (Transcript t = new Transcript("06-judge-limits.txt", "RuleBasedJudge(0.6) on hand-labelled answers to: " + question)) {
|
||||
t.line("%-26s %-9s %-14s %s", "answer type", "correct?", "judge says", "verdict");
|
||||
for (Probe p : probes) {
|
||||
boolean judged = new RelevancyEvaluator(ChatClient.builder(new RuleBasedJudge(0.6)))
|
||||
.evaluate(new EvaluationRequest(question, context, p.answer())).isPass();
|
||||
t.line("%-26s %-9s %-14s %s", p.label(), p.actuallyCorrect(), judged ? "pass" : "fail", judged == p.actuallyCorrect() ? "right" : "WRONG");
|
||||
}
|
||||
}
|
||||
|
||||
boolean negation = new RelevancyEvaluator(ChatClient.builder(new RuleBasedJudge(0.6))).evaluate(new EvaluationRequest(question, context, "Monthly plans are refundable.")).isPass();
|
||||
boolean paraphrase = new RelevancyEvaluator(ChatClient.builder(new RuleBasedJudge(0.6))).evaluate(new EvaluationRequest(question, context, "No, you cannot get your money back on month-to-month subscriptions.")).isPass();
|
||||
assertThat(negation).as("a dropped negation shares every content word, so the judge accepts it").isTrue();
|
||||
assertThat(paraphrase).as("a correct paraphrase shares almost no words, so the judge rejects it").isFalse();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package com.ankurm.evaluation;
|
||||
|
||||
import com.ankurm.evaluation.support.Answers;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* The nightly layer: the same golden dataset and the same evaluators, judged by a real model.
|
||||
* Skipped unless {@code OPENAI_API_KEY} is set, so a pull-request build never spends money. It was
|
||||
* compiled and its skip was recorded for this article, but it was NOT executed against a live
|
||||
* model, so no live-judge result appears anywhere in the post.
|
||||
*/
|
||||
@Tag("live")
|
||||
@EnabledIfEnvironmentVariable(named = "OPENAI_API_KEY", matches = ".+")
|
||||
@SpringBootTest(classes = LiveJudgeTest.App.class)
|
||||
class LiveJudgeTest {
|
||||
|
||||
@SpringBootApplication
|
||||
static class App {
|
||||
}
|
||||
|
||||
@Autowired
|
||||
ChatModel model;
|
||||
|
||||
@Test
|
||||
void realJudgeAgreesWithTheKnownGoodAndKnownBadBuilds() {
|
||||
var cases = GoldenDataset.load("/golden/support-golden.json");
|
||||
var healthy = new EvalRunner(new SupportAssistant(ChatClient.builder(Answers.appModel(Answers.good()))), model).run(cases);
|
||||
var regressed = new EvalRunner(new SupportAssistant(ChatClient.builder(Answers.appModel(Answers.regressed()))), model).run(cases);
|
||||
assertThat(healthy.passRate()).isGreaterThanOrEqualTo(0.85);
|
||||
assertThat(regressed.passRate()).isLessThan(healthy.passRate());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package com.ankurm.evaluation;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.ankurm.evaluation.support.Answers;
|
||||
import com.ankurm.evaluation.support.FlakyJudge;
|
||||
import com.ankurm.evaluation.support.Transcript;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* What a noisy judge does to a test suite. The noise is SIMULATED (a correct judge whose verdict
|
||||
* is flipped 3 or 8 percent of the time, seeded), so the numbers show the effect of noise on the gating
|
||||
* rule and say nothing about how often any real model is wrong. Writes output/04.
|
||||
*/
|
||||
class NoisyJudgeTest {
|
||||
|
||||
static final List<GoldenCase> GOLDEN = GoldenDataset.load("/golden/support-golden.json");
|
||||
|
||||
static final int RUNS = 200;
|
||||
|
||||
private static double passRate(Map<String, String> answers, double flip, long seed) {
|
||||
SupportAssistant app = new SupportAssistant(ChatClient.builder(Answers.appModel(answers)));
|
||||
return new EvalRunner(app, new FlakyJudge(new RuleBasedJudge(0.6), flip, seed)).run(GOLDEN).passRate();
|
||||
}
|
||||
|
||||
private static long below(double[] rates, double threshold) {
|
||||
return Arrays.stream(rates).filter(r -> r < threshold).count();
|
||||
}
|
||||
|
||||
@Test
|
||||
void strictPerCaseAssertionsVersusAnAggregateThreshold() {
|
||||
double[] flips = { 0.03, 0.08 };
|
||||
double[][] healthy = new double[flips.length][RUNS];
|
||||
double[][] regressed = new double[flips.length][RUNS];
|
||||
for (int f = 0; f < flips.length; f++) {
|
||||
for (int i = 0; i < RUNS; i++) {
|
||||
healthy[f][i] = passRate(Answers.good(), flips[f], i);
|
||||
regressed[f][i] = passRate(Answers.regressed(), flips[f], 10_000 + i);
|
||||
}
|
||||
}
|
||||
|
||||
try (Transcript t = new Transcript("04-noisy-judge.txt", "Simulated judge noise: %d runs of a %d-case suite per build".formatted(RUNS, GOLDEN.size()))) {
|
||||
t.line("noise-free pass rate: healthy build 1.00, regressed build 0.75");
|
||||
for (int f = 0; f < flips.length; f++) {
|
||||
t.blank();
|
||||
t.line("== %.0f%% of judge verdicts flipped (mean pass rate: healthy %.3f, regressed %.3f) ==", flips[f] * 100,
|
||||
Arrays.stream(healthy[f]).average().orElse(0), Arrays.stream(regressed[f]).average().orElse(0));
|
||||
t.line("%-22s %-28s %s", "gate", "healthy build fails it", "regressed build fails it");
|
||||
for (double threshold : new double[] { 1.0, 0.90, 0.85, 0.80 }) {
|
||||
String name = threshold == 1.0 ? "every case must pass" : "pass rate >= %.2f".formatted(threshold);
|
||||
t.line("%-22s %-28s %s", name, "%d of %d runs".formatted(below(healthy[f], threshold), RUNS), "%d of %d runs".formatted(below(regressed[f], threshold), RUNS));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// At 3 percent noise a 0.80 gate has few false alarms and still catches the regression; "every case must pass" does not.
|
||||
assertThat(below(healthy[0], 1.0)).isGreaterThan(below(healthy[0], 0.80));
|
||||
assertThat(below(regressed[0], 0.80)).isGreaterThan(below(healthy[0], 0.80));
|
||||
// At 8 percent noise even 0.80 raises false alarms: the gate has to be set from measured noise, not guessed.
|
||||
assertThat(below(healthy[1], 0.80)).isGreaterThan(below(healthy[0], 0.80));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package com.ankurm.evaluation;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.ankurm.evaluation.support.RecordingModel;
|
||||
import com.ankurm.evaluation.support.Transcript;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
import org.springframework.ai.chat.evaluation.FactCheckingEvaluator;
|
||||
import org.springframework.ai.chat.evaluation.RelevancyEvaluator;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.document.Document;
|
||||
import org.springframework.ai.evaluation.EvaluationRequest;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/** The built-in evaluators pass only on a bare "yes". This shows which judge replies count, and the normalising wrapper that fixes it. Writes output/02. */
|
||||
class VerdictParsingTest {
|
||||
|
||||
private static final EvaluationRequest REQUEST = new EvaluationRequest("Q?", List.of(Document.builder().text("ctx").build()), "answer");
|
||||
|
||||
private static boolean relevancy(ChatModel judge) {
|
||||
return new RelevancyEvaluator(ChatClient.builder(judge)).evaluate(REQUEST).isPass();
|
||||
}
|
||||
|
||||
private static boolean fact(ChatModel judge) {
|
||||
return FactCheckingEvaluator.builder(ChatClient.builder(judge)).build().evaluate(REQUEST).isPass();
|
||||
}
|
||||
|
||||
@Test
|
||||
void onlyABareYesPasses() {
|
||||
Map<String, Boolean> expectedRaw = new LinkedHashMap<>();
|
||||
expectedRaw.put("yes", true);
|
||||
expectedRaw.put("YES", true);
|
||||
expectedRaw.put(" yes\n", true);
|
||||
expectedRaw.put("Yes.", false);
|
||||
expectedRaw.put("yes!", false);
|
||||
expectedRaw.put("Yes, the response is in line with the context.", false);
|
||||
expectedRaw.put("**Yes**", false);
|
||||
expectedRaw.put("no", false);
|
||||
expectedRaw.put("No.", false);
|
||||
expectedRaw.put("", false);
|
||||
|
||||
try (Transcript t = new Transcript("02-verdict-parsing.txt", "Which judge replies the built-in evaluators accept")) {
|
||||
t.line("%-52s %-9s %-9s %-9s", "judge reply", "relevancy", "fact-chk", "wrapped");
|
||||
for (var e : expectedRaw.entrySet()) {
|
||||
ChatModel raw = RecordingModel.replying(e.getKey());
|
||||
boolean r = relevancy(raw);
|
||||
boolean f = fact(raw);
|
||||
boolean wrapped = relevancy(new VerdictNormalizingModel(raw));
|
||||
t.line("%-52s %-9s %-9s %-9s", "'" + e.getKey().replace("\n", "\\n") + "'", r, f, wrapped);
|
||||
assertThat(r).as("relevancy for '%s'", e.getKey()).isEqualTo(e.getValue());
|
||||
assertThat(f).as("fact for '%s'", e.getKey()).isEqualTo(e.getValue());
|
||||
}
|
||||
// The wrapper turns the punctuated, bold and sentence-style verdicts into passes, and leaves "no" alone.
|
||||
assertThat(relevancy(new VerdictNormalizingModel(RecordingModel.replying("Yes.")))).isTrue();
|
||||
assertThat(relevancy(new VerdictNormalizingModel(RecordingModel.replying("Yes, the response is in line with the context.")))).isTrue();
|
||||
assertThat(relevancy(new VerdictNormalizingModel(RecordingModel.replying("No.")))).isFalse();
|
||||
assertThat(relevancy(new VerdictNormalizingModel(RecordingModel.replying("**Yes**")))).isTrue();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package com.ankurm.evaluation.support;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
|
||||
/**
|
||||
* What the pretend application model says for each golden question. {@link #good()} is a healthy
|
||||
* build; {@link #regressed()} is the same build after a bad prompt change: one hallucinated
|
||||
* answer, one off-topic answer, and one with a plausible but wrong number.
|
||||
*/
|
||||
public final class Answers {
|
||||
|
||||
private Answers() {
|
||||
}
|
||||
|
||||
public static Map<String, String> good() {
|
||||
Map<String, String> m = new HashMap<>();
|
||||
m.put("How long do I have to request a refund on an annual plan?", "You can request a refund on an annual plan within 30 days of purchase.");
|
||||
m.put("Can I get a refund on a monthly plan?", "No. Monthly plans are not refundable.");
|
||||
m.put("How much storage does the Pro plan include?", "The Pro plan includes 200 GB of storage.");
|
||||
m.put("When is support open?", "Support is available Monday to Friday, 9:00 to 17:00 UTC.");
|
||||
m.put("Which plan has SAML single sign-on?", "SAML single sign-on is available on the Enterprise plan only.");
|
||||
m.put("Can I move my data to the US after signing up?", "No. The region can only be chosen at account creation and cannot be changed later.");
|
||||
m.put("What happens if I exceed the API rate limit?", "Exceeding 600 requests per minute returns HTTP 429.");
|
||||
m.put("How long are Pro backups kept?", "Pro backups are kept for 14 days.");
|
||||
m.put("Is there a macOS agent?", "I do not know. Acme Cloud supports Linux and Windows agents.");
|
||||
m.put("How long is a password reset link valid?", "A password reset link expires after 60 minutes.");
|
||||
m.put("How much does an extra seat cost?", "Extra seats cost 8 USD per month.");
|
||||
m.put("Do I need a credit card for the trial?", "No. The free trial does not require a credit card.");
|
||||
return m;
|
||||
}
|
||||
|
||||
public static Map<String, String> regressed() {
|
||||
Map<String, String> m = good();
|
||||
m.put("How long do I have to request a refund on an annual plan?", "You can request a refund within 60 days, including on monthly plans.");
|
||||
m.put("Which plan has SAML single sign-on?", "Our team is happy to help with any sign-in question you have.");
|
||||
m.put("How much does an extra seat cost?", "Extra seats cost 12 USD per month.");
|
||||
return m;
|
||||
}
|
||||
|
||||
/** The scripted application model: finds the golden question in the user message and returns its answer. */
|
||||
public static RecordingModel appModel(Map<String, String> answers) {
|
||||
return new RecordingModel((Prompt p) -> {
|
||||
String text = p.getInstructions().getLast().getText();
|
||||
return answers.entrySet().stream()
|
||||
.filter(e -> text.contains("Question: " + e.getKey()))
|
||||
.map(Map.Entry::getValue)
|
||||
.findFirst()
|
||||
.orElse("I do not know.");
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package com.ankurm.evaluation.support;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Random;
|
||||
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
|
||||
/**
|
||||
* A SIMULATION of judge noise, not a measurement of any real model: it forwards to a correct
|
||||
* judge, then with a fixed probability returns the opposite verdict. The random source is seeded,
|
||||
* so runs are repeatable. The point is to show what noise does to a test suite, not to claim a rate.
|
||||
*/
|
||||
public class FlakyJudge implements ChatModel {
|
||||
|
||||
private final ChatModel delegate;
|
||||
|
||||
private final double flipRate;
|
||||
|
||||
private final Random random;
|
||||
|
||||
public FlakyJudge(ChatModel delegate, double flipRate, long seed) {
|
||||
this.delegate = delegate;
|
||||
this.flipRate = flipRate;
|
||||
this.random = new Random(seed);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChatResponse call(Prompt prompt) {
|
||||
String verdict = delegate.call(prompt).getResult().getOutput().getText();
|
||||
if (random.nextDouble() < flipRate) {
|
||||
verdict = verdict.equals("yes") ? "no" : "yes";
|
||||
}
|
||||
return new ChatResponse(List.of(new Generation(new AssistantMessage(verdict))));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package com.ankurm.evaluation.support;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import java.util.function.Function;
|
||||
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
|
||||
/** A scripted {@link ChatModel}: no network, no key. It records every prompt and answers with whatever the replier returns. */
|
||||
public class RecordingModel implements ChatModel {
|
||||
|
||||
private final List<Prompt> prompts = new CopyOnWriteArrayList<>();
|
||||
|
||||
private final Function<Prompt, String> replier;
|
||||
|
||||
public RecordingModel(Function<Prompt, String> replier) {
|
||||
this.replier = replier;
|
||||
}
|
||||
|
||||
public static RecordingModel replying(String fixed) {
|
||||
return new RecordingModel(p -> fixed);
|
||||
}
|
||||
|
||||
public List<Prompt> prompts() {
|
||||
return prompts;
|
||||
}
|
||||
|
||||
public int callCount() {
|
||||
return prompts.size();
|
||||
}
|
||||
|
||||
public String lastUserText() {
|
||||
return prompts.getLast().getInstructions().getLast().getText();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChatResponse call(Prompt prompt) {
|
||||
prompts.add(prompt);
|
||||
return new ChatResponse(List.of(new Generation(new AssistantMessage(replier.apply(prompt)))));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package com.ankurm.evaluation.support;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.StringWriter;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
/**
|
||||
* Writes a numbered transcript under {@code output/} (repository root, not {@code docs/}) and
|
||||
* echoes it to the console. Every console block quoted in the article comes out of one of these
|
||||
* files verbatim.
|
||||
*/
|
||||
public final class Transcript implements AutoCloseable {
|
||||
|
||||
private final Path path;
|
||||
private final StringWriter buffer = new StringWriter();
|
||||
private final PrintWriter out = new PrintWriter(buffer);
|
||||
|
||||
public Transcript(String fileName, String title) {
|
||||
this.path = Path.of("output", fileName);
|
||||
out.println("# " + title);
|
||||
out.println();
|
||||
}
|
||||
|
||||
public Transcript line(String format, Object... args) {
|
||||
out.println(args.length == 0 ? format : String.format(format, args));
|
||||
return this;
|
||||
}
|
||||
|
||||
public Transcript blank() {
|
||||
out.println();
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
out.flush();
|
||||
try {
|
||||
Files.createDirectories(path.getParent());
|
||||
Files.writeString(path, buffer.toString());
|
||||
} catch (IOException e) {
|
||||
throw new IllegalStateException("could not write " + path, e);
|
||||
}
|
||||
System.out.print(buffer);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
[
|
||||
{"id": "refund-annual", "question": "How long do I have to request a refund on an annual plan?",
|
||||
"context": ["Annual plans can be refunded within 30 days of purchase. Monthly plans are not refundable."], "expectedFacts": ["30 days"]},
|
||||
{"id": "refund-monthly", "question": "Can I get a refund on a monthly plan?",
|
||||
"context": ["Annual plans can be refunded within 30 days of purchase. Monthly plans are not refundable."], "expectedFacts": ["not refundable"]},
|
||||
{"id": "storage-pro", "question": "How much storage does the Pro plan include?",
|
||||
"context": ["The Free plan includes 5 GB of storage. The Pro plan includes 200 GB of storage."], "expectedFacts": ["200 GB"]},
|
||||
{"id": "support-hours", "question": "When is support open?",
|
||||
"context": ["Support is available Monday to Friday, 9:00 to 17:00 UTC. Enterprise customers have 24/7 support."], "expectedFacts": ["Monday to Friday"]},
|
||||
{"id": "sso-plan", "question": "Which plan has SAML single sign-on?",
|
||||
"context": ["Single sign-on with SAML is available on the Enterprise plan only."], "expectedFacts": ["Enterprise"]},
|
||||
{"id": "data-region", "question": "Can I move my data to the US after signing up?",
|
||||
"context": ["Data is stored in the EU region by default. US storage can be selected at account creation and cannot be changed later."], "expectedFacts": ["cannot be changed"]},
|
||||
{"id": "api-rate-limit", "question": "What happens if I exceed the API rate limit?",
|
||||
"context": ["The API allows 600 requests per minute per key. Exceeding the limit returns HTTP 429."], "expectedFacts": ["429"]},
|
||||
{"id": "backup-retention", "question": "How long are Pro backups kept?",
|
||||
"context": ["Backups are kept for 14 days on the Pro plan and 35 days on the Enterprise plan."], "expectedFacts": ["14 days"]},
|
||||
{"id": "macos-agent", "question": "Is there a macOS agent?",
|
||||
"context": ["Acme Cloud supports Linux and Windows agents."], "expectedFacts": ["do not know"]},
|
||||
{"id": "reset-link", "question": "How long is a password reset link valid?",
|
||||
"context": ["Password reset links expire after 60 minutes."], "expectedFacts": ["60 minutes"]},
|
||||
{"id": "extra-seat", "question": "How much does an extra seat cost?",
|
||||
"context": ["Each Pro plan includes 5 seats. Extra seats cost 8 USD per month."], "expectedFacts": ["8 USD"]},
|
||||
{"id": "trial-card", "question": "Do I need a credit card for the trial?",
|
||||
"context": ["The free trial lasts 14 days and does not require a credit card."], "expectedFacts": ["does not require"]}
|
||||
]
|
||||
Reference in New Issue
Block a user