New posts
This commit is contained in:
@@ -0,0 +1,73 @@
|
||||
package com.ankurm.agentic;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/** Runs every happy path with 100 ms of fake latency per call and tabulates what each pattern really did. */
|
||||
class CostTest {
|
||||
|
||||
record Row(String name, ScriptedChatModel model) {}
|
||||
|
||||
@Test
|
||||
void callsStagesAndCharactersPerPattern() throws Exception {
|
||||
List<Row> rows = new ArrayList<>();
|
||||
|
||||
var single = new ScriptedChatModel().latency(100).whenNoSystem("one answer");
|
||||
Scenarios.client(single).prompt().user(Scenarios.GIT_LOG).call().content();
|
||||
rows.add(new Row("single call (baseline)", single));
|
||||
|
||||
var chain = Scenarios.chain().latency(100);
|
||||
new PromptChain(Scenarios.client(chain)).run(Scenarios.GIT_LOG);
|
||||
rows.add(new Row("prompt chain", chain));
|
||||
|
||||
var router = Scenarios.router("TECHNICAL").latency(100);
|
||||
new Router(Scenarios.client(router)).handle("The connector returns 401 since Monday");
|
||||
rows.add(new Row("routing", router));
|
||||
|
||||
var sectioning = Scenarios.sectioning().latency(100);
|
||||
new Parallelizer(Scenarios.client(sectioning)).sectioning(Scenarios.CODE);
|
||||
rows.add(new Row("sectioning x3", sectioning));
|
||||
|
||||
var voting = Scenarios.voting("SAFE", "SAFE", "UNSAFE").latency(100);
|
||||
new Parallelizer(Scenarios.client(voting)).vote("Free crypto, click here", 3);
|
||||
rows.add(new Row("voting x3", voting));
|
||||
|
||||
var orch = Scenarios.orchestrator().latency(100);
|
||||
new Orchestrator(Scenarios.client(orch)).run(Scenarios.TASK);
|
||||
rows.add(new Row("orchestrator, 3 steps", orch));
|
||||
|
||||
var evalOk = Scenarios.evaluatorConverging().latency(100);
|
||||
new EvaluatorOptimizer(Scenarios.client(evalOk)).run(Scenarios.TASK, 5);
|
||||
rows.add(new Row("evaluator, passes in round 3", evalOk));
|
||||
|
||||
var evalStuck = Scenarios.evaluatorStuck().latency(100);
|
||||
new EvaluatorOptimizer(Scenarios.client(evalStuck)).run(Scenarios.TASK, 3);
|
||||
rows.add(new Row("evaluator, hits the 3-round cap", evalStuck));
|
||||
|
||||
var t = new Transcript("12-cost-latency.txt");
|
||||
t.line("# calls, sequential stages and prompt characters per pattern (scripted model, 100 ms fake latency per call)");
|
||||
t.line("%-34s %5s %7s %9s %12s".formatted("pattern", "calls", "stages", "in flight", "prompt chars"));
|
||||
for (Row r : rows) {
|
||||
t.line("%-34s %5d %7d %9d %12d".formatted(r.name(), r.model().calls().size(), r.model().stages(),
|
||||
r.model().maxInFlight(), r.model().totalChars()));
|
||||
}
|
||||
t.write();
|
||||
|
||||
assertThat(rows.get(0).model().stages()).isEqualTo(1);
|
||||
assertThat(rows.get(1).model().calls()).hasSize(2);
|
||||
assertThat(rows.get(1).model().stages()).isEqualTo(2);
|
||||
assertThat(rows.get(2).model().stages()).isEqualTo(2);
|
||||
assertThat(rows.get(3).model().calls()).hasSize(3);
|
||||
assertThat(rows.get(3).model().stages()).isEqualTo(1);
|
||||
assertThat(rows.get(4).model().stages()).isEqualTo(1);
|
||||
assertThat(rows.get(5).model().calls()).hasSize(5);
|
||||
assertThat(rows.get(5).model().stages()).isEqualTo(5);
|
||||
assertThat(rows.get(6).model().calls()).hasSize(6);
|
||||
assertThat(rows.get(6).model().stages()).isEqualTo(6);
|
||||
assertThat(rows.get(7).model().calls()).hasSize(6);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package com.ankurm.agentic;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class EvaluatorOptimizerTest {
|
||||
|
||||
@Test
|
||||
void loopStopsOnTheFirstPass() throws Exception {
|
||||
var model = Scenarios.evaluatorConverging();
|
||||
var result = new EvaluatorOptimizer(Scenarios.client(model)).run(Scenarios.TASK, 5);
|
||||
|
||||
var calls = model.calls();
|
||||
assertThat(calls).hasSize(6);
|
||||
assertThat(calls).extracting(c -> c.system).containsExactly(
|
||||
EvaluatorOptimizer.GENERATOR, EvaluatorOptimizer.EVALUATOR,
|
||||
EvaluatorOptimizer.GENERATOR, EvaluatorOptimizer.EVALUATOR,
|
||||
EvaluatorOptimizer.GENERATOR, EvaluatorOptimizer.EVALUATOR);
|
||||
assertThat(calls.get(0).user).isEqualTo(Scenarios.TASK);
|
||||
assertThat(calls.get(2).user).isEqualTo(Scenarios.TASK
|
||||
+ "\nPrevious draft: Added a --json flag to the export command\nFeedback: Use the imperative mood.\nRevise the draft.");
|
||||
assertThat(calls.get(3).user).isEqualTo("Add a --json flag to the export command.");
|
||||
assertThat(result.converged()).isTrue();
|
||||
assertThat(result.rounds()).isEqualTo(3);
|
||||
assertThat(result.best()).isEqualTo("Add --json output flag");
|
||||
|
||||
var t = new Transcript("10-evaluator-converges.txt");
|
||||
t.line("# evaluator-optimizer: three rounds, PASS on the third");
|
||||
t.calls(model);
|
||||
t.line("converged: " + result.converged() + ", rounds: " + result.rounds() + ", calls: " + calls.size());
|
||||
t.line("best: " + result.best());
|
||||
t.line("feedback: " + result.feedback());
|
||||
t.write();
|
||||
}
|
||||
|
||||
@Test
|
||||
void aJudgeThatNeverPassesIsStoppedByTheRoundCap() throws Exception {
|
||||
var model = Scenarios.evaluatorStuck();
|
||||
var result = new EvaluatorOptimizer(Scenarios.client(model)).run(Scenarios.TASK, 3);
|
||||
|
||||
assertThat(model.calls()).hasSize(6); // the cap, not the judge, ended the loop
|
||||
assertThat(result.converged()).isFalse();
|
||||
assertThat(result.rounds()).isEqualTo(3);
|
||||
assertThat(result.feedback()).containsExactly("Longer than 50 characters.", "Looks good to me!", "Still longer than 50 characters.");
|
||||
assertThat(result.best()).isEqualTo("Add a --json output flag to the export command, CLI");
|
||||
assertThat(result.drafts()).allSatisfy(d -> assertThat(d.length()).isGreaterThan(50));
|
||||
|
||||
var t = new Transcript("11-evaluator-max-rounds.txt");
|
||||
t.line("# evaluator-optimizer: the judge never says PASS, maxRounds = 3");
|
||||
t.calls(model);
|
||||
t.line("converged: " + result.converged() + ", rounds: " + result.rounds() + ", calls: " + model.calls().size());
|
||||
t.line("best so far (returned, flagged): " + result.best() + " (" + result.best().length() + " chars)");
|
||||
t.line("feedback: " + result.feedback());
|
||||
t.write();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package com.ankurm.agentic;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class OrchestratorTest {
|
||||
|
||||
@Test
|
||||
void plannerDecidesTheStepsAndEachWorkerGetsOne() throws Exception {
|
||||
var model = Scenarios.orchestrator();
|
||||
var result = new Orchestrator(Scenarios.client(model)).run(Scenarios.TASK);
|
||||
|
||||
var calls = model.calls();
|
||||
assertThat(calls).hasSize(5); // 1 plan + 3 workers + 1 synthesis
|
||||
assertThat(calls.get(0).system).isEqualTo(Orchestrator.PLANNER);
|
||||
assertThat(calls.get(0).user).startsWith(Scenarios.TASK);
|
||||
assertThat(calls.get(0).user).contains("JSON"); // entity() appended the format instructions
|
||||
assertThat(calls.get(1).user).isEqualTo("Task: " + Scenarios.TASK
|
||||
+ "\nStep: Parse the flag\nInstruction: Register --json in the argument parser.");
|
||||
assertThat(calls.get(2).user).contains("Step: Serialise rows");
|
||||
assertThat(calls.get(3).user).contains("Step: Document it");
|
||||
assertThat(calls.get(4).system).isEqualTo(Orchestrator.SYNTHESISER);
|
||||
assertThat(calls.get(4).user).contains("## Parse the flag\nAdd a boolean option --json to ExportArgs.");
|
||||
assertThat(result.fellBack()).isFalse();
|
||||
assertThat(result.plan().steps()).hasSize(3);
|
||||
|
||||
var t = new Transcript("08-orchestrator-workers.txt");
|
||||
t.line("# orchestrator-workers: plan at run time, one worker per step, one synthesis");
|
||||
t.calls(model);
|
||||
t.line("plan steps: " + result.plan().steps().stream().map(Orchestrator.Step::title).toList());
|
||||
t.line("calls: " + calls.size() + " (1 plan + " + result.stepResults().size() + " workers + 1 synthesis)");
|
||||
t.line("answer: " + result.answer());
|
||||
t.write();
|
||||
|
||||
var p = new Transcript("13-planner-prompt-as-sent.txt");
|
||||
p.line("# system message the planner call carried");
|
||||
calls.get(0).system.lines().forEach(p::line);
|
||||
p.line("# user message the planner call carried: the task, then what entity(Plan.class) appended");
|
||||
calls.get(0).user.lines().forEach(p::line);
|
||||
p.write();
|
||||
}
|
||||
|
||||
@Test
|
||||
void aMalformedPlanIsRetriedOnceThenAnsweredDirectly() throws Exception {
|
||||
var t = new Transcript("09-orchestrator-malformed-plan.txt");
|
||||
|
||||
// (a) prose instead of JSON, then a good plan on the retry
|
||||
var a = new ScriptedChatModel()
|
||||
.when("You are a planner", "Sure! First I would parse the flag, then serialise the rows.", Scenarios.PLAN_JSON)
|
||||
.when("carry out one step", "step result")
|
||||
.when("Combine the step results", "final answer");
|
||||
var ra = new Orchestrator(Scenarios.client(a)).run(Scenarios.TASK);
|
||||
assertThat(ra.planAttempts()).isEqualTo(2);
|
||||
assertThat(ra.fellBack()).isFalse();
|
||||
assertThat(a.calls()).hasSize(6); // 2 plan + 3 workers + 1 synthesis
|
||||
assertThat(a.calls().get(1).user).contains("Your previous reply could not be used (" + ra.planErrors().get(0) + ")");
|
||||
t.line("# (a) prose first, valid JSON on the retry");
|
||||
t.calls(a);
|
||||
t.line("plan errors: " + ra.planErrors() + ", attempts: " + ra.planAttempts() + ", calls: " + a.calls().size());
|
||||
|
||||
// (b) valid JSON but seven steps: the plan validator rejects it, twice, then the direct fallback runs
|
||||
String seven = "{\"steps\":[" + "{\"title\":\"s\",\"instruction\":\"i\"},".repeat(6) + "{\"title\":\"s\",\"instruction\":\"i\"}]}";
|
||||
var b = new ScriptedChatModel()
|
||||
.when("You are a planner", seven)
|
||||
.whenNoSystem("Just do it: one direct answer.");
|
||||
var rb = new Orchestrator(Scenarios.client(b)).run(Scenarios.TASK);
|
||||
assertThat(rb.fellBack()).isTrue();
|
||||
assertThat(b.calls()).hasSize(3);
|
||||
assertThat(b.calls().get(2).system).isEmpty();
|
||||
assertThat(b.calls().get(2).user).isEqualTo(Scenarios.TASK);
|
||||
t.line("# (b) seven steps twice: over the limit of " + Orchestrator.MAX_STEPS);
|
||||
t.calls(b);
|
||||
t.line("plan errors: " + rb.planErrors() + ", fell back: " + rb.fellBack() + ", answer: " + rb.answer());
|
||||
|
||||
// (c) prose both times
|
||||
var c = new ScriptedChatModel()
|
||||
.when("You are a planner", "I cannot produce JSON today.")
|
||||
.whenNoSystem("Just do it: one direct answer.");
|
||||
var rc = new Orchestrator(Scenarios.client(c)).run(Scenarios.TASK);
|
||||
assertThat(rc.fellBack()).isTrue();
|
||||
assertThat(c.calls()).hasSize(3);
|
||||
t.line("# (c) prose twice");
|
||||
t.calls(c);
|
||||
t.line("plan errors: " + rc.planErrors() + ", fell back: " + rc.fellBack());
|
||||
t.write();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package com.ankurm.agentic;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class ParallelizationTest {
|
||||
|
||||
@Test
|
||||
void sectioningRunsThreeReviewsAtOnceAndKeepsAspectOrder() throws Exception {
|
||||
var model = Scenarios.sectioning().rendezvousOf(3); // times out if the calls ran one after another
|
||||
var sections = new Parallelizer(Scenarios.client(model)).sectioning(Scenarios.CODE);
|
||||
|
||||
assertThat(model.maxInFlight()).isEqualTo(3);
|
||||
assertThat(model.calls()).extracting(c -> c.system)
|
||||
.containsExactlyInAnyOrder(Parallelizer.aspectPrompt("security"),
|
||||
Parallelizer.aspectPrompt("performance"), Parallelizer.aspectPrompt("readability"));
|
||||
assertThat(model.calls()).allSatisfy(c -> assertThat(c.user).isEqualTo(Scenarios.CODE));
|
||||
assertThat(sections).extracting(Parallelizer.Section::aspect).containsExactly("security", "performance", "readability");
|
||||
|
||||
var t = new Transcript("05-sectioning.txt");
|
||||
t.line("# sectioning: three prompts, same input, merged in plain Java");
|
||||
t.callsUnordered(model);
|
||||
t.line("calls: " + model.calls().size() + ", most calls in flight at once: " + model.maxInFlight());
|
||||
sections.forEach(s -> t.line(s.aspect() + ": " + s.text()));
|
||||
t.write();
|
||||
}
|
||||
|
||||
@Test
|
||||
void aFailedBranchIsMarkedAndTheOthersSurvive() throws Exception {
|
||||
var model = Scenarios.sectioning();
|
||||
model = new ScriptedChatModel()
|
||||
.when("for security only", "SQL is built by string concatenation, which allows injection.")
|
||||
.when("for performance only", new IllegalStateException("429 rate limited"))
|
||||
.when("for readability only", "The query string should be a named constant.");
|
||||
var sections = new Parallelizer(Scenarios.client(model)).sectioning(Scenarios.CODE);
|
||||
|
||||
assertThat(sections).extracting(Parallelizer.Section::failed).containsExactly(false, true, false);
|
||||
|
||||
var t = new Transcript("06-sectioning-branch-failure.txt");
|
||||
t.line("# sectioning: the performance branch throws");
|
||||
t.callsUnordered(model);
|
||||
sections.forEach(s -> t.line(s.aspect() + (s.failed() ? " [FAILED] " : " ") + s.text()));
|
||||
t.write();
|
||||
}
|
||||
|
||||
@Test
|
||||
void votingCountsBallotsAndATieIsUnsafe() throws Exception {
|
||||
var t = new Transcript("07-voting.txt");
|
||||
t.line("# voting: the same prompt three times, majority wins, ties are UNSAFE");
|
||||
List<Object[]> cases = List.of(
|
||||
new Object[]{"SAFE", "UNSAFE", "UNSAFE"},
|
||||
new Object[]{"SAFE", "SAFE", "Probably fine!"},
|
||||
new Object[]{"SAFE", "UNSAFE", "Hmm"});
|
||||
List<String> expected = List.of("UNSAFE", "SAFE", "UNSAFE");
|
||||
for (int i = 0; i < cases.size(); i++) {
|
||||
var model = Scenarios.voting(cases.get(i)).rendezvousOf(3);
|
||||
var verdict = new Parallelizer(Scenarios.client(model)).vote("Free crypto, click here", 3);
|
||||
|
||||
assertThat(model.maxInFlight()).isEqualTo(3);
|
||||
assertThat(model.calls()).hasSize(3).allSatisfy(c -> {
|
||||
assertThat(c.system).isEqualTo(Parallelizer.VOTE);
|
||||
assertThat(c.user).isEqualTo("Free crypto, click here");
|
||||
});
|
||||
assertThat(verdict.verdict()).isEqualTo(expected.get(i));
|
||||
t.line("ballots %s -> %s (safe=%d, unsafe=%d, spoiled=%d)".formatted(
|
||||
String.join(" | ", java.util.Arrays.stream(cases.get(i)).map(Object::toString).toList()),
|
||||
verdict.verdict(), verdict.safe(), verdict.unsafe(), verdict.spoiled()));
|
||||
}
|
||||
t.line("calls per vote: 3, run at once");
|
||||
t.write();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package com.ankurm.agentic;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
class PromptChainTest {
|
||||
|
||||
@Test
|
||||
void chainRunsTwoCallsAndFeedsTheFirstAnswerIntoTheSecond() throws Exception {
|
||||
var model = Scenarios.chain();
|
||||
var result = new PromptChain(Scenarios.client(model)).run(Scenarios.GIT_LOG);
|
||||
|
||||
var calls = model.calls();
|
||||
assertThat(calls).hasSize(2);
|
||||
assertThat(calls.get(0).system).isEqualTo(PromptChain.EXTRACT);
|
||||
assertThat(calls.get(0).user).isEqualTo(Scenarios.GIT_LOG);
|
||||
assertThat(calls.get(1).system).isEqualTo(PromptChain.WRITE);
|
||||
assertThat(calls.get(1).user).isEqualTo(result.bullets()); // step 2 saw exactly what step 1 returned
|
||||
|
||||
var t = new Transcript("01-prompt-chain.txt");
|
||||
t.line("# prompt chain: git log -> bullets -> gate -> summary");
|
||||
t.calls(model);
|
||||
t.line("calls: " + calls.size());
|
||||
t.line("bullets: " + result.bullets().replace("\n", " / "));
|
||||
t.line("summary: " + result.summary());
|
||||
t.write();
|
||||
}
|
||||
|
||||
@Test
|
||||
void gateStopsTheChainBeforeTheSecondCall() throws Exception {
|
||||
var model = new ScriptedChatModel()
|
||||
.when("release-note bullets", "Sure! Here are the changes:\n- Added a --json flag");
|
||||
var chain = new PromptChain(Scenarios.client(model));
|
||||
|
||||
assertThatThrownBy(() -> chain.run(Scenarios.GIT_LOG))
|
||||
.isInstanceOf(PromptChain.GateFailedException.class)
|
||||
.hasMessage("step 1 line is not a bullet: Sure! Here are the changes:");
|
||||
assertThat(model.calls()).hasSize(1); // the summary call was never made
|
||||
|
||||
var t = new Transcript("02-chain-gate-failure.txt");
|
||||
t.line("# prompt chain: step 1 breaks the bullet format");
|
||||
t.calls(model);
|
||||
t.line("gate: step 1 line is not a bullet: Sure! Here are the changes:");
|
||||
t.line("calls: " + model.calls().size() + " (the summary call was not made)");
|
||||
t.write();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package com.ankurm.agentic;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
class RouterTest {
|
||||
|
||||
@Test
|
||||
void eachCategoryGetsItsOwnHandlerPrompt() throws Exception {
|
||||
var t = new Transcript("03-routing.txt");
|
||||
t.line("# routing: one classification call, then one specialised call");
|
||||
String[][] cases = {
|
||||
{"BILLING", "I was charged twice this month"},
|
||||
{"TECHNICAL", "The connector returns 401 since Monday"},
|
||||
{"GENERAL", "Do you have an office in Pune?"}};
|
||||
for (String[] c : cases) {
|
||||
var model = Scenarios.router(c[0]);
|
||||
var routed = new Router(Scenarios.client(model)).handle(c[1]);
|
||||
|
||||
var calls = model.calls();
|
||||
assertThat(calls).hasSize(2);
|
||||
assertThat(calls.get(0).system).isEqualTo(Router.CLASSIFY);
|
||||
assertThat(calls.get(0).user).isEqualTo(c[1]);
|
||||
assertThat(calls.get(1).system).isEqualTo(Router.HANDLERS.get(Router.Category.valueOf(c[0])));
|
||||
assertThat(routed.fellBack()).isFalse();
|
||||
|
||||
t.line("message: " + c[1]);
|
||||
t.calls(model);
|
||||
t.line("category: " + routed.category() + ", fell back: " + routed.fellBack());
|
||||
}
|
||||
t.write();
|
||||
}
|
||||
|
||||
@Test
|
||||
void anUnusableLabelFallsBackToGeneralInsteadOfGuessing() throws Exception {
|
||||
var t = new Transcript("04-routing-fallback.txt");
|
||||
t.line("# routing: what the classifier says versus where the message goes");
|
||||
String[][] labels = {
|
||||
{"Technical.", "TECHNICAL"}, // sloppy but parseable: accepted
|
||||
{"I think this is probably a billing question", "GENERAL"}, // prose: rejected, not substring-matched
|
||||
{"REFUNDS", "GENERAL"}, // a label we do not have
|
||||
{"", "GENERAL"}}; // empty reply
|
||||
for (String[] l : labels) {
|
||||
var model = Scenarios.router(l[0]);
|
||||
var routed = new Router(Scenarios.client(model)).handle("The connector returns 401 since Monday");
|
||||
assertThat(routed.category().name()).isEqualTo(l[1]);
|
||||
assertThat(routed.fellBack()).isEqualTo(l[1].equals("GENERAL"));
|
||||
assertThat(model.calls().get(1).system).isEqualTo(Router.HANDLERS.get(routed.category()));
|
||||
t.line("classifier said [" + l[0] + "] -> " + routed.category() + ", fell back: " + routed.fellBack());
|
||||
}
|
||||
t.write();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package com.ankurm.agentic;
|
||||
|
||||
import org.springframework.ai.chat.client.ChatClient;
|
||||
|
||||
/** The scripted conversations shared by the per-pattern tests and the cost test, so both measure the same thing. */
|
||||
final class Scenarios {
|
||||
|
||||
static final String GIT_LOG = """
|
||||
abc123 add --json flag to the export command
|
||||
def456 fix crash when the input file is empty
|
||||
0a1b2c refactor: rename Exporter fields""";
|
||||
|
||||
static final String CODE = "String q = \"select * from users where id=\" + id;";
|
||||
|
||||
static final String TASK = "Add a --json output flag to the export CLI";
|
||||
|
||||
static ChatClient client(ScriptedChatModel model) {
|
||||
return ChatClient.create(model);
|
||||
}
|
||||
|
||||
static ScriptedChatModel chain() {
|
||||
return new ScriptedChatModel()
|
||||
.when("release-note bullets", "- Added a --json flag to the export command\n- Fixed a crash on empty input files")
|
||||
.when("two-sentence summary", "Export can now write JSON with --json. Empty input files no longer crash it.");
|
||||
}
|
||||
|
||||
static ScriptedChatModel router(String label) {
|
||||
return new ScriptedChatModel()
|
||||
.when("Classify the support message", label)
|
||||
.when("billing specialist", "Invoices are issued on the 1st; the duplicate charge will be refunded.")
|
||||
.when("support engineer", "Check the connector's API key first; a 401 almost always means it expired.")
|
||||
.when("friendly support agent", "Thanks for writing in. I will connect you with a specialist.");
|
||||
}
|
||||
|
||||
static ScriptedChatModel sectioning() {
|
||||
return new ScriptedChatModel()
|
||||
.when("for security only", "SQL is built by string concatenation, which allows injection.")
|
||||
.when("for performance only", "select * pulls every column when one row is needed.")
|
||||
.when("for readability only", "The query string should be a named constant.");
|
||||
}
|
||||
|
||||
static ScriptedChatModel voting(Object... ballots) {
|
||||
return new ScriptedChatModel().when("safe to publish", ballots);
|
||||
}
|
||||
|
||||
static final String PLAN_JSON = """
|
||||
{"steps":[
|
||||
{"title":"Parse the flag","instruction":"Register --json in the argument parser."},
|
||||
{"title":"Serialise rows","instruction":"Write rows with Jackson when the flag is set."},
|
||||
{"title":"Document it","instruction":"Add --json to the help text and the README."}]}""";
|
||||
|
||||
static ScriptedChatModel orchestrator() {
|
||||
return new ScriptedChatModel()
|
||||
.when("You are a planner", PLAN_JSON)
|
||||
.when("carry out one step",
|
||||
"Add a boolean option --json to ExportArgs.",
|
||||
"Use JsonMapper.writeValue(out, rows) when json is true.",
|
||||
"Add one line to --help and a README example.")
|
||||
.when("Combine the step results", "Three changes: ExportArgs gets --json, Exporter writes JSON, docs mention it.");
|
||||
}
|
||||
|
||||
static ScriptedChatModel evaluatorConverging() {
|
||||
return new ScriptedChatModel()
|
||||
.when("Write a git commit message", "Added a --json flag to the export command", "Add a --json flag to the export command.", "Add --json output flag")
|
||||
.when("judge a commit message", "FAIL: Use the imperative mood.", "FAIL: Remove the trailing period.", "PASS");
|
||||
}
|
||||
|
||||
static ScriptedChatModel evaluatorStuck() {
|
||||
return new ScriptedChatModel()
|
||||
.when("Write a git commit message", "Add a --json output flag to the export command of the CLI",
|
||||
"Add a --json output flag to the export command of CLI", "Add a --json output flag to the export command, CLI")
|
||||
.when("judge a commit message", "FAIL: Longer than 50 characters.", "Looks good to me!", "FAIL: Still longer than 50 characters.");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
package com.ankurm.agentic;
|
||||
|
||||
import org.springframework.ai.chat.messages.AssistantMessage;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
import org.springframework.ai.chat.messages.MessageType;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.model.Generation;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Deque;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
/**
|
||||
* A ChatModel that never touches the network. Replies come from rules: the first rule whose text appears in the
|
||||
* system prompt wins, and each rule hands out its scripted replies in order (the last one repeats). A scripted
|
||||
* reply may be a RuntimeException, which is thrown instead of returned. Every prompt the model sees is recorded
|
||||
* with a logical start and end tick, so tests can assert the exact prompt sequence and measure how many calls
|
||||
* really ran one after another.
|
||||
*/
|
||||
final class ScriptedChatModel implements ChatModel {
|
||||
|
||||
/** One call as the model saw it. Ticks come from one shared counter, so they order events across threads. */
|
||||
static final class Call {
|
||||
final int index;
|
||||
final String system;
|
||||
final String user;
|
||||
final long start;
|
||||
volatile long end;
|
||||
|
||||
Call(int index, String system, String user, long start) {
|
||||
this.index = index;
|
||||
this.system = system;
|
||||
this.user = user;
|
||||
this.start = start;
|
||||
}
|
||||
|
||||
int chars() {
|
||||
return system.length() + user.length();
|
||||
}
|
||||
}
|
||||
|
||||
private static final class Rule {
|
||||
final String systemContains; // null means: matches only calls that have no system message
|
||||
final Deque<Object> replies = new ArrayDeque<>();
|
||||
Object last;
|
||||
|
||||
Rule(String systemContains, Object[] scripted) {
|
||||
this.systemContains = systemContains;
|
||||
replies.addAll(List.of(scripted));
|
||||
}
|
||||
|
||||
synchronized Object next() {
|
||||
if (!replies.isEmpty()) {
|
||||
last = replies.poll();
|
||||
}
|
||||
return last;
|
||||
}
|
||||
}
|
||||
|
||||
private final List<Rule> rules = new ArrayList<>();
|
||||
private final List<Call> calls = Collections.synchronizedList(new ArrayList<>());
|
||||
private final AtomicLong tick = new AtomicLong();
|
||||
private final AtomicInteger inFlight = new AtomicInteger();
|
||||
private final AtomicInteger maxInFlight = new AtomicInteger();
|
||||
private long latencyMs;
|
||||
private CountDownLatch rendezvous;
|
||||
|
||||
ScriptedChatModel when(String systemContains, Object... replies) {
|
||||
rules.add(new Rule(systemContains, replies));
|
||||
return this;
|
||||
}
|
||||
|
||||
ScriptedChatModel whenNoSystem(Object... replies) {
|
||||
rules.add(new Rule(null, replies));
|
||||
return this;
|
||||
}
|
||||
|
||||
/** Every call sleeps this long between its start tick and its end tick, so parallel calls overlap. */
|
||||
ScriptedChatModel latency(long ms) {
|
||||
this.latencyMs = ms;
|
||||
return this;
|
||||
}
|
||||
|
||||
/** Every call blocks until n callers are inside call() at once. Sequential code would time out. */
|
||||
ScriptedChatModel rendezvousOf(int n) {
|
||||
this.rendezvous = new CountDownLatch(n);
|
||||
return this;
|
||||
}
|
||||
|
||||
List<Call> calls() {
|
||||
synchronized (calls) {
|
||||
return List.copyOf(calls);
|
||||
}
|
||||
}
|
||||
|
||||
int maxInFlight() {
|
||||
return maxInFlight.get();
|
||||
}
|
||||
|
||||
/** Longest chain of calls where each one started after the previous one ended: the sequential depth. */
|
||||
int stages() {
|
||||
List<Call> sorted = new ArrayList<>(calls());
|
||||
sorted.sort((a, b) -> Long.compare(a.start, b.start));
|
||||
int[] depth = new int[sorted.size()];
|
||||
int best = 0;
|
||||
for (int i = 0; i < sorted.size(); i++) {
|
||||
depth[i] = 1;
|
||||
for (int j = 0; j < i; j++) {
|
||||
if (sorted.get(j).end < sorted.get(i).start) {
|
||||
depth[i] = Math.max(depth[i], depth[j] + 1);
|
||||
}
|
||||
}
|
||||
best = Math.max(best, depth[i]);
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
int totalChars() {
|
||||
return calls().stream().mapToInt(Call::chars).sum();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ChatResponse call(Prompt prompt) {
|
||||
StringBuilder system = new StringBuilder();
|
||||
StringBuilder user = new StringBuilder();
|
||||
for (Message m : prompt.getInstructions()) {
|
||||
if (m.getMessageType() == MessageType.SYSTEM) {
|
||||
system.append(m.getText());
|
||||
} else if (m.getMessageType() == MessageType.USER) {
|
||||
user.append(m.getText());
|
||||
}
|
||||
}
|
||||
Call call;
|
||||
synchronized (calls) {
|
||||
call = new Call(calls.size() + 1, system.toString(), user.toString(), tick.incrementAndGet());
|
||||
calls.add(call);
|
||||
}
|
||||
int now = inFlight.incrementAndGet();
|
||||
maxInFlight.accumulateAndGet(now, Math::max);
|
||||
try {
|
||||
if (rendezvous != null) {
|
||||
rendezvous.countDown();
|
||||
if (!rendezvous.await(5, TimeUnit.SECONDS)) {
|
||||
throw new IllegalStateException("calls did not overlap: this code ran them one after another");
|
||||
}
|
||||
}
|
||||
if (latencyMs > 0) {
|
||||
Thread.sleep(latencyMs);
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new IllegalStateException(e);
|
||||
} finally {
|
||||
inFlight.decrementAndGet();
|
||||
call.end = tick.incrementAndGet();
|
||||
}
|
||||
Object reply = scripted(call);
|
||||
if (reply instanceof RuntimeException e) {
|
||||
throw e;
|
||||
}
|
||||
return new ChatResponse(List.of(new Generation(new AssistantMessage((String) reply))));
|
||||
}
|
||||
|
||||
private Object scripted(Call call) {
|
||||
for (Rule rule : rules) {
|
||||
boolean hit = rule.systemContains == null ? call.system.isEmpty() : call.system.contains(rule.systemContains);
|
||||
if (hit) {
|
||||
return rule.next();
|
||||
}
|
||||
}
|
||||
throw new AssertionError("no scripted reply for system=[" + call.system + "] user=[" + call.user + "]");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package com.ankurm.agentic;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/** Collects lines and writes them to output/NN-name.txt, so the test IS the transcript. */
|
||||
final class Transcript {
|
||||
private final List<String> lines = new ArrayList<>();
|
||||
private final String file;
|
||||
|
||||
Transcript(String file) {
|
||||
this.file = file;
|
||||
}
|
||||
|
||||
void line(String s) {
|
||||
lines.add(s);
|
||||
}
|
||||
|
||||
/** One line per recorded call: its number, the system prompt and the user prompt, each cut to one short line. */
|
||||
void calls(ScriptedChatModel model) {
|
||||
for (var c : model.calls()) {
|
||||
line("call %d | system: %s | user: %s".formatted(c.index, clip(c.system), clip(c.user)));
|
||||
}
|
||||
}
|
||||
|
||||
/** For parallel patterns: the order calls arrive in is not deterministic, so list them sorted and unnumbered. */
|
||||
void callsUnordered(ScriptedChatModel model) {
|
||||
model.calls().stream()
|
||||
.map(c -> "call | system: %s | user: %s".formatted(clip(c.system), clip(c.user)))
|
||||
.sorted()
|
||||
.forEach(this::line);
|
||||
}
|
||||
|
||||
static String clip(String s) {
|
||||
String first = s.lines().findFirst().orElse("");
|
||||
String more = s.lines().count() > 1 ? " [+" + (s.lines().count() - 1) + " lines]" : "";
|
||||
String cut = first.length() > 56 ? first.substring(0, 56) + "..." : first;
|
||||
return (cut.isEmpty() ? "(none)" : cut) + more;
|
||||
}
|
||||
|
||||
void write() throws IOException {
|
||||
Path dir = Path.of("output");
|
||||
Files.createDirectories(dir);
|
||||
Files.write(dir.resolve(file), lines);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user