74 lines
3.3 KiB
Java
74 lines
3.3 KiB
Java
package com.ankurm.agentic;
|
|
|
|
import org.junit.jupiter.api.Test;
|
|
|
|
import java.util.ArrayList;
|
|
import java.util.List;
|
|
|
|
import static org.assertj.core.api.Assertions.assertThat;
|
|
|
|
/** Runs every happy path with 100 ms of fake latency per call and tabulates what each pattern really did. */
|
|
class CostTest {
|
|
|
|
record Row(String name, ScriptedChatModel model) {}
|
|
|
|
@Test
|
|
void callsStagesAndCharactersPerPattern() throws Exception {
|
|
List<Row> rows = new ArrayList<>();
|
|
|
|
var single = new ScriptedChatModel().latency(100).whenNoSystem("one answer");
|
|
Scenarios.client(single).prompt().user(Scenarios.GIT_LOG).call().content();
|
|
rows.add(new Row("single call (baseline)", single));
|
|
|
|
var chain = Scenarios.chain().latency(100);
|
|
new PromptChain(Scenarios.client(chain)).run(Scenarios.GIT_LOG);
|
|
rows.add(new Row("prompt chain", chain));
|
|
|
|
var router = Scenarios.router("TECHNICAL").latency(100);
|
|
new Router(Scenarios.client(router)).handle("The connector returns 401 since Monday");
|
|
rows.add(new Row("routing", router));
|
|
|
|
var sectioning = Scenarios.sectioning().latency(100);
|
|
new Parallelizer(Scenarios.client(sectioning)).sectioning(Scenarios.CODE);
|
|
rows.add(new Row("sectioning x3", sectioning));
|
|
|
|
var voting = Scenarios.voting("SAFE", "SAFE", "UNSAFE").latency(100);
|
|
new Parallelizer(Scenarios.client(voting)).vote("Free crypto, click here", 3);
|
|
rows.add(new Row("voting x3", voting));
|
|
|
|
var orch = Scenarios.orchestrator().latency(100);
|
|
new Orchestrator(Scenarios.client(orch)).run(Scenarios.TASK);
|
|
rows.add(new Row("orchestrator, 3 steps", orch));
|
|
|
|
var evalOk = Scenarios.evaluatorConverging().latency(100);
|
|
new EvaluatorOptimizer(Scenarios.client(evalOk)).run(Scenarios.TASK, 5);
|
|
rows.add(new Row("evaluator, passes in round 3", evalOk));
|
|
|
|
var evalStuck = Scenarios.evaluatorStuck().latency(100);
|
|
new EvaluatorOptimizer(Scenarios.client(evalStuck)).run(Scenarios.TASK, 3);
|
|
rows.add(new Row("evaluator, hits the 3-round cap", evalStuck));
|
|
|
|
var t = new Transcript("12-cost-latency.txt");
|
|
t.line("# calls, sequential stages and prompt characters per pattern (scripted model, 100 ms fake latency per call)");
|
|
t.line("%-34s %5s %7s %9s %12s".formatted("pattern", "calls", "stages", "in flight", "prompt chars"));
|
|
for (Row r : rows) {
|
|
t.line("%-34s %5d %7d %9d %12d".formatted(r.name(), r.model().calls().size(), r.model().stages(),
|
|
r.model().maxInFlight(), r.model().totalChars()));
|
|
}
|
|
t.write();
|
|
|
|
assertThat(rows.get(0).model().stages()).isEqualTo(1);
|
|
assertThat(rows.get(1).model().calls()).hasSize(2);
|
|
assertThat(rows.get(1).model().stages()).isEqualTo(2);
|
|
assertThat(rows.get(2).model().stages()).isEqualTo(2);
|
|
assertThat(rows.get(3).model().calls()).hasSize(3);
|
|
assertThat(rows.get(3).model().stages()).isEqualTo(1);
|
|
assertThat(rows.get(4).model().stages()).isEqualTo(1);
|
|
assertThat(rows.get(5).model().calls()).hasSize(5);
|
|
assertThat(rows.get(5).model().stages()).isEqualTo(5);
|
|
assertThat(rows.get(6).model().calls()).hasSize(6);
|
|
assertThat(rows.get(6).model().stages()).isEqualTo(6);
|
|
assertThat(rows.get(7).model().calls()).hasSize(6);
|
|
}
|
|
}
|