From ff58ff01893f85c458376b5f27856147cfd6d03c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 18:03:45 +0000 Subject: [PATCH] rerank: use an example where the cross-encoder fixes the order in 02, correct the download size Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01Cfo2K4u4rF18QsPXPFnytf --- rerank/README.md | 2 +- rerank/output/02-one-question-by-hand.txt | 22 +++++++++---------- rerank/scripts/run-all.sh | 2 +- .../com/ankurm/rerank/PairEncodingTest.java | 8 +++---- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/rerank/README.md b/rerank/README.md index 187d80a..0247b8b 100644 --- a/rerank/README.md +++ b/rerank/README.md @@ -25,7 +25,7 @@ A bi-encoder (all-MiniLM-L6-v2) retrieves candidates, a cross-encoder (ms-marco- ## Run -`scripts/run-all.sh`. The first run downloads about 250 MB of model files. Every file in `output/` is written by a test that asserts the same facts. Files 06 and 07 contain wall-clock latency and depend on your machine; the others were identical across two consecutive runs. +`scripts/run-all.sh`. The first run downloads about 200 MB of model files. Every file in `output/` is written by a test that asserts the same facts. Files 06 and 07 contain wall-clock latency and depend on your machine; the others were identical across two consecutive runs. | Output | Shows | |---|---| diff --git a/rerank/output/02-one-question-by-hand.txt b/rerank/output/02-one-question-by-hand.txt index 9f23b62..ba04163 100644 --- a/rerank/output/02-one-question-by-hand.txt +++ b/rerank/output/02-one-question-by-hand.txt @@ -1,16 +1,16 @@ # One question, the bi-encoder's top 8, rescored -question: A buyer told their bank they never got the item. What now? -gold passage: G-04 +question: Meaning of E-4133 +gold passage: E-4133 retrieved id cosine cross-encoder logit -1 G-06 0.446 -7.40 -2 G-04 0.440 -8.06 -3 E-4117 0.418 -11.08 -4 E-4101 0.418 -11.30 -5 G-03 0.405 -10.33 -6 E-4140 0.403 -9.91 -7 G-08 0.399 -9.62 -8 G-24 0.391 -9.61 +1 E-4134 0.424 4.27 +2 E-4140 0.405 2.17 +3 E-4135 0.394 3.98 +4 E-4130 0.393 0.73 +5 E-4133 0.386 4.90 +6 E-4139 0.377 2.85 +7 E-4132 0.368 2.74 +8 E-4138 0.345 1.68 -bi-encoder rank of G-04: 2; cross-encoder winner: G-06 +bi-encoder rank of E-4133: 5; cross-encoder winner: E-4133 diff --git a/rerank/scripts/run-all.sh b/rerank/scripts/run-all.sh index 0db4286..55035cf 100755 --- a/rerank/scripts/run-all.sh +++ b/rerank/scripts/run-all.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Regenerates every file in output/ by running the test suite. -# Needs: JDK 25, Maven, and internet on the first run (about 250 MB of models from Hugging Face, cached in ~/.cache/rerank). +# Needs: JDK 25, Maven, and internet on the first run (about 200 MB of models from Hugging Face, cached in ~/.cache/rerank). # No database, no API key. Latency files (06, 07) depend on your machine; every other file is deterministic. set -euo pipefail cd "$(dirname "$0")/.." diff --git a/rerank/src/test/java/com/ankurm/rerank/PairEncodingTest.java b/rerank/src/test/java/com/ankurm/rerank/PairEncodingTest.java index 786c47c..1c017b9 100644 --- a/rerank/src/test/java/com/ankurm/rerank/PairEncodingTest.java +++ b/rerank/src/test/java/com/ankurm/rerank/PairEncodingTest.java @@ -32,12 +32,12 @@ class PairEncodingTest { @Test void oneQuestionByHand() { - String question = "A buyer told their bank they never got the item. What now?"; + String question = "Meaning of E-4133"; List candidates = Fixture.retriever().withTopK(8).retrieve(new Query(question)); float[] scores = Fixture.fp32().score(question, candidates.stream().map(Document::getText).toList()); try (Transcript t = new Transcript("02-one-question-by-hand.txt", "One question, the bi-encoder's top 8, rescored")) { t.line("question: %s", question); - t.line("gold passage: G-04"); + t.line("gold passage: E-4133"); t.blank(); t.line("%-9s %-7s %-12s %s", "retrieved", "id", "cosine", "cross-encoder logit"); int goldRetrieved = -1; @@ -45,7 +45,7 @@ class PairEncodingTest { for (int i = 0; i < candidates.size(); i++) { Document d = candidates.get(i); t.line("%-9d %-7s %-12.3f %.2f", i + 1, d.getMetadata().get("pid"), d.getScore(), scores[i]); - if ("G-04".equals(d.getMetadata().get("pid"))) { + if ("E-4133".equals(d.getMetadata().get("pid"))) { goldRetrieved = i + 1; } if (scores[i] > scores[best]) { @@ -53,7 +53,7 @@ class PairEncodingTest { } } t.blank(); - t.line("bi-encoder rank of G-04: %d; cross-encoder winner: %s", goldRetrieved, candidates.get(best).getMetadata().get("pid")); + t.line("bi-encoder rank of E-4133: %d; cross-encoder winner: %s", goldRetrieved, candidates.get(best).getMetadata().get("pid")); } } }