Add evaluation module: RelevancyEvaluator and FactCheckingEvaluator, golden dataset with a pass-rate gate, deterministic CI judge and simulated judge noise

Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01JXVi2GMQ7bR5EmbUFdDj7N
This commit is contained in:
Claude
2026-10-09 06:15:41 +00:00
parent 80db4eb8ac
commit 65f580c5bc
37 changed files with 1353 additions and 0 deletions
+16
View File
@@ -0,0 +1,16 @@
#!/usr/bin/env bash
# Regenerates every file under output/. Tests 01-07 are written by the suite itself through the
# Transcript helper; 08 is cut from the surefire report to show the live-judge test being skipped.
# No Docker, no database and no API key is needed.
set -euo pipefail
cd "$(dirname "$0")/.."
rm -rf target
mvn -q -B test 2>&1 | grep -E "Tests run:|BUILD|FAIL" || true
f=target/surefire-reports/TEST-com.ankurm.evaluation.LiveJudgeTest.xml
{
echo "# LiveJudgeTest as recorded by surefire when OPENAI_API_KEY is not set"
echo
grep -o '<testcase name="[^"]*"' "$f" | sed 's/<testcase name=/testcase: /'
grep -o '<skipped message="[^"]*"' "$f" | sed 's/<skipped message=/skipped : /'
} > output/08-live-judge-skipped.txt
ls output