Add evaluation module: RelevancyEvaluator and FactCheckingEvaluator, golden dataset with a pass-rate gate, deterministic CI judge and simulated judge noise
Co-Authored-By: Claude Sonnet 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01JXVi2GMQ7bR5EmbUFdDj7N
This commit is contained in:
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
# Regenerates every file under output/. Tests 01-07 are written by the suite itself through the
|
||||
# Transcript helper; 08 is cut from the surefire report to show the live-judge test being skipped.
|
||||
# No Docker, no database and no API key is needed.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
rm -rf target
|
||||
mvn -q -B test 2>&1 | grep -E "Tests run:|BUILD|FAIL" || true
|
||||
f=target/surefire-reports/TEST-com.ankurm.evaluation.LiveJudgeTest.xml
|
||||
{
|
||||
echo "# LiveJudgeTest as recorded by surefire when OPENAI_API_KEY is not set"
|
||||
echo
|
||||
grep -o '<testcase name="[^"]*"' "$f" | sed 's/<testcase name=/testcase: /'
|
||||
grep -o '<skipped message="[^"]*"' "$f" | sed 's/<skipped message=/skipped : /'
|
||||
} > output/08-live-judge-skipped.txt
|
||||
ls output
|
||||
Reference in New Issue
Block a user