Add rag-ingestion module: PDF, Word and HTML readers and chunking strategies compared by retrieval hit rate; metadata in the embedding, content-hash idempotent ingestion, re-embedding cost of an edit

Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_0146opDKBYtADXvkc39zdiyw

rag-ingestion/scripts/run-all.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# Regenerates every file in output/ by running the test suite.
+# Needs: JDK 25, Maven, internet for the first run (about 90 MB for the MiniLM model, cached in ~/.cache/minilm),
+# and a PostgreSQL 16 with pgvector 0.8 on localhost:5432 (database ragdemo, user demo / demo) for the two ingestion tests.
+# Set RAG_DB_URL to point somewhere else.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+rm -rf output
+mvn -B -q test
+ls output

rag-ingestion/scripts/services-up.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# One way to get the database the ingestion tests expect: a pgvector container on :5432.
+set -euo pipefail
+docker run -d --name ragdemo-pg -e POSTGRES_USER=demo -e POSTGRES_PASSWORD=demo -e POSTGRES_DB=ragdemo \
+  -p 5432:5432 pgvector/pgvector:0.8.1-pg16
+until docker exec ragdemo-pg pg_isready -U demo -d ragdemo >/dev/null 2>&1; do sleep 1; done
+echo "ready: jdbc:postgresql://localhost:5432/ragdemo (demo / demo)"