asmhatre/spring-ai
1
0
Fork 0
Code Activity

Add rag-ingestion module: PDF, Word and HTML readers and chunking strategies compared by retrieval hit rate; metadata in the embedding, content-hash idempotent ingestion, re-embedding cost of an edit

Browse Source 
Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_0146opDKBYtADXvkc39zdiyw
This commit is contained in:
Claude
2026-10-22 18:04:49 +00:00
parent c13195295d
commit d7e68a14af
35 changed files with 1699 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rag-ingestion/src/test/java/com/ankurm/ragingestion/ReadersTest.java
+143
View File
@@ -0,0 +1,143 @@
package com.ankurm.ragingestion;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import com.ankurm.ragingestion.support.Fixtures;
import com.ankurm.ragingestion.support.Transcript;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.ai.document.Document;
import org.springframework.ai.reader.jsoup.JsoupDocumentReader;
import org.springframework.ai.reader.jsoup.config.JsoupDocumentReaderConfig;
import org.springframework.ai.reader.pdf.PagePdfDocumentReader;
import org.springframework.ai.reader.pdf.config.PdfDocumentReaderConfig;
import org.springframework.ai.reader.tika.TikaDocumentReader;
import org.springframework.core.io.FileSystemResource;
/** What each reader hands back for the same runbook, written as PDF, Word and HTML. */
class ReadersTest {
@TempDir
Path dir;
@Test
void whatEachReaderReturns() throws Exception {
Path pdf = Fixtures.pdf(dir);
Path docx = Fixtures.docx(dir);
Path html = Fixtures.html(dir);
String footerMarker = "Confidential";
try (Transcript t = new Transcript("01-readers.txt", "What each reader returns for the same 36-paragraph runbook")) {
t.line("corpus: %d characters of body text (title, 12 headings, 36 paragraphs)", Corpus.fullText().length());
t.section("PagePdfDocumentReader, default config");
List<Document> pages = new PagePdfDocumentReader(new FileSystemResource(pdf)).get();
t.line("Documents returned: %d (one per PDF page)", pages.size());
t.line("metadata of page 1: %s", new TreeMap<>(pages.get(0).getMetadata()));
t.line("pages whose text contains the running footer: %d of %d", count(pages, footerMarker), pages.size());
t.line("characters as read: %d, after collapsing runs of whitespace: %d (the corpus itself is %d)", total(pages),
pages.stream().mapToInt(d -> norm(d.getText()).length()).sum(), Corpus.fullText().length());
t.line("page 1 ends with: \"%s\"", tail(pages.get(0).getText(), 70));
t.line("page 2 starts with: \"%s\"", norm(head(pages.get(1).getText(), 90)));
t.line("paragraphs that no single page contains whole, because a page break cuts them: %d of 36", cutParagraphs(pages));
t.section("PagePdfDocumentReader with pageBottomMargin");
for (int margin : new int[] {30, 60, 100}) {
PdfDocumentReaderConfig cfg = PdfDocumentReaderConfig.builder().withPageBottomMargin(margin).build();
List<Document> cut = new PagePdfDocumentReader(new FileSystemResource(pdf), cfg).get();
t.line("pageBottomMargin=%3d: footer on %d of %d pages, answer fragments still in the text: %d of %d", margin,
count(cut, footerMarker), cut.size(), fragments(cut), Corpus.QUESTIONS.size());
}
t.line("no margin: footer on %d of %d pages, answer fragments still in the text: %d of %d",
count(pages, footerMarker), pages.size(), fragments(pages), Corpus.QUESTIONS.size());
t.section("TikaDocumentReader on the .docx");
List<Document> word = new TikaDocumentReader(new FileSystemResource(docx)).get();
t.line("Documents returned: %d", word.size());
t.line("metadata: %s", new TreeMap<>(word.get(0).getMetadata()));
t.line("characters: %d, contains the footer text: %s", word.get(0).getText().length(),
word.get(0).getText().contains(footerMarker));
t.line("starts with: \"%s\"", head(word.get(0).getText(), 90));
t.line("blank-line-separated blocks in the text: %d, newline-separated lines: %d", word.get(0).getText().split("\n\n+").length,
word.get(0).getText().split("\n").length);
t.line("answer fragments in the text: %d of %d", fragments(word), Corpus.QUESTIONS.size());
t.section("TikaDocumentReader on the same PDF and HTML");
List<Document> tikaPdf = new TikaDocumentReader(new FileSystemResource(pdf)).get();
List<Document> tikaHtml = new TikaDocumentReader(new FileSystemResource(html)).get();
t.line(".pdf -> %d Document, %d characters, metadata keys %s", tikaPdf.size(), tikaPdf.get(0).getText().length(),
tikaPdf.get(0).getMetadata().keySet());
t.line(".html -> %d Document, %d characters, contains \"Cookie settings\": %s, contains tracking script text: %s",
tikaHtml.size(), tikaHtml.get(0).getText().length(), tikaHtml.get(0).getText().contains("Cookie settings"),
tikaHtml.get(0).getText().contains("noise noise"));
t.section("JsoupDocumentReader on the .html");
List<Document> body = new JsoupDocumentReader(new FileSystemResource(html)).get();
t.line("default config: %d Document, %d characters, contains nav link text \"Status\": %s, contains \"Cookie settings\": %s",
body.size(), body.get(0).getText().length(), body.get(0).getText().contains("Status"),
body.get(0).getText().contains("Cookie settings"));
t.line("default metadata: %s", new TreeMap<>(body.get(0).getMetadata()));
JsoupDocumentReaderConfig article = JsoupDocumentReaderConfig.builder().selector("article").build();
List<Document> art = new JsoupDocumentReader(new FileSystemResource(html), article).get();
t.line("selector=article: %d Document, %d characters, nav text: %s, footer text: %s", art.size(),
art.get(0).getText().length(), art.get(0).getText().contains("Status"),
art.get(0).getText().contains("Cookie settings"));
JsoupDocumentReaderConfig bySection = JsoupDocumentReaderConfig.builder().selector("section").allElements(false)
.groupByElement(true).build();
List<Document> sections = new JsoupDocumentReader(new FileSystemResource(html), bySection).get();
t.line("selector=section, groupByElement=true: %d Documents", sections.size());
t.line("first section Document: %d characters, starts \"%s\"", sections.get(0).getText().length(),
head(sections.get(0).getText(), 60));
JsoupDocumentReaderConfig meta = JsoupDocumentReaderConfig.builder().selector("article")
.metadataTag("description").additionalMetadata("format", "html").build();
Map<String, Object> m = new TreeMap<>(new JsoupDocumentReader(new FileSystemResource(html), meta).get().get(0).getMetadata());
t.line("with metadataTag(\"description\") and additionalMetadata: %s", m);
}
}
private static int count(List<Document> docs, String needle) {
return (int) docs.stream().filter(d -> d.getText().contains(needle)).count();
}
/** Collapse every run of whitespace to one space, the same normalisation the hit-rate check applies. */
static String norm(String s) {
return s.replaceAll("\\s+", " ").strip();
}
private static int fragments(List<Document> docs) {
String all = norm(docs.stream().map(Document::getText).collect(java.util.stream.Collectors.joining(" ")));
return (int) Corpus.QUESTIONS.stream().filter(q -> all.contains(q.answer())).count();
}
private static int cutParagraphs(List<Document> pages) {
int cut = 0;
for (Corpus.Section s : Corpus.SECTIONS) {
for (String p : s.paragraphs()) {
if (pages.stream().noneMatch(d -> norm(d.getText()).contains(p))) {
cut++;
}
}
}
return cut;
}
private static int total(List<Document> docs) {
return docs.stream().mapToInt(d -> d.getText().length()).sum();
}
private static String head(String s, int n) {
String one = s.strip().replace("\n", "\\n");
return one.length() <= n ? one : one.substring(0, n);
}
private static String tail(String s, int n) {
String one = s.strip().replace("\n", "\\n");
return one.length() <= n ? one : one.substring(one.length() - n);
}
}