Add pdf-invoice module: one invoice in OpenPDF 3.0.5, iText Core 9.8.0 and Flying Saucer 10.7.0, Devanagari shaping checked by rendering, streamed vs buffered download, subset sizes and timings

Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_017ic4b6Kg9RWSTJzyz6KP9X

pdf-invoice/scripts/compare.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""Crop the item table out of each probe PDF and stack the crops under a Pillow/libraqm reference line.
+
+Usage: compare.py <pdf-dir> <png-dir> <font.ttf>
+
+The reference is the same probe text drawn by Pillow's RAQM layout engine (libraqm -> HarfBuzz) with the same
+Noto Sans Devanagari file: an independent shaper, there to show what "correct" looks like.
+"""
+import re, subprocess, sys
+from pathlib import Path
+from PIL import Image, ImageDraw, ImageFont
+
+pdfdir, pngdir, fontfile = Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3]
+PROBE = "क्षत्रिय ज्ञान श्री द्वार कर्म कि प्रिय"
+DPI = 200
+ENGINES = [
+    ("openpdf", "OpenPDF 3.0.5, defaults"),
+    ("itext", "iText Core 9.8.0, defaults"),
+    ("flyingsaucer", "Flying Saucer 10.7.0, defaults"),
+    ("openpdf-fop", "OpenPDF + Apache FOP 2.11 on the classpath"),
+    ("openpdf-jdk-actualtext", "OpenPDF GlyphLayoutManager (JDK shaper)"),
+    ("flyingsaucer-shaped", "Flying Saucer + OpenPDF LayoutProcessor"),
+]
+
+def box(pdf, word):
+    out = subprocess.run(["pdftotext", "-bbox", str(pdf), "-"], capture_output=True, text=True).stdout
+    hits = re.findall(r'<word xMin="([\d.]+)" yMin="([\d.]+)" xMax="([\d.]+)" yMax="([\d.]+)">%s[^<]*</word>' % re.escape(word), out)
+    return [tuple(map(float, h)) for h in hits]
+
+font_big = ImageFont.truetype(fontfile, 40, layout_engine=ImageFont.Layout.RAQM)
+font_lbl = ImageFont.truetype(fontfile, 22, layout_engine=ImageFont.Layout.RAQM)
+rows = []
+
+ref = Image.new("RGB", (1500, 120), "white")
+d = ImageDraw.Draw(ref)
+d.text((10, 6), "reference: Pillow + libraqm (HarfBuzz), same font", font=font_lbl, fill="#555")
+d.text((10, 44), PROBE, font=font_big, fill="black")
+rows.append(ref)
+
+for engine, label in ENGINES:
+    pdf = pdfdir / f"probe-{engine}.pdf"
+    if not pdf.exists():
+        print("skip", engine, "(no probe pdf)")
+        continue
+    item = box(pdf, "Item")[0]
+    total = box(pdf, "Total")[-1]
+    y0 = int((item[1] - 8) * DPI / 72)
+    y1 = int((total[3] + 62) * DPI / 72)
+    subprocess.run(["pdftoppm", "-r", str(DPI), "-png", "-singlefile", "-x", str(int(36 * DPI / 72)), "-y", str(y0),
+                    "-W", "1500", "-H", str(y1 - y0), str(pdf), str(pngdir / f"crop-{engine}")], check=True)
+    img = Image.open(pngdir / f"crop-{engine}.png").convert("RGB")
+    band = Image.new("RGB", (1500, 34), "#f4f5f7")
+    ImageDraw.Draw(band).text((10, 4), label, font=font_lbl, fill="#222")
+    rows.append(band)
+    rows.append(img)
+    (pngdir / f"crop-{engine}.png").unlink()
+
+W = 1500
+H = sum(r.height for r in rows)
+out = Image.new("RGB", (W, H), "white")
+y = 0
+for r in rows:
+    out.paste(r, (0, y))
+    y += r.height
+out = out.convert("P", palette=Image.ADAPTIVE, colors=32)
+out.save(pngdir / "shaping-comparison.png", optimize=True)
+print("wrote", pngdir / "shaping-comparison.png", out.size)

pdf-invoice/scripts/licences.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Licence facts, read from the sources at run time: Maven POMs and the iText licence page.
+set -u
+echo "# licences as declared in the POMs Maven downloaded (~/.m2) and in the pdfCalligraph POM on iText's own repository, fetched $(date -u +%F)"
+M2="${HOME}/.m2/repository"
+lic() { grep -A1 '<license>' "$1" | grep '<name>' | sed 's/^ *<name>//; s#</name>##' | tr '\n' ';'; }
+echo "openpdf 3.0.5 (declared in openpdf-parent):     $(lic "$M2/com/github/librepdf/openpdf-parent/3.0.5/openpdf-parent-3.0.5.pom")"
+echo "itext kernel/layout 9.8.0 (root pom):            $(lic "$M2/com/itextpdf/root/9.8.0/root-9.8.0.pom")"
+echo "flying-saucer-pdf 10.7.0 (flying-saucer-parent): $(lic "$M2/org/xhtmlrenderer/flying-saucer-parent/10.7.0/flying-saucer-parent-10.7.0.pom")"
+pom() { curl -s --retry 3 "$1" | grep -A2 '<license>' | grep -E '<name>' | sed 's/^ *<name>//; s#</name>##' | tr '\n' ';'; }
+echo "typography (pdfCalligraph) 5.1.1:           $(pom https://repo.itextsupport.com/releases/com/itextpdf/typography/5.1.1/typography-5.1.1.pom | tr '\n' ' ')"
+echo
+echo "# sentences from https://itextpdf.com/how-buy/agpl-license (verbatim, page text with tags stripped)"
+curl -sL https://itextpdf.com/how-buy/agpl-license | python3 -c '
+import re, sys, html
+t = sys.stdin.read()
+t = re.sub(r"<script.*?</script>|<style.*?</style>", "", t, flags=re.S)
+t = html.unescape(re.sub(r"<[^>]+>", "\n", t))
+keys = ("dual-licensed", "AGPLv3 license. This license applies", "You may not deploy it on a network", "Producer", "producer line",
+        "violation to use", "release you from the AGPLv3")
+seen = set()
+for line in (l.strip() for l in t.split("\n")):
+    if any(k in line for k in keys) and line not in seen and len(line) > 40:
+        seen.add(line); print("- " + line)
+'
+echo
+echo "# versions: latest GA per maven-metadata.xml"
+for a in com/github/librepdf/openpdf com/itextpdf/kernel org/xhtmlrenderer/flying-saucer-pdf org/xhtmlrenderer/flying-saucer-pdf-openpdf org/apache/pdfbox/pdfbox; do
+  v=""
+  for _ in 1 2 3 4 5 6; do   # Maven Central answers 429 now and then
+    v="$(curl -s https://repo1.maven.org/maven2/$a/maven-metadata.xml | tr '<' '\n' | grep -E '^(release|latest)>' | head -1 | sed 's/>/ /')"
+    [ -n "$v" ] && break; sleep 15
+  done
+  printf '%-45s %s\n' "$a" "${v:-(rate limited, not read)}"
+done

pdf-invoice/scripts/make_fonts.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+"""Cut static Regular and Bold instances out of the Noto Sans Devanagari variable font.
+
+Source (SIL OFL 1.1, copyright The Noto Project Authors; OFL.txt next to the output files):
+  https://cdn.jsdelivr.net/gh/google/fonts@main/ofl/notosansdevanagari/NotoSansDevanagari%5Bwdth%2Cwght%5D.ttf
+  (a jsDelivr mirror of github.com/google/fonts; raw.githubusercontent.com was blocked in the sandbox this was built in)
+
+Why static instances: OpenPDF and iText read a variable font's default master only, and Flying Saucer
+picks weights by file. Two small static files make "regular" and "bold" mean the same thing in all three.
+Needs: pip install fonttools
+"""
+import sys
+from fontTools.ttLib import TTFont
+from fontTools.varLib import instancer
+
+src, outdir = sys.argv[1], sys.argv[2]
+for name, wght in (("Regular", 400), ("Bold", 700)):
+    font = TTFont(src)
+    inst = instancer.instantiateVariableFont(font, {"wght": wght, "wdth": 100}, updateFontNames=True)
+    inst.save(f"{outdir}/NotoSansDevanagari-{name}.ttf")
+    print("wrote", name)

pdf-invoice/scripts/run-all.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+# Regenerates everything under output/. Needs JDK 25, Maven, curl, poppler (pdftoppm, pdftotext, pdffonts),
+# python3 with Pillow built with libraqm. Network is needed once for Maven and for 00-versions-and-licences.txt.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+mkdir -p output/pdf output/png
+JAR=target/pdf-invoice-1.0.0.jar
+HOST="$(java -version 2>&1 | grep -v JAVA_TOOL | head -1) | $(nproc) vCPU | $(awk '/MemTotal/ {printf "%d MB RAM", $2/1024}' /proc/meminfo)"
+
+scripts/licences.sh > output/00-versions-and-licences.txt || echo "licence check skipped (offline?)"
+
+mvn -q -B test                                  # writes output/01, 04, 05
+mvn -q -B -DskipTests package
+
+serve() {  # serve <port> : start the packaged jar, wait until it answers
+  setsid nohup java -jar "$JAR" --server.port="$1" > "target/app-$1.log" 2>&1 < /dev/null &
+  echo $! > "target/app-$1.pid"
+  for _ in $(seq 1 60); do curl -s -o /dev/null "localhost:$1/invoices/openpdf.pdf" && return 0; sleep 1; done
+  echo "app did not start" >&2; return 1
+}
+stop() { kill "$(cat "target/app-$1.pid")" 2>/dev/null || true; }
+
+P=$((20000 + RANDOM % 10000))
+serve "$P"
+ENGINES="openpdf itext flyingsaucer openpdf-jdk openpdf-jdk-actualtext flyingsaucer-shaped"
+
+for e in $ENGINES; do
+  curl -s -o "output/pdf/$e.pdf" "localhost:$P/invoices/$e.pdf"
+  curl -s -o "output/pdf/probe-$e.pdf" "localhost:$P/invoices/$e.pdf?probe=true"
+done
+
+{
+  echo "# HTTP: status and headers per engine (curl -D -), then streaming vs buffered, then a failure after rendering"
+  echo "# $HOST"
+  for e in $ENGINES; do
+    echo "--- GET /invoices/$e.pdf"
+    curl -s -D - -o /dev/null "localhost:$P/invoices/$e.pdf" | tr -d '\r' | grep -i -E '^HTTP|^content-type|^content-disposition|^content-length|^transfer-encoding'
+  done
+  echo "--- GET /invoices/openpdf.pdf?mode=buffered"
+  curl -s -D - -o /dev/null "localhost:$P/invoices/openpdf.pdf?mode=buffered" | tr -d '\r' | grep -i -E '^HTTP|^content-length|^transfer-encoding'
+  echo "--- GET /invoices/openpdf.pdf?disposition=inline"
+  curl -s -D - -o /dev/null "localhost:$P/invoices/openpdf.pdf?disposition=inline" | tr -d '\r' | grep -i -E '^HTTP|^content-disposition'
+  echo "--- GET /invoices/nope.pdf"
+  curl -s -D - -o /dev/null "localhost:$P/invoices/nope.pdf" | tr -d '\r' | grep -i -E '^HTTP'
+  echo "--- GET /invoices/openpdf.pdf?failAtEnd=true   (streamed: headers are already gone when the failure happens)"
+  curl -s -D - -o target/fail-stream.pdf -w "curl exit=%{exitcode} bytes_received=%{size_download}\n" "localhost:$P/invoices/openpdf.pdf?failAtEnd=true" 2>&1 | tr -d '\r' | grep -i -E '^HTTP|^content-length|^transfer-encoding|curl exit' || true
+  echo "--- GET /invoices/openpdf.pdf?failAtEnd=true&mode=buffered   (rendered into memory first)"
+  curl -s -D - -o target/fail-buffered.bin -w "curl exit=%{exitcode} bytes_received=%{size_download}\n" "localhost:$P/invoices/openpdf.pdf?failAtEnd=true&mode=buffered" 2>&1 | tr -d '\r' | grep -i -E '^HTTP|^content-length|curl exit' || true
+} > output/02-endpoints.txt
+
+{
+  echo "# pdffonts per engine (poppler): name, type, embedded, subset"
+  for e in $ENGINES; do echo "--- $e ($(stat -c %s "output/pdf/$e.pdf") bytes)"; pdffonts "output/pdf/$e.pdf"; done
+  echo; echo "# Producer/Creator lines the libraries write into the PDF (pdfinfo)"
+  for e in openpdf itext flyingsaucer; do echo "--- $e"; pdfinfo "output/pdf/$e.pdf" | grep -E '^(Title|Creator|Producer)'; done
+} > output/03-pdffonts-and-producer.txt
+
+stop "$P"
+
+# the FOP variant needs Apache FOP (121 jars with Batik) on the classpath: Maven profile fop
+mvn -q -B -Pfop -DskipTests package
+P2=$((20000 + RANDOM % 10000))
+serve "$P2"
+curl -s -o output/pdf/openpdf-fop.pdf "localhost:$P2/invoices/openpdf-fop.pdf"
+curl -s -o output/pdf/probe-openpdf-fop.pdf "localhost:$P2/invoices/openpdf-fop.pdf?probe=true"
+stop "$P2"
+mvn -q -B -DskipTests package                    # back to the default jar
+
+for e in openpdf itext flyingsaucer openpdf-jdk-actualtext flyingsaucer-shaped openpdf-fop; do
+  pdftoppm -r 110 -png -singlefile -H 800 "output/pdf/$e.pdf" "output/png/invoice-$e"
+done
+python3 scripts/compare.py output/pdf output/png src/main/resources/fonts/NotoSansDevanagari-Regular.ttf
+
+{
+  echo "# in-process timing, $HOST"
+  echo "# indicative: shared sandbox, one JVM, one run per line"
+  java -jar "$JAR" --spring.profiles.active=bench --spring.main.web-application-type=none \
+       --bench.n="${BENCH_N:-500}" --bench.engines=openpdf,itext,flyingsaucer,openpdf-jdk-actualtext,flyingsaucer-shaped 2>/dev/null | grep -E '^#|^[a-z]'
+} > output/06-bench.txt
+echo done