streaming-huge-json: 1 GiB streaming vs tree vs binding benchmark (Jackson 3.2.4, JDK 25) for the rewritten streaming post

Co-Authored-By: Claude Sonnet 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01CifR2BE7PBvRy5hT8TS9oz
2026-10-26 02:03:57 +00:00
parent b1663c235d
commit 1d0cd8b36e
17 changed files with 560 additions and 0 deletions
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Runs each approach against the huge file under a ladder of -Xmx limits, one fresh JVM per run.
+# Streaming modes run every rung (to show memory stays flat). Tree/bind climb until the first
+# rung that succeeds, then stop. Output: output/bench.txt   (timings vary run to run)
+set -u
+cd "$(dirname "$0")/.."
+FILE="${1:-target/huge.json}"
+CP="target/classes:$(cat target/cp.txt)"
+OUT=output/bench.txt
+mkdir -p output
+{
+  echo "file: $FILE ($(stat -c %s "$FILE") bytes)"
+  echo "java: $(java -version 2>&1 | grep -v 'Picked up' | head -1)"
+  echo "cpus: $(nproc)   ram: $(free -m | awk '/Mem:/{print $2}') MB"
+  echo
+  for mode in stream hybrid; do
+    for x in 32 64 128 1024; do
+      java -Xmx${x}m -cp "$CP" com.ankurm.jackson3.streaminghuge.Scan $mode "$FILE" 2>&1 | grep -v '^Picked up'
+    done
+  done
+  for mode in tree bind; do
+    for x in 64 256 1024 2048 3072 4096 6144; do
+      line=$(timeout 900 java -Xmx${x}m -cp "$CP" com.ankurm.jackson3.streaminghuge.Scan $mode "$FILE" 2>&1 | grep -v '^Picked up'; echo "exit=${PIPESTATUS[0]}")
+      # no output at all = the JVM was SIGKILLed (exit 137) by the container's OOM killer
+      case "$line" in exit=*) line=$(printf '%-7s -Xmx=%5d MB  KILLED  no Java output, %s (SIGKILL by the OS: the heap limit was larger than free RAM)' "$mode" "$x" "$line");; *) line="${line%exit=*}"; line="${line%$'\n'}";; esac
+      echo "$line"
+      case "$line" in *" OK "*) break;; esac
+    done
+  done
+} | tee "$OUT"
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Builds the module, runs the small demos (captured to output/), generates the 1 GiB file
+# (NOT committed: target/ is git-ignored) and runs the memory ladder. Needs a JDK 25 on PATH.
+#   ./scripts/run-all.sh            full run, ~5-10 minutes
+#   ./scripts/run-all.sh --quick    small demos only, no 1 GiB file
+set -eu
+cd "$(dirname "$0")/.."
+mvn -B -q compile
+mvn -B -q dependency:build-classpath -Dmdep.outputFile=target/cp.txt
+CP="target/classes:$(cat target/cp.txt)"
+for c in NestedTrap TruncatedFile ReadLimits; do
+  # grep -v strips the sandbox's injected "Picked up JAVA_TOOL_OPTIONS" notice (not program output)
+  java -cp "$CP" com.ankurm.jackson3.streaminghuge.$c 2>&1 | grep -v '^Picked up' > output/$c.txt
+  echo "wrote output/$c.txt"
+done
+[ "${1:-}" = "--quick" ] && exit 0
+[ -f target/huge.json ] || java -Xmx64m -cp "$CP" com.ankurm.jackson3.streaminghuge.GenerateHugeJson 2>&1 \
+  | grep -v '^Picked up' | tee output/GenerateHugeJson.txt
+# the naive hybrid: shows the trailing-token failure on a small file
+java -Xmx64m -cp "$CP" com.ankurm.jackson3.streaminghuge.GenerateHugeJson target/small.json 5000000 2>&1 | grep -v '^Picked up' > /dev/null
+java -Xmx512m -cp "$CP" com.ankurm.jackson3.streaminghuge.Scan hybrid-naive target/small.json 2>&1 \
+  | grep -v '^Picked up' | head -4 > output/HybridNaive.txt || true
+rm -f target/small.json
+scripts/bench.sh target/huge.json