From 1d0cd8b36e30a3cc9454e1b80122d2356513a676 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 02:03:57 +0000 Subject: [PATCH] streaming-huge-json: 1 GiB streaming vs tree vs binding benchmark (Jackson 3.2.4, JDK 25) for the rewritten streaming post Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01CifR2BE7PBvRy5hT8TS9oz --- README.md | 1 + streaming-huge-json/.gitignore | 1 + streaming-huge-json/README.md | 34 ++++ .../output/GenerateHugeJson.txt | 1 + streaming-huge-json/output/HybridNaive.txt | 3 + streaming-huge-json/output/NestedTrap.txt | 10 ++ streaming-huge-json/output/ReadLimits.txt | 5 + streaming-huge-json/output/TruncatedFile.txt | 13 ++ streaming-huge-json/output/bench.txt | 24 +++ streaming-huge-json/pom.xml | 39 +++++ streaming-huge-json/scripts/bench.sh | 30 ++++ streaming-huge-json/scripts/run-all.sh | 24 +++ .../streaminghuge/GenerateHugeJson.java | 85 ++++++++++ .../jackson3/streaminghuge/NestedTrap.java | 72 +++++++++ .../jackson3/streaminghuge/ReadLimits.java | 18 +++ .../ankurm/jackson3/streaminghuge/Scan.java | 146 ++++++++++++++++++ .../jackson3/streaminghuge/TruncatedFile.java | 54 +++++++ 17 files changed, 560 insertions(+) create mode 100644 streaming-huge-json/.gitignore create mode 100644 streaming-huge-json/README.md create mode 100644 streaming-huge-json/output/GenerateHugeJson.txt create mode 100644 streaming-huge-json/output/HybridNaive.txt create mode 100644 streaming-huge-json/output/NestedTrap.txt create mode 100644 streaming-huge-json/output/ReadLimits.txt create mode 100644 streaming-huge-json/output/TruncatedFile.txt create mode 100644 streaming-huge-json/output/bench.txt create mode 100644 streaming-huge-json/pom.xml create mode 100755 streaming-huge-json/scripts/bench.sh create mode 100755 streaming-huge-json/scripts/run-all.sh create mode 100644 streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/GenerateHugeJson.java create mode 100644 streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/NestedTrap.java create mode 100644 streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/ReadLimits.java create mode 100644 streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/Scan.java create mode 100644 streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/TruncatedFile.java diff --git a/README.md b/README.md index a9c93ae..31e62fa 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ mvn -q exec:java -Dexec.mainClass=com.ankurm.jackson3.part5polymorphic.F01Serial | [Custom Serialisers & Mix-ins](https://ankurm.com/jackson-custom-serializer-mixin/) | [`part4custom`](src/main/java/com/ankurm/jackson3/part4custom) | [E01](src/main/java/com/ankurm/jackson3/part4custom/E01MoneyValueSerializer.java) [E02](src/main/java/com/ankurm/jackson3/part4custom/E02MoneyValueDeserializer.java) [E03](src/main/java/com/ankurm/jackson3/part4custom/E03SimpleModuleRegistration.java) [E04](src/main/java/com/ankurm/jackson3/part4custom/E04MixinAnnotations.java) [E05](src/main/java/com/ankurm/jackson3/part4custom/E05ValueSerializerDirect.java) [E06](src/main/java/com/ankurm/jackson3/part4custom/E06ValueDeserializerDirect.java) [E07](src/main/java/com/ankurm/jackson3/part4custom/E07MixinOnRecord.java) | [write-up](docs/part4-custom.md) (now superseded by the rewritten post itself — see below) | | [Polymorphic Deserialisation](https://ankurm.com/jackson-polymorphic-deserialization/) | [`part5polymorphic`](src/main/java/com/ankurm/jackson3/part5polymorphic) | [F01](src/main/java/com/ankurm/jackson3/part5polymorphic/F01SerialiseMixedList.java) [F02](src/main/java/com/ankurm/jackson3/part5polymorphic/F02DeserialiseMixedList.java) [F03](src/main/java/com/ankurm/jackson3/part5polymorphic/F03IncludeStrategies.java) [F04](src/main/java/com/ankurm/jackson3/part5polymorphic/F04UnknownTypeId.java) [F05](src/main/java/com/ankurm/jackson3/part5polymorphic/F05DefaultTypingModes.java) [F06](src/main/java/com/ankurm/jackson3/part5polymorphic/F06TypeValidation.java) [F07](src/main/java/com/ankurm/jackson3/part5polymorphic/F07SealedHierarchies.java) [F08](src/main/java/com/ankurm/jackson3/part5polymorphic/F08UnknownSubtypes.java) | [write-up](docs/part5-polymorphic.md) (F01–F04; F05–F08 are explained in the rewritten post, output in `docs/output/`) | | [Streaming API & JsonNode](https://ankurm.com/jackson-streaming-api-jsonnode/) | [`part6streaming`](src/main/java/com/ankurm/jackson3/part6streaming) | [G01](src/main/java/com/ankurm/jackson3/part6streaming/G01StreamingParserFilter.java) [G02](src/main/java/com/ankurm/jackson3/part6streaming/G02StreamingGenerator.java) [G03](src/main/java/com/ankurm/jackson3/part6streaming/G03TreeModelNavigation.java) [G04](src/main/java/com/ankurm/jackson3/part6streaming/G04ThreeApproachesMeasured.java) | [write-up](docs/part6-streaming.md) | +| [Streaming API & JsonNode — the 1 GiB benchmark](https://ankurm.com/jackson-streaming-api-jsonnode/) | [`streaming-huge-json`](streaming-huge-json) (standalone module, JDK 25, Jackson 3.2.4) | [GenerateHugeJson](streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/GenerateHugeJson.java) [Scan](streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/Scan.java) [NestedTrap](streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/NestedTrap.java) [TruncatedFile](streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/TruncatedFile.java) [ReadLimits](streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/ReadLimits.java) | [README](streaming-huge-json/README.md), [output/](streaming-huge-json/output) | | [Security Best Practices](https://ankurm.com/jackson-security-best-practices/) | [`part7security`](src/main/java/com/ankurm/jackson3/part7security) | [H01](src/main/java/com/ankurm/jackson3/part7security/H01SafePolymorphismByAnnotation.java) [H02](src/main/java/com/ankurm/jackson3/part7security/H02DefaultTypingRemoved.java) [H03](src/main/java/com/ankurm/jackson3/part7security/H03PolymorphicTypeValidatorAllowlist.java) [H04](src/main/java/com/ankurm/jackson3/part7security/H04StreamReadConstraints.java) [H05](src/main/java/com/ankurm/jackson3/part7security/H05NeverDeserialiseIntoObject.java) | [write-up](docs/part7-security.md) | | — (beyond the posts) | [`beyond`](src/main/java/com/ankurm/jackson3/beyond) | [Y01](src/main/java/com/ankurm/jackson3/beyond/Y01UncheckedExceptions.java) [Y02](src/main/java/com/ankurm/jackson3/beyond/Y02TrailingTokens.java) [Y03](src/main/java/com/ankurm/jackson3/beyond/Y03DateTimeDefaults.java) [Y04](src/main/java/com/ankurm/jackson3/beyond/Y04ImmutableMapperAndReaders.java) [Y05](src/main/java/com/ankurm/jackson3/beyond/Y05CreatorDetection.java) [Y06](src/main/java/com/ankurm/jackson3/beyond/Y06RecyclerPoolTuning.java) | [write-up](docs/beyond.md) | diff --git a/streaming-huge-json/.gitignore b/streaming-huge-json/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/streaming-huge-json/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/streaming-huge-json/README.md b/streaming-huge-json/README.md new file mode 100644 index 0000000..c48473d --- /dev/null +++ b/streaming-huge-json/README.md @@ -0,0 +1,34 @@ +# streaming-huge-json + +Companion code for the rewritten post +[Jackson Streaming API and JsonNode (Jackson 3)](https://ankurm.com/jackson-streaming-api-jsonnode/). + +A self-contained Maven module (not part of the root build). It generates a 1 GiB JSON array, +then counts the `ERROR` records four ways under fixed `-Xmx` limits. + +``` +Jackson 3.2.4 (tools.jackson.core:jackson-databind, GA on Maven Central 2026-10-18) +JDK Temurin 25.0.5 (LTS) +``` + +| Class | What it shows | +|---|---| +| [`GenerateHugeJson`](src/main/java/com/ankurm/jackson3/streaminghuge/GenerateHugeJson.java) | Writing 1 GiB with `JsonGenerator` under `-Xmx64m` | +| [`Scan`](src/main/java/com/ankurm/jackson3/streaminghuge/Scan.java) | `stream`, `hybrid`, `tree`, `bind` (and the broken `hybrid-naive`) | +| [`NestedTrap`](src/main/java/com/ankurm/jackson3/streaminghuge/NestedTrap.java) | The flat-object loop breaking on nested JSON; `skipChildren()` fix | +| [`TruncatedFile`](src/main/java/com/ankurm/jackson3/streaminghuge/TruncatedFile.java) | What a cut-off file does to streaming vs tree | +| [`ReadLimits`](src/main/java/com/ankurm/jackson3/streaminghuge/ReadLimits.java) | Default `StreamReadConstraints` values | + +```bash +cd streaming-huge-json +./scripts/run-all.sh --quick # small demos only +./scripts/run-all.sh # also generates target/huge.json (1 GiB, git-ignored) and runs the ladder +``` + +Captured output is in [`output/`](output). Memory figures are from one run on a 2-CPU, 8 GB +container; `peakUsed` is sampled `total - free` heap and includes garbage not yet collected. +Timings vary between runs; the pass/fail pattern per `-Xmx` is the stable part. + +Note on `output/bench.txt`: the `tree -Xmx= 6144 MB KILLED` line comes from a separate manual run +(the first ladder run printed a blank line for it because the OS killed the JVM, exit 137); +`scripts/bench.sh` now records that case itself. diff --git a/streaming-huge-json/output/GenerateHugeJson.txt b/streaming-huge-json/output/GenerateHugeJson.txt new file mode 100644 index 0000000..2a75f76 --- /dev/null +++ b/streaming-huge-json/output/GenerateHugeJson.txt @@ -0,0 +1 @@ +wrote target/regen.json : 1,073,741,995 bytes (1.00 GiB), 4,088,325 records, 6,279 ms diff --git a/streaming-huge-json/output/HybridNaive.txt b/streaming-huge-json/output/HybridNaive.txt new file mode 100644 index 0000000..d6bebbb --- /dev/null +++ b/streaming-huge-json/output/HybridNaive.txt @@ -0,0 +1,3 @@ +Exception in thread "main" tools.jackson.databind.exc.MismatchedInputException: Trailing token (`JsonToken.START_OBJECT`) found after value (bound as `com.ankurm.jackson3.streaminghuge.Scan$LogEntry`): not allowed as per `DeserializationFeature.FAIL_ON_TRAILING_TOKENS` + at [Source: REDACTED (`StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION` disabled); byte offset: #258] + at tools.jackson.databind.exc.MismatchedInputException.from(MismatchedInputException.java:70) diff --git a/streaming-huge-json/output/NestedTrap.txt b/streaming-huge-json/output/NestedTrap.txt new file mode 100644 index 0000000..d83f852 --- /dev/null +++ b/streaming-huge-json/output/NestedTrap.txt @@ -0,0 +1,10 @@ +expected: 2 ERROR records + +-- original loop (flat objects only) -- +ERROR: null +guard tripped (inner loop) + +-- with skipChildren() -- +ERROR: database down +ERROR: disk full +Total errors found: 2 diff --git a/streaming-huge-json/output/ReadLimits.txt b/streaming-huge-json/output/ReadLimits.txt new file mode 100644 index 0000000..ceb5b23 --- /dev/null +++ b/streaming-huge-json/output/ReadLimits.txt @@ -0,0 +1,5 @@ +maxNestingDepth = 500 +maxStringLength = 100000000 +maxNumberLength = 1000 +maxNameLength = 50000 +maxDocumentLength = -1 (-1 = unlimited) diff --git a/streaming-huge-json/output/TruncatedFile.txt b/streaming-huge-json/output/TruncatedFile.txt new file mode 100644 index 0000000..2153372 --- /dev/null +++ b/streaming-huge-json/output/TruncatedFile.txt @@ -0,0 +1,13 @@ +input (truncated): [{"id":1,"level":"ERROR"},{"id":2,"level":"INFO"},{"id":3,"level":"ERROR"}, + +-- streaming -- +handled an ERROR record, running total 1 +handled an ERROR record, running total 2 +exception : tools.jackson.core.exc.StreamReadException +is unchecked: true +first line: Unexpected end-of-input within/between Array entries +records already handled before the exception: 2 + +-- readTree -- +exception : tools.jackson.core.exc.StreamReadException +records handled: 0 (no tree was returned) diff --git a/streaming-huge-json/output/bench.txt b/streaming-huge-json/output/bench.txt new file mode 100644 index 0000000..f1005fc --- /dev/null +++ b/streaming-huge-json/output/bench.txt @@ -0,0 +1,24 @@ +file: target/huge.json (1073741995 bytes) +java: openjdk version "25.0.5" 2026-10-20 LTS +cpus: 2 ram: 8027 MB + +stream -Xmx= 32 MB OK records=4,088,325 errors=81,728 errLatencySum=81,712,926 8,898 ms peakUsed= 19 MB +stream -Xmx= 64 MB OK records=4,088,325 errors=81,728 errLatencySum=81,712,926 9,654 ms peakUsed= 38 MB +stream -Xmx= 128 MB OK records=4,088,325 errors=81,728 errLatencySum=81,712,926 10,482 ms peakUsed= 56 MB +stream -Xmx= 1024 MB OK records=4,088,325 errors=81,728 errLatencySum=81,712,926 12,047 ms peakUsed= 56 MB +hybrid -Xmx= 32 MB OK records=4,088,325 errors=81,728 errLatencySum=81,712,926 19,128 ms peakUsed= 20 MB +hybrid -Xmx= 64 MB OK records=4,088,325 errors=81,728 errLatencySum=81,712,926 20,717 ms peakUsed= 39 MB +hybrid -Xmx= 128 MB OK records=4,088,325 errors=81,728 errLatencySum=81,712,926 29,960 ms peakUsed= 57 MB +hybrid -Xmx= 1024 MB OK records=4,088,325 errors=81,728 errLatencySum=81,712,926 30,921 ms peakUsed= 122 MB +tree -Xmx= 64 MB FAIL java.lang.OutOfMemoryError: Java heap space after 4,278 ms +tree -Xmx= 256 MB FAIL java.lang.OutOfMemoryError: Java heap space after 16,122 ms +tree -Xmx= 1024 MB FAIL java.lang.OutOfMemoryError: Java heap space after 69,293 ms +tree -Xmx= 2048 MB FAIL java.lang.OutOfMemoryError: Java heap space after 161,614 ms +tree -Xmx= 3072 MB FAIL java.lang.OutOfMemoryError: Java heap space after 82,230 ms +tree -Xmx= 4096 MB FAIL java.lang.OutOfMemoryError: Java heap space after 78,019 ms +tree -Xmx= 6144 MB KILLED no Java output, exit=137 (SIGKILL by the OS: the heap limit was larger than free RAM) +bind -Xmx= 64 MB FAIL java.lang.OutOfMemoryError: Java heap space after 6,245 ms +bind -Xmx= 256 MB FAIL java.lang.OutOfMemoryError: Java heap space after 22,555 ms +bind -Xmx= 1024 MB FAIL java.lang.OutOfMemoryError: Java heap space after 124,703 ms +bind -Xmx= 2048 MB FAIL java.lang.OutOfMemoryError: Java heap space after 177,038 ms +bind -Xmx= 3072 MB OK records=4,088,325 errors=81,728 errLatencySum=81,712,926 32,180 ms peakUsed=2,943 MB diff --git a/streaming-huge-json/pom.xml b/streaming-huge-json/pom.xml new file mode 100644 index 0000000..75081de --- /dev/null +++ b/streaming-huge-json/pom.xml @@ -0,0 +1,39 @@ + + + 4.0.0 + + + com.ankurm + streaming-huge-json + 1.0.0 + jar + streaming-huge-json + + + 25 + UTF-8 + 3.2.4 + + + + + + tools.jackson + jackson-bom + ${jackson.version} + pom + import + + + + + + + tools.jackson.core + jackson-databind + + + diff --git a/streaming-huge-json/scripts/bench.sh b/streaming-huge-json/scripts/bench.sh new file mode 100755 index 0000000..bc2b6ce --- /dev/null +++ b/streaming-huge-json/scripts/bench.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Runs each approach against the huge file under a ladder of -Xmx limits, one fresh JVM per run. +# Streaming modes run every rung (to show memory stays flat). Tree/bind climb until the first +# rung that succeeds, then stop. Output: output/bench.txt (timings vary run to run) +set -u +cd "$(dirname "$0")/.." +FILE="${1:-target/huge.json}" +CP="target/classes:$(cat target/cp.txt)" +OUT=output/bench.txt +mkdir -p output +{ + echo "file: $FILE ($(stat -c %s "$FILE") bytes)" + echo "java: $(java -version 2>&1 | grep -v 'Picked up' | head -1)" + echo "cpus: $(nproc) ram: $(free -m | awk '/Mem:/{print $2}') MB" + echo + for mode in stream hybrid; do + for x in 32 64 128 1024; do + java -Xmx${x}m -cp "$CP" com.ankurm.jackson3.streaminghuge.Scan $mode "$FILE" 2>&1 | grep -v '^Picked up' + done + done + for mode in tree bind; do + for x in 64 256 1024 2048 3072 4096 6144; do + line=$(timeout 900 java -Xmx${x}m -cp "$CP" com.ankurm.jackson3.streaminghuge.Scan $mode "$FILE" 2>&1 | grep -v '^Picked up'; echo "exit=${PIPESTATUS[0]}") + # no output at all = the JVM was SIGKILLed (exit 137) by the container's OOM killer + case "$line" in exit=*) line=$(printf '%-7s -Xmx=%5d MB KILLED no Java output, %s (SIGKILL by the OS: the heap limit was larger than free RAM)' "$mode" "$x" "$line");; *) line="${line%exit=*}"; line="${line%$'\n'}";; esac + echo "$line" + case "$line" in *" OK "*) break;; esac + done + done +} | tee "$OUT" diff --git a/streaming-huge-json/scripts/run-all.sh b/streaming-huge-json/scripts/run-all.sh new file mode 100755 index 0000000..d1780cc --- /dev/null +++ b/streaming-huge-json/scripts/run-all.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Builds the module, runs the small demos (captured to output/), generates the 1 GiB file +# (NOT committed: target/ is git-ignored) and runs the memory ladder. Needs a JDK 25 on PATH. +# ./scripts/run-all.sh full run, ~5-10 minutes +# ./scripts/run-all.sh --quick small demos only, no 1 GiB file +set -eu +cd "$(dirname "$0")/.." +mvn -B -q compile +mvn -B -q dependency:build-classpath -Dmdep.outputFile=target/cp.txt +CP="target/classes:$(cat target/cp.txt)" +for c in NestedTrap TruncatedFile ReadLimits; do + # grep -v strips the sandbox's injected "Picked up JAVA_TOOL_OPTIONS" notice (not program output) + java -cp "$CP" com.ankurm.jackson3.streaminghuge.$c 2>&1 | grep -v '^Picked up' > output/$c.txt + echo "wrote output/$c.txt" +done +[ "${1:-}" = "--quick" ] && exit 0 +[ -f target/huge.json ] || java -Xmx64m -cp "$CP" com.ankurm.jackson3.streaminghuge.GenerateHugeJson 2>&1 \ + | grep -v '^Picked up' | tee output/GenerateHugeJson.txt +# the naive hybrid: shows the trailing-token failure on a small file +java -Xmx64m -cp "$CP" com.ankurm.jackson3.streaminghuge.GenerateHugeJson target/small.json 5000000 2>&1 | grep -v '^Picked up' > /dev/null +java -Xmx512m -cp "$CP" com.ankurm.jackson3.streaminghuge.Scan hybrid-naive target/small.json 2>&1 \ + | grep -v '^Picked up' | head -4 > output/HybridNaive.txt || true +rm -f target/small.json +scripts/bench.sh target/huge.json diff --git a/streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/GenerateHugeJson.java b/streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/GenerateHugeJson.java new file mode 100644 index 0000000..4282dc9 --- /dev/null +++ b/streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/GenerateHugeJson.java @@ -0,0 +1,85 @@ +package com.ankurm.jackson3.streaminghuge; + +import tools.jackson.core.ObjectWriteContext; +import tools.jackson.core.json.JsonFactory; +import tools.jackson.core.JsonGenerator; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.OutputStream; +import java.util.SplittableRandom; + +/** + * Writes a huge JSON array of log records using the streaming JsonGenerator, so the + * generator itself needs almost no heap. Deterministic (fixed seed): every run produces + * the same bytes, so every approach in {@link Scan} must report the same counts. + * + * Usage: GenerateHugeJson [file] [targetBytes] + * defaults: target/huge.json 1073741824 (1 GiB) + */ +public class GenerateHugeJson { + + static final String[] SERVICES = {"auth", "billing", "cart", "catalog", "checkout", "gateway", + "inventory", "mailer", "orders", "search", "shipping", "users"}; + static final String[] PLANS = {"free", "pro", "team", "enterprise"}; + static final String[] MESSAGES = { + "request completed", "cache miss, loading from database", "connection pool at 80 percent", + "retrying upstream call", "payment authorised", "session refreshed", "slow query detected", + "user profile updated", "background job finished", "rate limit approaching"}; + + public static void main(String[] args) throws Exception { + File file = new File(args.length > 0 ? args[0] : "target/huge.json"); + long target = args.length > 1 ? Long.parseLong(args[1]) : 1L << 30; + file.getAbsoluteFile().getParentFile().mkdirs(); + + SplittableRandom rnd = new SplittableRandom(42); + JsonFactory factory = new JsonFactory(); + long start = System.nanoTime(); + long id = 0; + + CountingStream counter = new CountingStream(new BufferedOutputStream(new FileOutputStream(file), 1 << 16)); + try (OutputStream out = counter; + JsonGenerator g = factory.createGenerator(ObjectWriteContext.empty(), out)) { + g.writeStartArray(); + while (counter.count + g.streamWriteOutputBuffered() < target) { + int r = rnd.nextInt(100); + String level = r < 2 ? "ERROR" : r < 10 ? "WARN" : "INFO"; + g.writeStartObject(); + g.writeNumberProperty("id", id); + g.writeStringProperty("ts", "2026-10-" + String.format("%02d", 1 + (int) (id % 28)) + + "T" + String.format("%02d:%02d:%02d", (int) (id % 24), (int) (id % 60), (int) (id % 53)) + "Z"); + g.writeStringProperty("level", level); + g.writeStringProperty("service", SERVICES[rnd.nextInt(SERVICES.length)]); + g.writeStringProperty("host", "node-" + rnd.nextInt(64)); + g.writeStringProperty("message", MESSAGES[rnd.nextInt(MESSAGES.length)] + + " [req=" + Long.toHexString(rnd.nextLong()) + "]"); + g.writeNumberProperty("latencyMs", rnd.nextInt(2000)); + g.writeName("user"); + g.writeStartObject(); + g.writeNumberProperty("id", rnd.nextInt(5_000_000)); + g.writeStringProperty("name", "user" + rnd.nextInt(100_000)); + g.writeStringProperty("plan", PLANS[rnd.nextInt(PLANS.length)]); + g.writeEndObject(); + g.writeName("tags"); + g.writeStartArray(); + for (int t = 0; t < 3; t++) g.writeString("tag-" + rnd.nextInt(500)); + g.writeEndArray(); + g.writeEndObject(); + id++; + } + g.writeEndArray(); + } + long ms = (System.nanoTime() - start) / 1_000_000; + System.out.printf("wrote %s : %,d bytes (%.2f GiB), %,d records, %,d ms%n", + file, file.length(), file.length() / (double) (1L << 30), id, ms); + } + + /** Counts bytes that have actually been handed to the file. */ + static final class CountingStream extends java.io.FilterOutputStream { + long count; + CountingStream(OutputStream out) { super(out); } + @Override public void write(int b) throws java.io.IOException { out.write(b); count++; } + @Override public void write(byte[] b, int off, int len) throws java.io.IOException { out.write(b, off, len); count += len; } + } +} diff --git a/streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/NestedTrap.java b/streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/NestedTrap.java new file mode 100644 index 0000000..c7128c1 --- /dev/null +++ b/streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/NestedTrap.java @@ -0,0 +1,72 @@ +package com.ankurm.jackson3.streaminghuge; + +import tools.jackson.core.JsonParser; +import tools.jackson.core.JsonToken; +import tools.jackson.core.ObjectReadContext; +import tools.jackson.core.json.JsonFactory; + +/** + * The classic streaming bug. The post's original loop ("walk each field until END_OBJECT") + * is correct for FLAT objects. The moment a record contains a nested object or array, the + * nested END_OBJECT ends the inner loop early and the parser state is one level out of step. + * This runs the original loop and the skipChildren() version over the same nested input. + */ +public class NestedTrap { + + static final String JSON = """ + [{"level":"INFO","user":{"id":1},"message":"started"}, + {"level":"ERROR","user":{"id":2},"message":"database down"}, + {"level":"ERROR","user":{"id":3},"message":"disk full"}]"""; + + public static void main(String[] args) { + System.out.println("expected: 2 ERROR records"); + System.out.println(); + System.out.println("-- original loop (flat objects only) --"); + try { original(); } catch (RuntimeException e) { + System.out.println("threw " + e.getClass().getName()); + } + System.out.println(); + System.out.println("-- with skipChildren() --"); + fixed(); + } + + static void original() { + int errorCount = 0; + int guard = 0; // not in the original: without it this loop never terminates (see the output) + try (JsonParser parser = new JsonFactory().createParser(ObjectReadContext.empty(), JSON)) { + parser.nextToken(); + while (parser.nextToken() != JsonToken.END_ARRAY) { + if (++guard > 20) { System.out.println("guard tripped: nextToken() keeps returning null at end of input; the original loop would spin forever"); break; } + String logLevel = null, logMessage = null; + while (parser.nextToken() != JsonToken.END_OBJECT) { + if (++guard > 200) { System.out.println("guard tripped (inner loop)"); return; } + String fieldName = parser.currentName(); + parser.nextToken(); + if ("level".equals(fieldName)) logLevel = parser.getString(); + else if ("message".equals(fieldName)) logMessage = parser.getString(); + } + if ("ERROR".equals(logLevel)) { System.out.println("ERROR: " + logMessage); errorCount++; } + } + } + System.out.println("Total errors found: " + errorCount); + } + + static void fixed() { + int errorCount = 0; + try (JsonParser parser = new JsonFactory().createParser(ObjectReadContext.empty(), JSON)) { + parser.nextToken(); + while (parser.nextToken() == JsonToken.START_OBJECT) { + String logLevel = null, logMessage = null; + while (parser.nextToken() == JsonToken.PROPERTY_NAME) { + String fieldName = parser.currentName(); + parser.nextToken(); + if ("level".equals(fieldName)) logLevel = parser.getString(); + else if ("message".equals(fieldName)) logMessage = parser.getString(); + else parser.skipChildren(); // no-op for scalars, skips a whole {..} or [..] + } + if ("ERROR".equals(logLevel)) { System.out.println("ERROR: " + logMessage); errorCount++; } + } + } + System.out.println("Total errors found: " + errorCount); + } +} diff --git a/streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/ReadLimits.java b/streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/ReadLimits.java new file mode 100644 index 0000000..f8fb443 --- /dev/null +++ b/streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/ReadLimits.java @@ -0,0 +1,18 @@ +package com.ankurm.jackson3.streaminghuge; + +import tools.jackson.core.StreamReadConstraints; + +/** + * Jackson 3 ships guard rails on what a parser will accept. None of them limits the total SIZE of a + * document by default, which is why a 1 GB file parses fine. Printed from the library, not from memory. + */ +public class ReadLimits { + public static void main(String[] args) { + StreamReadConstraints c = StreamReadConstraints.defaults(); + System.out.println("maxNestingDepth = " + c.getMaxNestingDepth()); + System.out.println("maxStringLength = " + c.getMaxStringLength()); + System.out.println("maxNumberLength = " + c.getMaxNumberLength()); + System.out.println("maxNameLength = " + c.getMaxNameLength()); + System.out.println("maxDocumentLength = " + c.getMaxDocumentLength() + " (-1 = unlimited)"); + } +} diff --git a/streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/Scan.java b/streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/Scan.java new file mode 100644 index 0000000..8cb5d6a --- /dev/null +++ b/streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/Scan.java @@ -0,0 +1,146 @@ +package com.ankurm.jackson3.streaminghuge; + +import tools.jackson.core.JsonParser; +import tools.jackson.core.JsonToken; +import tools.jackson.core.ObjectReadContext; +import tools.jackson.core.json.JsonFactory; +import tools.jackson.core.type.TypeReference; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.json.JsonMapper; + +import java.io.File; +import java.util.List; + +/** + * The same job four ways (plus one deliberately broken variant): count the ERROR records in a huge JSON array and sum their latency. + * + * stream JsonParser, token by token, skipChildren() for fields we do not need + * hybrid JsonParser walks the array; each element is bound to a record, then discarded + * tree JsonMapper.readTree(file) -> the whole document as JsonNode objects + * bind JsonMapper.readValue(file, List) -> the whole document as records + * + * Usage: Scan {stream|hybrid|hybrid-naive|tree|bind} file + * Prints exactly one result line. A sampler thread records the peak of (total - free) heap, + * which INCLUDES garbage not yet collected, so for the streaming modes it overstates the + * live data. Run with different -Xmx values; see scripts/bench.sh. + */ +public class Scan { + + public record User(long id, String name, String plan) { } + public record LogEntry(long id, String ts, String level, String service, String host, + String message, int latencyMs, User user, List tags) { } + + static volatile long peakUsed; + + public static void main(String[] args) throws Exception { + String mode = args[0]; + File file = new File(args[1]); + Runtime rt = Runtime.getRuntime(); + Thread sampler = new Thread(() -> { + while (true) { + long used = rt.totalMemory() - rt.freeMemory(); + if (used > peakUsed) peakUsed = used; + try { Thread.sleep(10); } catch (InterruptedException e) { return; } + } + }); + sampler.setDaemon(true); + sampler.start(); + + long start = System.nanoTime(); + try { + long[] r = switch (mode) { + case "stream" -> stream(file); + case "hybrid" -> hybrid(file); + case "hybrid-naive" -> hybridNaive(file); + case "tree" -> tree(file); + case "bind" -> bind(file); + default -> throw new IllegalArgumentException(mode); + }; + long ms = (System.nanoTime() - start) / 1_000_000; + System.out.printf("%-7s -Xmx=%5d MB OK records=%,d errors=%,d errLatencySum=%,d %,7d ms peakUsed=%,5d MB%n", + mode, rt.maxMemory() >> 20, r[0], r[1], r[2], ms, peakUsed >> 20); + } catch (OutOfMemoryError oom) { + long ms = (System.nanoTime() - start) / 1_000_000; + // Drop references before printing so the message itself can allocate. + System.out.printf("%-7s -Xmx=%5d MB FAIL java.lang.OutOfMemoryError: %s after %,d ms%n", + mode, rt.maxMemory() >> 20, oom.getMessage(), ms); + System.exit(3); + } + } + + /** Pure streaming: no object is ever built for a record. */ + static long[] stream(File file) { + long records = 0, errors = 0, latency = 0; + try (JsonParser p = new JsonFactory().createParser(ObjectReadContext.empty(), file)) { + if (p.nextToken() != JsonToken.START_ARRAY) throw new IllegalStateException("expected array"); + while (p.nextToken() == JsonToken.START_OBJECT) { // one record + records++; + String level = null; + int lat = 0; + while (p.nextToken() == JsonToken.PROPERTY_NAME) { // one field of the record + String name = p.currentName(); + p.nextToken(); // move onto the value + switch (name) { + case "level" -> level = p.getString(); + case "latencyMs" -> lat = p.getIntValue(); + default -> p.skipChildren(); // nested user{} / tags[] skipped whole + } + } + if ("ERROR".equals(level)) { errors++; latency += lat; } + } + } + return new long[]{records, errors, latency}; + } + + /** + * The obvious hybrid, and a trap: mapper.readValue(parser, type) is a whole-document call. After the + * first record it sees the next START_OBJECT and, because FAIL_ON_TRAILING_TOKENS is on, throws. + */ + static long[] hybridNaive(File file) { + JsonMapper mapper = JsonMapper.builder().build(); + long records = 0, errors = 0, latency = 0; + try (JsonParser p = mapper.createParser(file)) { + if (p.nextToken() != JsonToken.START_ARRAY) throw new IllegalStateException("expected array"); + while (p.nextToken() == JsonToken.START_OBJECT) { + LogEntry e = mapper.readValue(p, LogEntry.class); + records++; + if ("ERROR".equals(e.level())) { errors++; latency += e.latencyMs(); } + } + } + return new long[]{records, errors, latency}; + } + + /** Streaming over the array, data binding per element: convenient and still flat on memory. */ + static long[] hybrid(File file) { + JsonMapper mapper = JsonMapper.builder().build(); + long records = 0, errors = 0, latency = 0; + try (JsonParser p = mapper.createParser(file)) { + if (p.nextToken() != JsonToken.START_ARRAY) throw new IllegalStateException("expected array"); + while (p.nextToken() == JsonToken.START_OBJECT) { + LogEntry e = p.readValueAs(LogEntry.class); // binds exactly this object, nothing after it + records++; + if ("ERROR".equals(e.level())) { errors++; latency += e.latencyMs(); } + } + } + return new long[]{records, errors, latency}; + } + + /** Whole document as a tree. */ + static long[] tree(File file) { + JsonNode root = JsonMapper.builder().build().readTree(file); + long records = 0, errors = 0, latency = 0; + for (JsonNode n : root) { + records++; + if ("ERROR".equals(n.path("level").asString())) { errors++; latency += n.path("latencyMs").asInt(); } + } + return new long[]{records, errors, latency}; + } + + /** Whole document as a List of records. */ + static long[] bind(File file) { + List all = JsonMapper.builder().build().readValue(file, new TypeReference>() { }); + long errors = 0, latency = 0; + for (LogEntry e : all) if ("ERROR".equals(e.level())) { errors++; latency += e.latencyMs(); } + return new long[]{all.size(), errors, latency}; + } +} diff --git a/streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/TruncatedFile.java b/streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/TruncatedFile.java new file mode 100644 index 0000000..26b8f1a --- /dev/null +++ b/streaming-huge-json/src/main/java/com/ankurm/jackson3/streaminghuge/TruncatedFile.java @@ -0,0 +1,54 @@ +package com.ankurm.jackson3.streaminghuge; + +import tools.jackson.core.JacksonException; +import tools.jackson.core.JsonParser; +import tools.jackson.core.JsonToken; +import tools.jackson.core.ObjectReadContext; +import tools.jackson.core.json.JsonFactory; +import tools.jackson.databind.json.JsonMapper; + +/** + * A streaming parser acts on records before it has seen the end of the file. If the file is cut + * off (a failed upload, a full disk) you have ALREADY processed the early records when the + * exception arrives. A tree or data-binding read fails before returning anything. + * Also shows that Jackson 3's exceptions are unchecked (JacksonException extends RuntimeException). + */ +public class TruncatedFile { + + public static void main(String[] args) { + String full = """ + [{"id":1,"level":"ERROR"},{"id":2,"level":"INFO"},{"id":3,"level":"ERROR"},{"id":4,"level":"ERROR"}]"""; + String cut = full.substring(0, full.length() - 25); + System.out.println("input (truncated): " + cut); + System.out.println(); + + System.out.println("-- streaming --"); + int errors = 0; + try (JsonParser p = new JsonFactory().createParser(ObjectReadContext.empty(), cut)) { + p.nextToken(); + while (p.nextToken() == JsonToken.START_OBJECT) { + String level = null; + while (p.nextToken() == JsonToken.PROPERTY_NAME) { + String n = p.currentName(); + p.nextToken(); + if ("level".equals(n)) level = p.getString(); + } + if ("ERROR".equals(level)) { errors++; System.out.println("handled an ERROR record, running total " + errors); } + } + } catch (JacksonException e) { + System.out.println("exception : " + e.getClass().getName()); + System.out.println("is unchecked: " + (e instanceof RuntimeException)); + System.out.println("first line: " + e.getOriginalMessage()); + System.out.println("records already handled before the exception: " + errors); + } + + System.out.println(); + System.out.println("-- readTree --"); + try { + JsonMapper.builder().build().readTree(cut); + } catch (JacksonException e) { + System.out.println("exception : " + e.getClass().getName()); + System.out.println("records handled: 0 (no tree was returned)"); + } + } +}