bufferedreader: read a file line by line module (BufferedReader vs Scanner vs Files.lines vs readAllLines on a 1 GB file; heap at -Xmx64m, timings, read-call counts, one-huge-line OOM, charset/BOM/line-terminator pitfalls, Scanner swallowing IOException), 10 contract tests

Co-Authored-By: Claude Sonnet 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_017ic4b3pDoGLiWk6LLBeHjW
2026-11-15 01:20:37 +05:30
parent 5faa57bb2d
commit f8c9e244a3
25 changed files with 809 additions and 0 deletions
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# Regenerates every file in ../output/. Requires JDK25_HOME. Needs ~1.3 GB of free disk in $DATA (default /tmp/bufferedreader-data)
+# and ~6 GB of RAM for the readAllLines timing run; generated files are deleted on exit.
+# Set BIG_BYTES (default 1000000000) to use a smaller "big" file. Timings are indicative; counts, exceptions and text are exact.
+set -euo pipefail
+[[ -z "${JDK25_HOME:-}" ]] && { echo "JDK25_HOME must be set" >&2; exit 1; }
+cd "$(dirname "$0")/.."
+OUT=output; mkdir -p "$OUT"
+DATA="${DATA:-/tmp/bufferedreader-data}"; mkdir -p "$DATA"
+BIG_BYTES="${BIG_BYTES:-1000000000}"
+MED_BYTES=200000000
+trap 'rm -rf "$DATA"' EXIT
+export JAVA_HOME="$JDK25_HOME"
+mvn -q -f ../pom.xml -pl bufferedreader -am package
+J="$JDK25_HOME/bin/java -Dstdout.encoding=UTF-8"; CP=target/classes; B=com.ankurm.bufferedreader
+strip() { grep -v "Picked up"; }
+
+{ echo '$ uname -sr'; uname -sr; echo '$ nproc'; nproc; echo '$ java -version'; $JAVA_HOME/bin/java -version 2>&1 | strip
+  echo '$ free -m (before)'; free -m | sed 's/  */ /g' | head -2; } > "$OUT/01-environment.txt"
+
+echo "==> generate files"
+{ $J -cp $CP $B.TestData lines "$DATA/big.log" "$BIG_BYTES" 2>&1 | strip
+  $J -cp $CP $B.TestData lines "$DATA/medium.log" "$MED_BYTES" 2>&1 | strip
+  $J -cp $CP $B.TestData oneline "$DATA/long.txt" "$MED_BYTES" 2>&1 | strip
+  echo '$ ls -l (generated files)'; ls -l "$DATA"/*.log "$DATA"/*.txt | awk '{print $5, $9}' | sed "s#$DATA/##"
+  echo '$ head -3 big.log'; head -3 "$DATA/big.log"
+  echo '$ wc -l big.log'; wc -l < "$DATA/big.log"; } > "$OUT/02-generate.txt"
+
+echo "==> which approaches survive -Xmx64m, and how much heap readAllLines needs"
+{ for m in reader scanner lines readAllLines; do echo "\$ java -Xmx64m LineCount $m big.log"; $J -Xmx64m -cp $CP $B.LineCount $m "$DATA/big.log" 2>&1 | strip; done
+  echo; echo "How much heap does readAllLines need for this file?"
+  for x in 1g 2g 3g 4g; do echo "\$ java -Xmx$x LineCount readAllLines big.log"; $J -Xmx$x -cp $CP $B.LineCount readAllLines "$DATA/big.log" 2>&1 | strip; done
+  echo; echo "And the streaming reader at a tiny heap:"
+  echo "\$ java -Xmx16m LineCount reader big.log"; $J -Xmx16m -cp $CP $B.LineCount reader "$DATA/big.log" 2>&1 | strip; } > "$OUT/03-heap.txt"
+
+echo "==> timing, fresh JVM per run, -Xmx5g, 3 rounds"
+{ echo '$ java -Xmx5g LineCount <approach> big.log   (round-robin, 3 rounds, fresh JVM each)'
+  for round in 1 2 3; do echo "-- round $round"
+    for m in reader reader64k scanner lines readAllLines; do $J -Xmx5g -cp $CP $B.LineCount $m "$DATA/big.log" 2>&1 | strip; done; done; } > "$OUT/04-timing.txt"
+
+echo "==> read calls"
+$J -cp $CP $B.ReadCalls "$DATA/medium.log" 2>&1 | strip > "$OUT/05-read-calls.txt"
+
+echo "==> one huge line"
+{ for m in line chunks; do echo "\$ java -Xmx128m LongLineDemo $m long.txt"; $J -Xmx128m -cp $CP $B.LongLineDemo $m "$DATA/long.txt" 2>&1 | strip; done; } > "$OUT/06-long-line.txt"
+
+echo "==> charset, endings, scanner"
+{ echo '$ java CharsetDemo'; $J -cp $CP $B.CharsetDemo 2>&1 | strip; } > "$OUT/07-charset.txt"
+{ echo '$ LC_ALL=C java -Dfile.encoding=COMPAT CharsetDemo'; LC_ALL=C $J -Dfile.encoding=COMPAT -cp $CP $B.CharsetDemo 2>&1 | strip; } > "$OUT/08-charset-compat.txt"
+$J -cp $CP $B.LineEndingsDemo 2>&1 | strip > "$OUT/09-line-endings.txt"
+$J -cp $CP $B.ScannerSwallowsDemo 2>&1 | strip > "$OUT/10-scanner-swallows.txt"
+
+cp target/surefire-reports/com.ankurm.bufferedreader.ReadLineContractTest.txt "$OUT/11-tests.txt"
+echo Done