From f8c9e244a380d180baa74779b6e9772776fffdeb Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 15 Nov 2026 01:20:37 +0530 Subject: [PATCH] bufferedreader: read a file line by line module (BufferedReader vs Scanner vs Files.lines vs readAllLines on a 1 GB file; heap at -Xmx64m, timings, read-call counts, one-huge-line OOM, charset/BOM/line-terminator pitfalls, Scanner swallowing IOException), 10 contract tests Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_017ic4b3pDoGLiWk6LLBeHjW --- README.md | 1 + bufferedreader/README.md | 41 +++++++ bufferedreader/output/01-environment.txt | 11 ++ bufferedreader/output/02-generate.txt | 13 +++ bufferedreader/output/03-heap.txt | 22 ++++ bufferedreader/output/04-timing.txt | 19 +++ bufferedreader/output/05-read-calls.txt | 17 +++ bufferedreader/output/06-long-line.txt | 6 + bufferedreader/output/07-charset.txt | 22 ++++ bufferedreader/output/08-charset-compat.txt | 22 ++++ bufferedreader/output/09-line-endings.txt | 10 ++ bufferedreader/output/10-scanner-swallows.txt | 9 ++ bufferedreader/output/11-tests.txt | 4 + bufferedreader/pom.xml | 43 +++++++ bufferedreader/scripts/run-all.sh | 54 +++++++++ .../ankurm/bufferedreader/CharsetDemo.java | 76 ++++++++++++ .../java/com/ankurm/bufferedreader/Esc.java | 17 +++ .../com/ankurm/bufferedreader/LineCount.java | 84 ++++++++++++++ .../bufferedreader/LineEndingsDemo.java | 38 ++++++ .../ankurm/bufferedreader/LongLineDemo.java | 36 ++++++ .../com/ankurm/bufferedreader/ReadCalls.java | 66 +++++++++++ .../bufferedreader/ScannerSwallowsDemo.java | 46 ++++++++ .../com/ankurm/bufferedreader/TestData.java | 43 +++++++ .../bufferedreader/ReadLineContractTest.java | 108 ++++++++++++++++++ pom.xml | 1 + 25 files changed, 809 insertions(+) create mode 100644 bufferedreader/README.md create mode 100644 bufferedreader/output/01-environment.txt create mode 100644 bufferedreader/output/02-generate.txt create mode 100644 bufferedreader/output/03-heap.txt create mode 100644 bufferedreader/output/04-timing.txt create mode 100644 bufferedreader/output/05-read-calls.txt create mode 100644 bufferedreader/output/06-long-line.txt create mode 100644 bufferedreader/output/07-charset.txt create mode 100644 bufferedreader/output/08-charset-compat.txt create mode 100644 bufferedreader/output/09-line-endings.txt create mode 100644 bufferedreader/output/10-scanner-swallows.txt create mode 100644 bufferedreader/output/11-tests.txt create mode 100644 bufferedreader/pom.xml create mode 100755 bufferedreader/scripts/run-all.sh create mode 100644 bufferedreader/src/main/java/com/ankurm/bufferedreader/CharsetDemo.java create mode 100644 bufferedreader/src/main/java/com/ankurm/bufferedreader/Esc.java create mode 100644 bufferedreader/src/main/java/com/ankurm/bufferedreader/LineCount.java create mode 100644 bufferedreader/src/main/java/com/ankurm/bufferedreader/LineEndingsDemo.java create mode 100644 bufferedreader/src/main/java/com/ankurm/bufferedreader/LongLineDemo.java create mode 100644 bufferedreader/src/main/java/com/ankurm/bufferedreader/ReadCalls.java create mode 100644 bufferedreader/src/main/java/com/ankurm/bufferedreader/ScannerSwallowsDemo.java create mode 100644 bufferedreader/src/main/java/com/ankurm/bufferedreader/TestData.java create mode 100644 bufferedreader/src/test/java/com/ankurm/bufferedreader/ReadLineContractTest.java diff --git a/README.md b/README.md index db8905f..de79837 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ article; each module's own README has that article's version table, quickstart, | [`equals-hashcode`](equals-hashcode/) | equals() and hashCode() in Java: The Contract, Real Bugs and How Records Change It | | [`copying`](copying/) | Deep Copy vs Shallow Copy in Java: clone(), Copy Constructors, Serialization and Records | | [`datetimeformatter`](datetimeformatter/) | DateTimeFormatter Cheat Sheet: Patterns, the yyyy vs YYYY Bug, Locales and ISO Formats | +| [`bufferedreader`](bufferedreader/) | Java BufferedReader: Read a File Line by Line (vs Scanner, Files.lines and readAllLines) | ## License diff --git a/bufferedreader/README.md b/bufferedreader/README.md new file mode 100644 index 0000000..d4f1f79 --- /dev/null +++ b/bufferedreader/README.md @@ -0,0 +1,41 @@ +# bufferedreader + +Companion code for the ankurm.com post *"Java BufferedReader: Read a File Line by Line (vs Scanner, Files.lines and readAllLines)."* +Module `bufferedreader` in `java-core-examples`. The explanation lives in the post; this module holds the runnable evidence and the captured output. + +## Versions + +| Component | Version | +|---|---| +| JDK | 25.0.5+8 (Temurin, LTS) | +| JUnit Jupiter | 5.11.0 | +| OS | Linux 6.18, 2 vCPU x86-64 VM (timings are indicative, not a leaderboard) | +| Big file | 1,000,000,064 bytes, 9,623,347 ASCII log lines, generated then deleted by the script | + +## Quickstart + +```bash +export JDK25_HOME=/path/to/jdk-25 +./scripts/run-all.sh # needs ~1.3 GB free in /tmp/bufferedreader-data and ~6 GB RAM; regenerates output/ +BIG_BYTES=200000000 ./scripts/run-all.sh # smaller "big" file if disk or RAM is tight +mvn -q -pl bufferedreader -am test # 10 assertions, tiny files only +``` + +## What is in here + +| File | Shows | Output | +|---|---|---| +| `TestData` | deterministic generator: ~100-byte log lines, or one giant line with no newline | `02` | +| `LineCount` | `BufferedReader`, `Scanner`, `Files.lines`, `Files.readAllLines` on the same file: which survive `-Xmx64m`, what `readAllLines` needs, time and GC | `03`, `04` | +| `ReadCalls` | read calls that reach the underlying stream, and per-character vs `readLine()` time | `05` | +| `LongLineDemo` | `readLine()` on one 200 MB line at `-Xmx128m` vs chunked `read(char[])` | `06` | +| `CharsetDemo` | invalid UTF-8 through five readers, the JDK 18+ default charset, `-Dfile.encoding=COMPAT`, and a UTF-8 BOM | `07`, `08` | +| `LineEndingsDemo` | `\n`, `\r\n`, `\r`, final newline, empty file: what counts as a line | `09` | +| `ScannerSwallowsDemo` | a reader that fails mid-file: `BufferedReader` throws, `Scanner` looks like end of file | `10` | +| `ReadLineContractTest` | 10 assertions behind the claims above | `11` | + +`01` records the environment. Timings move with whatever else the machine is doing; counts, exceptions, line totals and heap outcomes do not. + +## License + +MIT - see [LICENSE](../LICENSE). diff --git a/bufferedreader/output/01-environment.txt b/bufferedreader/output/01-environment.txt new file mode 100644 index 0000000..d997537 --- /dev/null +++ b/bufferedreader/output/01-environment.txt @@ -0,0 +1,11 @@ +$ uname -sr +Linux 6.18.44-fc-v470 +$ nproc +2 +$ java -version +openjdk version "25.0.5" 2026-10-20 LTS +OpenJDK Runtime Environment Temurin-25.0.5+8 (build 25.0.5+8-LTS) +OpenJDK 64-Bit Server VM Temurin-25.0.5+8 (build 25.0.5+8-LTS, mixed mode, sharing) +$ free -m (before) + total used free shared buff/cache available +Mem: 8024 615 6866 16 784 7409 diff --git a/bufferedreader/output/02-generate.txt b/bufferedreader/output/02-generate.txt new file mode 100644 index 0000000..a08aa1c --- /dev/null +++ b/bufferedreader/output/02-generate.txt @@ -0,0 +1,13 @@ +generated big.log: 1,000,000,064 bytes in 9620 ms +generated medium.log: 200,000,066 bytes in 2297 ms +generated long.txt: 200,000,071 bytes in 237 ms +$ ls -l (generated files) +1000000064 big.log +200000071 long.txt +200000066 medium.log +$ head -3 big.log +2026-01-01T00:00:00.000Z INFO [worker-00] request id=000000000 path=/api/v1/items/0 status=500 ms=0 +2026-01-01T00:00:01.001Z INFO [worker-01] request id=000000001 path=/api/v1/items/1 status=200 ms=1 +2026-01-01T00:00:02.002Z INFO [worker-02] request id=000000002 path=/api/v1/items/2 status=200 ms=2 +$ wc -l big.log +9623347 diff --git a/bufferedreader/output/03-heap.txt b/bufferedreader/output/03-heap.txt new file mode 100644 index 0000000..e489f97 --- /dev/null +++ b/bufferedreader/output/03-heap.txt @@ -0,0 +1,22 @@ +$ java -Xmx64m LineCount reader big.log +reader lines=9,623,347 chars=990,376,717 time=1,288 ms gc=37 collections / 31 ms (-Xmx64 MB) +$ java -Xmx64m LineCount scanner big.log +scanner lines=9,623,347 chars=990,376,717 time=6,233 ms gc=140 collections / 116 ms (-Xmx64 MB) +$ java -Xmx64m LineCount lines big.log +lines lines=9,623,347 chars=990,376,717 time=1,179 ms gc=37 collections / 32 ms (-Xmx64 MB) +$ java -Xmx64m LineCount readAllLines big.log +readAllLines OutOfMemoryError: Java heap space (-Xmx64 MB) + +How much heap does readAllLines need for this file? +$ java -Xmx1g LineCount readAllLines big.log +readAllLines OutOfMemoryError: Java heap space (-Xmx1024 MB) +$ java -Xmx2g LineCount readAllLines big.log +readAllLines lines=9,623,347 chars=990,376,717 time=2,872 ms gc=24 collections / 922 ms (-Xmx2048 MB) +$ java -Xmx3g LineCount readAllLines big.log +readAllLines lines=9,623,347 chars=990,376,717 time=2,729 ms gc=23 collections / 949 ms (-Xmx3072 MB) +$ java -Xmx4g LineCount readAllLines big.log +readAllLines lines=9,623,347 chars=990,376,717 time=2,862 ms gc=23 collections / 905 ms (-Xmx4096 MB) + +And the streaming reader at a tiny heap: +$ java -Xmx16m LineCount reader big.log +reader lines=9,623,347 chars=990,376,717 time=1,347 ms gc=171 collections / 101 ms (-Xmx16 MB) diff --git a/bufferedreader/output/04-timing.txt b/bufferedreader/output/04-timing.txt new file mode 100644 index 0000000..d8159c0 --- /dev/null +++ b/bufferedreader/output/04-timing.txt @@ -0,0 +1,19 @@ +$ java -Xmx5g LineCount big.log (round-robin, 3 rounds, fresh JVM each) +-- round 1 +reader lines=9,623,347 chars=990,376,717 time=1,334 ms gc=18 collections / 21 ms (-Xmx5120 MB) +reader64k lines=9,623,347 chars=990,376,717 time=1,260 ms gc=18 collections / 21 ms (-Xmx5120 MB) +scanner lines=9,623,347 chars=990,376,717 time=5,367 ms gc=49 collections / 49 ms (-Xmx5120 MB) +lines lines=9,623,347 chars=990,376,717 time=1,311 ms gc=18 collections / 24 ms (-Xmx5120 MB) +readAllLines lines=9,623,347 chars=990,376,717 time=2,892 ms gc=23 collections / 1048 ms (-Xmx5120 MB) +-- round 2 +reader lines=9,623,347 chars=990,376,717 time=1,216 ms gc=19 collections / 20 ms (-Xmx5120 MB) +reader64k lines=9,623,347 chars=990,376,717 time=1,233 ms gc=18 collections / 21 ms (-Xmx5120 MB) +scanner lines=9,623,347 chars=990,376,717 time=5,645 ms gc=50 collections / 51 ms (-Xmx5120 MB) +lines lines=9,623,347 chars=990,376,717 time=1,341 ms gc=17 collections / 26 ms (-Xmx5120 MB) +readAllLines lines=9,623,347 chars=990,376,717 time=2,788 ms gc=23 collections / 955 ms (-Xmx5120 MB) +-- round 3 +reader lines=9,623,347 chars=990,376,717 time=1,254 ms gc=18 collections / 21 ms (-Xmx5120 MB) +reader64k lines=9,623,347 chars=990,376,717 time=1,325 ms gc=18 collections / 21 ms (-Xmx5120 MB) +scanner lines=9,623,347 chars=990,376,717 time=5,300 ms gc=49 collections / 45 ms (-Xmx5120 MB) +lines lines=9,623,347 chars=990,376,717 time=1,398 ms gc=18 collections / 25 ms (-Xmx5120 MB) +readAllLines lines=9,623,347 chars=990,376,717 time=2,985 ms gc=24 collections / 1236 ms (-Xmx5120 MB) diff --git a/bufferedreader/output/05-read-calls.txt b/bufferedreader/output/05-read-calls.txt new file mode 100644 index 0000000..c889586 --- /dev/null +++ b/bufferedreader/output/05-read-calls.txt @@ -0,0 +1,17 @@ +== 1. read() calls that reach the underlying InputStream (first 50 MB of the file) == +InputStreamReader.read() per char underlying calls= 6,104 avg bytes/call= 8,192.0 +BufferedReader(8192).read() per char underlying calls= 6,104 avg bytes/call= 8,192.0 +BufferedReader(8192).readLine() underlying calls= 6,104 avg bytes/call= 8,192.0 +BufferedReader(65536).readLine() underlying calls= 6,104 avg bytes/call= 8,192.0 +BufferedReader(1).readLine() underlying calls= 6,104 avg bytes/call= 8,192.0 + +== 2. time to consume the whole file (3 passes each; indicative, same JVM) == +pass 1 InputStreamReader.read() per char 6,493 ms +pass 1 BufferedReader.read() per char 4,902 ms +pass 1 BufferedReader.readLine() 206 ms +pass 2 InputStreamReader.read() per char 7,373 ms +pass 2 BufferedReader.read() per char 4,948 ms +pass 2 BufferedReader.readLine() 279 ms +pass 3 InputStreamReader.read() per char 7,395 ms +pass 3 BufferedReader.read() per char 5,077 ms +pass 3 BufferedReader.readLine() 269 ms diff --git a/bufferedreader/output/06-long-line.txt b/bufferedreader/output/06-long-line.txt new file mode 100644 index 0000000..97f388f --- /dev/null +++ b/bufferedreader/output/06-long-line.txt @@ -0,0 +1,6 @@ +$ java -Xmx128m LongLineDemo line long.txt +mode=line -Xmx128 MB +OutOfMemoryError: Java heap space +$ java -Xmx128m LongLineDemo chunks long.txt +mode=chunks -Xmx128 MB +read(char[]) processed 200,000,071 chars, 2,739,727 commas, never holding more than 8192 chars diff --git a/bufferedreader/output/07-charset.txt b/bufferedreader/output/07-charset.txt new file mode 100644 index 0000000..3cddcaf --- /dev/null +++ b/bufferedreader/output/07-charset.txt @@ -0,0 +1,22 @@ +$ java CharsetDemo +Charset.defaultCharset() = UTF-8 +native.encoding = ANSI_X3.4-1968 +file.encoding (property) = UTF-8 + +latin1.txt bytes: 63 61 66 e9 0a (the text is "café" saved as ISO-8859-1) +new FileReader(path) -> "caf\uFFFD" +new FileReader(path, UTF_8) -> "caf\uFFFD" +Files.newBufferedReader(path) -> MalformedInputException: Input length = 1 +Files.readAllLines(path) -> MalformedInputException: Input length = 1 +Files.lines(path).toList() -> UncheckedIOException: java.nio.charset.MalformedInputException: Input length = 1 [cause: MalformedInputException: Input length = 1] +new Scanner(path).nextLine() -> "caf\uFFFD" +readAllLines(path, ISO_8859_1) <- fix -> [café] + +utf8.txt bytes: 63 61 66 c3 a9 20 e2 82 b9 31 30 30 0a ("café ₹100" saved as UTF-8) +new FileReader(path) -> "caf\u00E9 \u20B9100" +new FileReader(path, UTF_8) <- fix -> "caf\u00E9 \u20B9100" + +bom.csv bytes: ef bb bf 69 64 2c 6e 61 6d 65 0a 31 2c 41 ... +header line = "\uFEFFid,name" length=8 (the text is 7 characters) +header.equals("id,name") = false +after stripping U+FEFF = "id,name" equals=true diff --git a/bufferedreader/output/08-charset-compat.txt b/bufferedreader/output/08-charset-compat.txt new file mode 100644 index 0000000..2cb94a9 --- /dev/null +++ b/bufferedreader/output/08-charset-compat.txt @@ -0,0 +1,22 @@ +$ LC_ALL=C java -Dfile.encoding=COMPAT CharsetDemo +Charset.defaultCharset() = US-ASCII +native.encoding = ANSI_X3.4-1968 +file.encoding (property) = ANSI_X3.4-1968 + +latin1.txt bytes: 63 61 66 e9 0a (the text is "café" saved as ISO-8859-1) +new FileReader(path) -> "caf\uFFFD" +new FileReader(path, UTF_8) -> "caf\uFFFD" +Files.newBufferedReader(path) -> MalformedInputException: Input length = 1 +Files.readAllLines(path) -> MalformedInputException: Input length = 1 +Files.lines(path).toList() -> UncheckedIOException: java.nio.charset.MalformedInputException: Input length = 1 [cause: MalformedInputException: Input length = 1] +new Scanner(path).nextLine() -> "caf\uFFFD" +readAllLines(path, ISO_8859_1) <- fix -> [café] + +utf8.txt bytes: 63 61 66 c3 a9 20 e2 82 b9 31 30 30 0a ("café ₹100" saved as UTF-8) +new FileReader(path) -> "caf\uFFFD\uFFFD \uFFFD\uFFFD\uFFFD100" +new FileReader(path, UTF_8) <- fix -> "caf\u00E9 \u20B9100" + +bom.csv bytes: ef bb bf 69 64 2c 6e 61 6d 65 0a 31 2c 41 ... +header line = "\uFEFFid,name" length=8 (the text is 7 characters) +header.equals("id,name") = false +after stripping U+FEFF = "id,name" equals=true diff --git a/bufferedreader/output/09-line-endings.txt b/bufferedreader/output/09-line-endings.txt new file mode 100644 index 0000000..2f4230f --- /dev/null +++ b/bufferedreader/output/09-line-endings.txt @@ -0,0 +1,10 @@ +input | BufferedReader.readLine() | Scanner.nextLine() | split("\n").length +-------------------------------------------------------------------------------------------------------- +"a\nb" | ["a", "b"] | ["a", "b"] | 2 +"a\nb\n" | ["a", "b"] | ["a", "b"] | 2 +"a\nb\n\n" | ["a", "b", ""] | ["a", "b", ""] | 2 +"a\r\nb" | ["a", "b"] | ["a", "b"] | 2 +"a\rb" | ["a", "b"] | ["a", "b"] | 1 +"" | [] | [] | 1 +"\n" | [""] | [""] | 0 +"a\n\rb" | ["a", "", "b"] | ["a", "", "b"] | 2 diff --git a/bufferedreader/output/10-scanner-swallows.txt b/bufferedreader/output/10-scanner-swallows.txt new file mode 100644 index 0000000..44dced4 --- /dev/null +++ b/bufferedreader/output/10-scanner-swallows.txt @@ -0,0 +1,9 @@ +-- BufferedReader -- +got: line one +got: line two +IOException after 2 lines: simulated read failure (disk gone) +-- Scanner -- +got: line one +got: line two +loop ended normally after 2 lines +sc.ioException() = java.io.IOException: simulated read failure (disk gone) diff --git a/bufferedreader/output/11-tests.txt b/bufferedreader/output/11-tests.txt new file mode 100644 index 0000000..5dbd8f0 --- /dev/null +++ b/bufferedreader/output/11-tests.txt @@ -0,0 +1,4 @@ +------------------------------------------------------------------------------- +Test set: com.ankurm.bufferedreader.ReadLineContractTest +------------------------------------------------------------------------------- +Tests run: 10, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.194 s -- in com.ankurm.bufferedreader.ReadLineContractTest diff --git a/bufferedreader/pom.xml b/bufferedreader/pom.xml new file mode 100644 index 0000000..2a4f853 --- /dev/null +++ b/bufferedreader/pom.xml @@ -0,0 +1,43 @@ + + + 4.0.0 + + + com.ankurm + java-core-examples + 1.0 + + + bufferedreader + bufferedreader + Reading a text file line by line in Java: BufferedReader vs Scanner vs Files.lines vs Files.readAllLines on a 1 GB file, charset pitfalls, line terminators, BOM, Scanner swallowing IOException, and the single-huge-line OOM. + + + + org.junit.jupiter + junit-jupiter + 5.11.0 + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + 25 + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + + + + diff --git a/bufferedreader/scripts/run-all.sh b/bufferedreader/scripts/run-all.sh new file mode 100755 index 0000000..c931695 --- /dev/null +++ b/bufferedreader/scripts/run-all.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Regenerates every file in ../output/. Requires JDK25_HOME. Needs ~1.3 GB of free disk in $DATA (default /tmp/bufferedreader-data) +# and ~6 GB of RAM for the readAllLines timing run; generated files are deleted on exit. +# Set BIG_BYTES (default 1000000000) to use a smaller "big" file. Timings are indicative; counts, exceptions and text are exact. +set -euo pipefail +[[ -z "${JDK25_HOME:-}" ]] && { echo "JDK25_HOME must be set" >&2; exit 1; } +cd "$(dirname "$0")/.." +OUT=output; mkdir -p "$OUT" +DATA="${DATA:-/tmp/bufferedreader-data}"; mkdir -p "$DATA" +BIG_BYTES="${BIG_BYTES:-1000000000}" +MED_BYTES=200000000 +trap 'rm -rf "$DATA"' EXIT +export JAVA_HOME="$JDK25_HOME" +mvn -q -f ../pom.xml -pl bufferedreader -am package +J="$JDK25_HOME/bin/java -Dstdout.encoding=UTF-8"; CP=target/classes; B=com.ankurm.bufferedreader +strip() { grep -v "Picked up"; } + +{ echo '$ uname -sr'; uname -sr; echo '$ nproc'; nproc; echo '$ java -version'; $JAVA_HOME/bin/java -version 2>&1 | strip + echo '$ free -m (before)'; free -m | sed 's/ */ /g' | head -2; } > "$OUT/01-environment.txt" + +echo "==> generate files" +{ $J -cp $CP $B.TestData lines "$DATA/big.log" "$BIG_BYTES" 2>&1 | strip + $J -cp $CP $B.TestData lines "$DATA/medium.log" "$MED_BYTES" 2>&1 | strip + $J -cp $CP $B.TestData oneline "$DATA/long.txt" "$MED_BYTES" 2>&1 | strip + echo '$ ls -l (generated files)'; ls -l "$DATA"/*.log "$DATA"/*.txt | awk '{print $5, $9}' | sed "s#$DATA/##" + echo '$ head -3 big.log'; head -3 "$DATA/big.log" + echo '$ wc -l big.log'; wc -l < "$DATA/big.log"; } > "$OUT/02-generate.txt" + +echo "==> which approaches survive -Xmx64m, and how much heap readAllLines needs" +{ for m in reader scanner lines readAllLines; do echo "\$ java -Xmx64m LineCount $m big.log"; $J -Xmx64m -cp $CP $B.LineCount $m "$DATA/big.log" 2>&1 | strip; done + echo; echo "How much heap does readAllLines need for this file?" + for x in 1g 2g 3g 4g; do echo "\$ java -Xmx$x LineCount readAllLines big.log"; $J -Xmx$x -cp $CP $B.LineCount readAllLines "$DATA/big.log" 2>&1 | strip; done + echo; echo "And the streaming reader at a tiny heap:" + echo "\$ java -Xmx16m LineCount reader big.log"; $J -Xmx16m -cp $CP $B.LineCount reader "$DATA/big.log" 2>&1 | strip; } > "$OUT/03-heap.txt" + +echo "==> timing, fresh JVM per run, -Xmx5g, 3 rounds" +{ echo '$ java -Xmx5g LineCount big.log (round-robin, 3 rounds, fresh JVM each)' + for round in 1 2 3; do echo "-- round $round" + for m in reader reader64k scanner lines readAllLines; do $J -Xmx5g -cp $CP $B.LineCount $m "$DATA/big.log" 2>&1 | strip; done; done; } > "$OUT/04-timing.txt" + +echo "==> read calls" +$J -cp $CP $B.ReadCalls "$DATA/medium.log" 2>&1 | strip > "$OUT/05-read-calls.txt" + +echo "==> one huge line" +{ for m in line chunks; do echo "\$ java -Xmx128m LongLineDemo $m long.txt"; $J -Xmx128m -cp $CP $B.LongLineDemo $m "$DATA/long.txt" 2>&1 | strip; done; } > "$OUT/06-long-line.txt" + +echo "==> charset, endings, scanner" +{ echo '$ java CharsetDemo'; $J -cp $CP $B.CharsetDemo 2>&1 | strip; } > "$OUT/07-charset.txt" +{ echo '$ LC_ALL=C java -Dfile.encoding=COMPAT CharsetDemo'; LC_ALL=C $J -Dfile.encoding=COMPAT -cp $CP $B.CharsetDemo 2>&1 | strip; } > "$OUT/08-charset-compat.txt" +$J -cp $CP $B.LineEndingsDemo 2>&1 | strip > "$OUT/09-line-endings.txt" +$J -cp $CP $B.ScannerSwallowsDemo 2>&1 | strip > "$OUT/10-scanner-swallows.txt" + +cp target/surefire-reports/com.ankurm.bufferedreader.ReadLineContractTest.txt "$OUT/11-tests.txt" +echo Done diff --git a/bufferedreader/src/main/java/com/ankurm/bufferedreader/CharsetDemo.java b/bufferedreader/src/main/java/com/ankurm/bufferedreader/CharsetDemo.java new file mode 100644 index 0000000..8fb05bd --- /dev/null +++ b/bufferedreader/src/main/java/com/ankurm/bufferedreader/CharsetDemo.java @@ -0,0 +1,76 @@ +package com.ankurm.bufferedreader; + +import java.io.*; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HexFormat; +import java.util.Scanner; +import java.util.concurrent.Callable; + +/** + * The same file read five ways when its bytes are NOT valid UTF-8, and a UTF-8 BOM. Run it twice: + * once normally and once with -Dfile.encoding=COMPAT to see the pre-JDK-18 default come back. + */ +public final class CharsetDemo { + + public static void main(String[] a) throws Exception { + Path dir = Files.createTempDirectory("charset"); + System.out.println("Charset.defaultCharset() = " + Charset.defaultCharset()); + System.out.println("native.encoding = " + System.getProperty("native.encoding")); + System.out.println("file.encoding (property) = " + System.getProperty("file.encoding")); + System.out.println(); + + Path latin1 = dir.resolve("latin1.txt"); + Files.write(latin1, "caf\u00e9\n".getBytes(StandardCharsets.ISO_8859_1)); + System.out.println("latin1.txt bytes: " + HexFormat.ofDelimiter(" ").formatHex(Files.readAllBytes(latin1)) + " (the text is \"caf\u00e9\" saved as ISO-8859-1)"); + run("new FileReader(path) ", () -> firstLine(new BufferedReader(new FileReader(latin1.toFile())))); + run("new FileReader(path, UTF_8) ", () -> firstLine(new BufferedReader(new FileReader(latin1.toFile(), StandardCharsets.UTF_8)))); + run("Files.newBufferedReader(path) ", () -> firstLine(Files.newBufferedReader(latin1))); + run("Files.readAllLines(path) ", () -> Files.readAllLines(latin1).toString()); + run("Files.lines(path).toList() ", () -> { try (var s = Files.lines(latin1)) { return s.toList().toString(); } }); + run("new Scanner(path).nextLine() ", () -> { try (Scanner sc = new Scanner(latin1)) { return Esc.of(sc.nextLine()); } }); + run("readAllLines(path, ISO_8859_1) <- fix ", () -> Files.readAllLines(latin1, StandardCharsets.ISO_8859_1).toString()); + System.out.println(); + + Path utf8 = dir.resolve("utf8.txt"); + Files.writeString(utf8, "caf\u00e9 \u20b9100\n", StandardCharsets.UTF_8); + System.out.println("utf8.txt bytes: " + HexFormat.ofDelimiter(" ").formatHex(Files.readAllBytes(utf8)) + " (\"caf\u00e9 \u20b9100\" saved as UTF-8)"); + run("new FileReader(path) ", () -> firstLine(new BufferedReader(new FileReader(utf8.toFile())))); + run("new FileReader(path, UTF_8) <- fix ", () -> firstLine(new BufferedReader(new FileReader(utf8.toFile(), StandardCharsets.UTF_8)))); + System.out.println(); + + Path bom = dir.resolve("bom.csv"); + Files.write(bom, concat(new byte[]{(byte) 0xEF, (byte) 0xBB, (byte) 0xBF}, "id,name\n1,Asha\n".getBytes(StandardCharsets.UTF_8))); + System.out.println("bom.csv bytes: " + HexFormat.ofDelimiter(" ").formatHex(Files.readAllBytes(bom)).substring(0, 41) + " ..."); + try (BufferedReader br = Files.newBufferedReader(bom)) { + String header = br.readLine(); + System.out.println("header line = " + Esc.of(header) + " length=" + header.length() + " (the text is 7 characters)"); + System.out.println("header.equals(\"id,name\") = " + header.equals("id,name")); + String cleaned = header.startsWith("\uFEFF") ? header.substring(1) : header; + System.out.println("after stripping U+FEFF = " + Esc.of(cleaned) + " equals=" + cleaned.equals("id,name")); + } + } + + static byte[] concat(byte[] x, byte[] y) { + byte[] r = new byte[x.length + y.length]; + System.arraycopy(x, 0, r, 0, x.length); + System.arraycopy(y, 0, r, x.length, y.length); + return r; + } + + static String firstLine(BufferedReader br) throws IOException { + try (br) { return Esc.of(br.readLine()); } + } + + static void run(String label, Callable c) { + String result; + try { result = c.call(); } + catch (Exception e) { + result = e.getClass().getSimpleName() + ": " + e.getMessage(); + if (e.getCause() != null) result += " [cause: " + e.getCause().getClass().getSimpleName() + ": " + e.getCause().getMessage() + "]"; + } + System.out.println(label + " -> " + result); + } +} diff --git a/bufferedreader/src/main/java/com/ankurm/bufferedreader/Esc.java b/bufferedreader/src/main/java/com/ankurm/bufferedreader/Esc.java new file mode 100644 index 0000000..148ce0e --- /dev/null +++ b/bufferedreader/src/main/java/com/ankurm/bufferedreader/Esc.java @@ -0,0 +1,17 @@ +package com.ankurm.bufferedreader; + +/** Makes non-ASCII and control characters visible so transcripts show what is really in the String. */ +final class Esc { + private Esc() {} + static String of(String s) { + if (s == null) return "null"; + StringBuilder sb = new StringBuilder("\""); + for (char c : s.toCharArray()) { + if (c == '\n') sb.append("\\n"); + else if (c == '\r') sb.append("\\r"); + else if (c >= 32 && c < 127) sb.append(c); + else sb.append(String.format("\\u%04X", (int) c)); + } + return sb.append('"').toString(); + } +} diff --git a/bufferedreader/src/main/java/com/ankurm/bufferedreader/LineCount.java b/bufferedreader/src/main/java/com/ankurm/bufferedreader/LineCount.java new file mode 100644 index 0000000..2ebbbe4 --- /dev/null +++ b/bufferedreader/src/main/java/com/ankurm/bufferedreader/LineCount.java @@ -0,0 +1,84 @@ +package com.ankurm.bufferedreader; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Scanner; +import java.util.stream.Stream; + +/** + * Reads one file line by line with one of four approaches and prints lines, characters, wall time and GC cost. + * Every approach does identical work (count lines, sum their lengths) so the totals must match. + * Run each approach in its own fresh JVM: {@code java -Xmx64m LineCount reader big.log}. + */ +public final class LineCount { + + public static void main(String[] a) throws Exception { + String approach = a[0]; + Path p = Path.of(a[1]); + long t0 = System.nanoTime(); + long[] r; + try { + r = switch (approach) { + case "reader" -> reader(p, 8192); + case "reader64k" -> reader(p, 65536); + case "scanner" -> scanner(p); + case "lines" -> lines(p); + case "readAllLines" -> readAllLines(p); + default -> throw new IllegalArgumentException("unknown approach " + approach); + }; + } catch (OutOfMemoryError e) { + System.out.printf("%-13s OutOfMemoryError: %s (-Xmx%d MB)%n", approach, e.getMessage(), Runtime.getRuntime().maxMemory() >> 20); + return; + } + long ms = (System.nanoTime() - t0) / 1_000_000; + long gcCount = 0, gcMs = 0; + for (GarbageCollectorMXBean gc : ManagementFactory.getGarbageCollectorMXBeans()) { + gcCount += gc.getCollectionCount(); + gcMs += gc.getCollectionTime(); + } + System.out.printf("%-13s lines=%,d chars=%,d time=%,d ms gc=%d collections / %d ms (-Xmx%d MB)%n", + approach, r[0], r[1], ms, gcCount, gcMs, Runtime.getRuntime().maxMemory() >> 20); + } + + /** BufferedReader.readLine() in try-with-resources, charset stated explicitly. */ + static long[] reader(Path p, int bufferChars) throws Exception { + long n = 0, chars = 0; + try (BufferedReader br = new BufferedReader(new FileReader(p.toFile(), StandardCharsets.UTF_8), bufferChars)) { + String line; + while ((line = br.readLine()) != null) { n++; chars += line.length(); } + } + return new long[]{n, chars}; + } + + /** Scanner.nextLine(). */ + static long[] scanner(Path p) throws Exception { + long n = 0, chars = 0; + try (Scanner sc = new Scanner(p, StandardCharsets.UTF_8)) { + while (sc.hasNextLine()) { chars += sc.nextLine().length(); n++; } + } + return new long[]{n, chars}; + } + + /** Files.lines(): lazy Stream, must be closed. */ + static long[] lines(Path p) throws Exception { + try (Stream s = Files.lines(p, StandardCharsets.UTF_8)) { + long[] acc = new long[2]; + s.forEach(l -> { acc[0]++; acc[1] += l.length(); }); + return acc; + } + } + + /** Files.readAllLines(): the whole file as a List before you see line one. */ + static long[] readAllLines(Path p) throws Exception { + List all = Files.readAllLines(p, StandardCharsets.UTF_8); + long chars = 0; + for (String l : all) chars += l.length(); + return new long[]{all.size(), chars}; + } +} diff --git a/bufferedreader/src/main/java/com/ankurm/bufferedreader/LineEndingsDemo.java b/bufferedreader/src/main/java/com/ankurm/bufferedreader/LineEndingsDemo.java new file mode 100644 index 0000000..7abc42c --- /dev/null +++ b/bufferedreader/src/main/java/com/ankurm/bufferedreader/LineEndingsDemo.java @@ -0,0 +1,38 @@ +package com.ankurm.bufferedreader; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; +import java.util.Scanner; + +/** What counts as "a line"? Terminators, a final newline, empty files, and what readLine() cannot tell you. */ +public final class LineEndingsDemo { + + public static void main(String[] a) throws IOException { + System.out.printf("%-16s | %-34s | %-34s | %s%n", "input", "BufferedReader.readLine()", "Scanner.nextLine()", "split(\"\\n\").length"); + System.out.println("-".repeat(104)); + String[] inputs = {"a\nb", "a\nb\n", "a\nb\n\n", "a\r\nb", "a\rb", "", "\n", "a\n\rb"}; + for (String s : inputs) { + System.out.printf("%-16s | %-34s | %-34s | %d%n", Esc.of(s), brLines(s), scLines(s), s.split("\n").length); + } + } + + static List brLines(String s) throws IOException { + List out = new ArrayList<>(); + try (BufferedReader br = new BufferedReader(new StringReader(s))) { + String l; + while ((l = br.readLine()) != null) out.add(Esc.of(l)); + } + return out; + } + + static List scLines(String s) { + List out = new ArrayList<>(); + try (Scanner sc = new Scanner(new StringReader(s))) { + while (sc.hasNextLine()) out.add(Esc.of(sc.nextLine())); + } + return out; + } +} diff --git a/bufferedreader/src/main/java/com/ankurm/bufferedreader/LongLineDemo.java b/bufferedreader/src/main/java/com/ankurm/bufferedreader/LongLineDemo.java new file mode 100644 index 0000000..edca369 --- /dev/null +++ b/bufferedreader/src/main/java/com/ankurm/bufferedreader/LongLineDemo.java @@ -0,0 +1,36 @@ +package com.ankurm.bufferedreader; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; + +/** + * readLine() has no maximum length. One giant line (minified JSON, a log with a stack trace flattened onto one line) + * is read into a single String. Two modes: {@code line} (the naive loop) and {@code chunks} (read(char[]) in 8 KB pieces). + */ +public final class LongLineDemo { + + public static void main(String[] a) throws Exception { + String mode = a[0]; + Path p = Path.of(a[1]); + System.out.printf("mode=%s -Xmx%d MB%n", mode, Runtime.getRuntime().maxMemory() >> 20); + try (BufferedReader br = new BufferedReader(new FileReader(p.toFile(), StandardCharsets.UTF_8))) { + if (mode.equals("line")) { + String line = br.readLine(); + System.out.printf("readLine() returned %,d chars%n", line == null ? -1 : line.length()); + } else { + char[] buf = new char[8192]; + long total = 0, commas = 0; + int n; + while ((n = br.read(buf, 0, buf.length)) != -1) { + total += n; + for (int i = 0; i < n; i++) if (buf[i] == ',') commas++; + } + System.out.printf("read(char[]) processed %,d chars, %,d commas, never holding more than %d chars%n", total, commas, buf.length); + } + } catch (OutOfMemoryError e) { + System.out.println("OutOfMemoryError: " + e.getMessage()); + } + } +} diff --git a/bufferedreader/src/main/java/com/ankurm/bufferedreader/ReadCalls.java b/bufferedreader/src/main/java/com/ankurm/bufferedreader/ReadCalls.java new file mode 100644 index 0000000..5a628c4 --- /dev/null +++ b/bufferedreader/src/main/java/com/ankurm/bufferedreader/ReadCalls.java @@ -0,0 +1,66 @@ +package com.ankurm.bufferedreader; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; + +/** + * What does BufferedReader actually save? Counts the read calls that reach the underlying InputStream, then times + * three ways of consuming the same file. Corrects a common claim: an unbuffered InputStreamReader/FileReader does NOT + * make one operating-system read per character, because its decoder already reads 8 KB of bytes at a time. + */ +public final class ReadCalls { + + /** Counts calls and bytes that reach the stream under the reader. */ + static final class CountingStream extends FilterInputStream { + long calls, bytes; + CountingStream(InputStream in) { super(in); } + @Override public int read() throws IOException { int b = super.read(); calls++; if (b >= 0) bytes++; return b; } + @Override public int read(byte[] b, int off, int len) throws IOException { + int n = super.read(b, off, len); calls++; if (n > 0) bytes += n; return n; + } + } + + public static void main(String[] a) throws Exception { + Path p = Path.of(a[0]); + System.out.println("== 1. read() calls that reach the underlying InputStream (first 50 MB of the file) =="); + count("InputStreamReader.read() per char", p, false, 0, 'c'); + count("BufferedReader(8192).read() per char", p, true, 8192, 'c'); + count("BufferedReader(8192).readLine()", p, true, 8192, 'l'); + count("BufferedReader(65536).readLine()", p, true, 65536, 'l'); + count("BufferedReader(1).readLine()", p, true, 1, 'l'); + System.out.println(); + System.out.println("== 2. time to consume the whole file (3 passes each; indicative, same JVM) =="); + for (int pass = 1; pass <= 3; pass++) { + System.out.printf("pass %d InputStreamReader.read() per char %,6d ms%n", pass, time(p, false, 0, 'c')); + System.out.printf("pass %d BufferedReader.read() per char %,6d ms%n", pass, time(p, true, 8192, 'c')); + System.out.printf("pass %d BufferedReader.readLine() %,6d ms%n", pass, time(p, true, 8192, 'l')); + } + } + + static final long LIMIT = 50_000_000L; + + static void count(String label, Path p, boolean buffered, int size, char mode) throws Exception { + CountingStream cs = new CountingStream(new FileInputStream(p.toFile())); + Reader base = new InputStreamReader(cs, StandardCharsets.UTF_8); + Reader r = buffered ? new BufferedReader(base, size) : base; + long chars = 0; + try (r) { + if (mode == 'c') { int c; while (chars < LIMIT && (c = r.read()) != -1) chars++; } + else { BufferedReader br = (BufferedReader) r; String l; while (chars < LIMIT && (l = br.readLine()) != null) chars += l.length() + 1; } + } + System.out.printf("%-38s underlying calls=%,9d avg bytes/call=%,8.1f%n", label, cs.calls, (double) cs.bytes / cs.calls); + } + + static long time(Path p, boolean buffered, int size, char mode) throws Exception { + long t0 = System.nanoTime(), x = 0; + Reader base = new InputStreamReader(new FileInputStream(p.toFile()), StandardCharsets.UTF_8); + Reader r = buffered ? new BufferedReader(base, size) : base; + try (r) { + if (mode == 'c') { int c; while ((c = r.read()) != -1) x += c; } + else { BufferedReader br = (BufferedReader) r; String l; while ((l = br.readLine()) != null) x += l.length(); } + } + if (x == 42) System.out.print(""); // keep the loop alive + return (System.nanoTime() - t0) / 1_000_000; + } +} diff --git a/bufferedreader/src/main/java/com/ankurm/bufferedreader/ScannerSwallowsDemo.java b/bufferedreader/src/main/java/com/ankurm/bufferedreader/ScannerSwallowsDemo.java new file mode 100644 index 0000000..833f89d --- /dev/null +++ b/bufferedreader/src/main/java/com/ankurm/bufferedreader/ScannerSwallowsDemo.java @@ -0,0 +1,46 @@ +package com.ankurm.bufferedreader; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.Reader; +import java.util.Scanner; + +/** + * A reader that serves two lines and then fails, as a dying disk or a dropped network share would. + * BufferedReader tells you; Scanner looks exactly like a clean end of file. + */ +public final class ScannerSwallowsDemo { + + /** Delivers "line one\nline two\n" in one read, then throws on the next. */ + static final class FailingReader extends Reader { + private boolean first = true; + @Override public int read(char[] buf, int off, int len) throws IOException { + if (!first) throw new IOException("simulated read failure (disk gone)"); + first = false; + String s = "line one\nline two\n"; + s.getChars(0, s.length(), buf, off); + return s.length(); + } + @Override public void close() {} + } + + public static void main(String[] a) { + System.out.println("-- BufferedReader --"); + int n = 0; + try (BufferedReader br = new BufferedReader(new FailingReader())) { + String l; + while ((l = br.readLine()) != null) { n++; System.out.println("got: " + l); } + System.out.println("loop ended normally after " + n + " lines"); + } catch (IOException e) { + System.out.println("IOException after " + n + " lines: " + e.getMessage()); + } + + System.out.println("-- Scanner --"); + n = 0; + try (Scanner sc = new Scanner(new FailingReader())) { + while (sc.hasNextLine()) { n++; System.out.println("got: " + sc.nextLine()); } + System.out.println("loop ended normally after " + n + " lines"); + System.out.println("sc.ioException() = " + sc.ioException()); + } + } +} diff --git a/bufferedreader/src/main/java/com/ankurm/bufferedreader/TestData.java b/bufferedreader/src/main/java/com/ankurm/bufferedreader/TestData.java new file mode 100644 index 0000000..0adf26a --- /dev/null +++ b/bufferedreader/src/main/java/com/ankurm/bufferedreader/TestData.java @@ -0,0 +1,43 @@ +package com.ankurm.bufferedreader; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Deterministic generator for the synthetic files used by the benchmarks. + * + * 
+ *   java TestData lines  big.log 1000000000     # ~1 GB of ~100-byte ASCII log lines
+ *   java TestData oneline long.txt 200000000    # 200 MB with NO newline at all
+ *
+ */ +public final class TestData { + public static void main(String[] a) throws IOException { + String kind = a[0]; + Path p = Path.of(a[1]); + long bytes = Long.parseLong(a[2]); + long written = 0; + long t0 = System.nanoTime(); + try (OutputStream out = new BufferedOutputStream(Files.newOutputStream(p), 1 << 16)) { + if (kind.equals("lines")) { + long i = 0; + while (written < bytes) { + String line = "2026-01-01T00:00:%02d.%03dZ INFO [worker-%02d] request id=%09d path=/api/v1/items/%d status=%d ms=%d\n" + .formatted(i % 60, i % 1000, i % 32, i, i % 977, i % 7 == 0 ? 500 : 200, i % 113); + byte[] b = line.getBytes(StandardCharsets.US_ASCII); + out.write(b); + written += b.length; + i++; + } + } else if (kind.equals("oneline")) { + byte[] chunk = "{\"k\":\"vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\"},".getBytes(StandardCharsets.US_ASCII); + while (written < bytes) { out.write(chunk); written += chunk.length; } + } else throw new IllegalArgumentException(kind); + } + System.out.printf("generated %s: %,d bytes in %d ms%n", p.getFileName(), Files.size(p), (System.nanoTime() - t0) / 1_000_000); + } +} diff --git a/bufferedreader/src/test/java/com/ankurm/bufferedreader/ReadLineContractTest.java b/bufferedreader/src/test/java/com/ankurm/bufferedreader/ReadLineContractTest.java new file mode 100644 index 0000000..af0bebf --- /dev/null +++ b/bufferedreader/src/test/java/com/ankurm/bufferedreader/ReadLineContractTest.java @@ -0,0 +1,108 @@ +package com.ankurm.bufferedreader; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.*; +import java.nio.charset.MalformedInputException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Scanner; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** Every behaviour the post states about line reading, pinned as an assertion. */ +class ReadLineContractTest { + + @TempDir Path dir; + + static List br(String s) throws IOException { + List out = new ArrayList<>(); + try (BufferedReader r = new BufferedReader(new StringReader(s))) { String l; while ((l = r.readLine()) != null) out.add(l); } + return out; + } + + @Test void readLineAcceptsAllThreeTerminators() throws IOException { + assertEquals(List.of("a", "b"), br("a\nb")); + assertEquals(List.of("a", "b"), br("a\r\nb")); + assertEquals(List.of("a", "b"), br("a\rb")); + } + + @Test void finalNewlineDoesNotCreateAnEmptyLastLine() throws IOException { + assertEquals(List.of("a", "b"), br("a\nb\n")); + assertEquals(List.of("a", "b", ""), br("a\nb\n\n")); + } + + @Test void emptyFileHasZeroLinesAndLoneNewlineHasOne() throws IOException { + assertEquals(List.of(), br("")); + assertEquals(List.of(""), br("\n")); + } + + @Test void readLineCannotTellWhetherTheLastLineHadATerminator() throws IOException { + assertEquals(br("a\nb"), br("a\nb\n")); + } + + @Test void allFourApproachesAgreeOnAWellFormedFile() throws Exception { + Path p = dir.resolve("f.txt"); + Files.writeString(p, "one\ntwo\r\nthree\n\nfive"); + List viaReader = new ArrayList<>(); + try (BufferedReader r = Files.newBufferedReader(p)) { String l; while ((l = r.readLine()) != null) viaReader.add(l); } + List viaScanner = new ArrayList<>(); + try (Scanner sc = new Scanner(p)) { while (sc.hasNextLine()) viaScanner.add(sc.nextLine()); } + List viaLines; + try (Stream s = Files.lines(p)) { viaLines = s.toList(); } + List viaAll = Files.readAllLines(p); + assertEquals(List.of("one", "two", "three", "", "five"), viaReader); + assertEquals(viaReader, viaScanner); + assertEquals(viaReader, viaLines); + assertEquals(viaReader, viaAll); + } + + @Test void invalidUtf8IsStrictInFilesApiAndSilentlyReplacedInFileReader() throws Exception { + Path p = dir.resolve("latin1.txt"); + Files.write(p, "caf\u00e9\n".getBytes(StandardCharsets.ISO_8859_1)); + assertThrows(MalformedInputException.class, () -> Files.readAllLines(p)); + UncheckedIOException u = assertThrows(UncheckedIOException.class, () -> { try (Stream s = Files.lines(p)) { s.toList(); } }); + assertInstanceOf(MalformedInputException.class, u.getCause()); + try (BufferedReader r = new BufferedReader(new FileReader(p.toFile(), StandardCharsets.UTF_8))) { + assertEquals("caf\uFFFD", r.readLine()); + } + assertEquals(List.of("caf\u00e9"), Files.readAllLines(p, StandardCharsets.ISO_8859_1)); + } + + @Test void utf8BomSurvivesAsTheFirstCharacter() throws Exception { + Path p = dir.resolve("bom.csv"); + Files.write(p, new byte[]{(byte) 0xEF, (byte) 0xBB, (byte) 0xBF, 'i', 'd', '\n'}); + try (BufferedReader r = Files.newBufferedReader(p)) { + String first = r.readLine(); + assertEquals("\uFEFFid", first); + assertNotEquals("id", first); + } + } + + @Test void scannerHidesIoExceptionButBufferedReaderThrows() { + assertThrows(IOException.class, () -> { + try (BufferedReader r = new BufferedReader(new ScannerSwallowsDemo.FailingReader())) { while (r.readLine() != null) { } } + }); + Scanner sc = new Scanner(new ScannerSwallowsDemo.FailingReader()); + int n = 0; + while (sc.hasNextLine()) { sc.nextLine(); n++; } + assertEquals(2, n); + assertNotNull(sc.ioException()); + } + + @Test void readLineReturnsAStringAsLongAsTheLine() throws Exception { + Path p = dir.resolve("long.txt"); + Files.writeString(p, "x".repeat(1_000_000)); + try (BufferedReader r = Files.newBufferedReader(p)) { assertEquals(1_000_000, r.readLine().length()); } + } + + @Test void unclosedFilesLinesStillReadsButClosingIsYourJob() throws Exception { + Path p = dir.resolve("f.txt"); + Files.writeString(p, "a\nb\n"); + try (Stream s = Files.lines(p)) { assertEquals(2, s.count()); } + } +} diff --git a/pom.xml b/pom.xml index 0d11739..93b6c8b 100644 --- a/pom.xml +++ b/pom.xml @@ -40,6 +40,7 @@ equals-hashcode copying datetimeformatter + bufferedreader