From 9b1da079c65179f6e2e102ae0f413fd8559cb78a Mon Sep 17 00:00:00 2001 From: asmhatre Date: Thu, 29 Oct 2026 06:18:41 +0000 Subject: [PATCH] Add excel-export module: Apache POI 6.0.0 XSSF vs SXSSF under a fixed heap, 1,000,000 rows streamed with StreamingResponseBody, OpenCSV, sheet rolling, temp-file cleanup Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_017ic4b4VQVEJWetTT1dYzyJ --- README.md | 1 + excel-export/README.md | 25 ++++ excel-export/output/01-xlsx-structure.txt | 25 ++++ excel-export/output/02-temp-files.txt | 7 + excel-export/output/03-csv.txt | 18 +++ excel-export/output/04-endpoints.txt | 3 + excel-export/output/05-xssf-sweep.txt | 7 + excel-export/output/06-sxssf-sweep.txt | 8 + excel-export/output/07-million-rows.txt | 6 + excel-export/output/08-sheet-rolling.txt | 9 ++ excel-export/output/09-compress-temp.txt | 5 + excel-export/output/10-libreoffice.txt | 6 + excel-export/pom.xml | 58 ++++++++ excel-export/scripts/libreoffice-check.sh | 26 ++++ excel-export/scripts/measure.sh | 51 +++++++ excel-export/scripts/run-all.sh | 50 +++++++ .../com/ankurm/excelexport/CsvExporter.java | 60 ++++++++ .../excelexport/ExcelExportApplication.java | 12 ++ .../ankurm/excelexport/ExportController.java | 70 +++++++++ .../ankurm/excelexport/ProbeController.java | 60 ++++++++ .../java/com/ankurm/excelexport/SaleRow.java | 13 ++ .../com/ankurm/excelexport/SaleSource.java | 54 +++++++ .../com/ankurm/excelexport/XlsxExporter.java | 128 ++++++++++++++++ .../src/main/resources/application.properties | 2 + .../java/com/ankurm/excelexport/CsvTest.java | 97 ++++++++++++ .../com/ankurm/excelexport/EndpointTest.java | 52 +++++++ .../test/java/com/ankurm/excelexport/Out.java | 34 +++++ .../com/ankurm/excelexport/TempFileTest.java | 87 +++++++++++ .../ankurm/excelexport/XlsxStructureTest.java | 139 ++++++++++++++++++ 29 files changed, 1113 insertions(+) create mode 100644 excel-export/README.md create mode 100644 excel-export/output/01-xlsx-structure.txt create mode 100644 excel-export/output/02-temp-files.txt create mode 100644 excel-export/output/03-csv.txt create mode 100644 excel-export/output/04-endpoints.txt create mode 100644 excel-export/output/05-xssf-sweep.txt create mode 100644 excel-export/output/06-sxssf-sweep.txt create mode 100644 excel-export/output/07-million-rows.txt create mode 100644 excel-export/output/08-sheet-rolling.txt create mode 100644 excel-export/output/09-compress-temp.txt create mode 100644 excel-export/output/10-libreoffice.txt create mode 100644 excel-export/pom.xml create mode 100755 excel-export/scripts/libreoffice-check.sh create mode 100755 excel-export/scripts/measure.sh create mode 100755 excel-export/scripts/run-all.sh create mode 100644 excel-export/src/main/java/com/ankurm/excelexport/CsvExporter.java create mode 100644 excel-export/src/main/java/com/ankurm/excelexport/ExcelExportApplication.java create mode 100644 excel-export/src/main/java/com/ankurm/excelexport/ExportController.java create mode 100644 excel-export/src/main/java/com/ankurm/excelexport/ProbeController.java create mode 100644 excel-export/src/main/java/com/ankurm/excelexport/SaleRow.java create mode 100644 excel-export/src/main/java/com/ankurm/excelexport/SaleSource.java create mode 100644 excel-export/src/main/java/com/ankurm/excelexport/XlsxExporter.java create mode 100644 excel-export/src/main/resources/application.properties create mode 100644 excel-export/src/test/java/com/ankurm/excelexport/CsvTest.java create mode 100644 excel-export/src/test/java/com/ankurm/excelexport/EndpointTest.java create mode 100644 excel-export/src/test/java/com/ankurm/excelexport/Out.java create mode 100644 excel-export/src/test/java/com/ankurm/excelexport/TempFileTest.java create mode 100644 excel-export/src/test/java/com/ankurm/excelexport/XlsxStructureTest.java diff --git a/README.md b/README.md index b89d3bc..1c2c390 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ files. | Directory | Article | What it demonstrates | |---|---|---| +| [`excel-export/`](excel-export) | [Export Excel and CSV from Spring Boot: Streaming 1 Million Rows with Apache POI SXSSF](https://ankurm.com/spring-boot-export-excel-csv-streaming-poi-sxssf/) | XSSF vs SXSSF under a fixed 512 MB heap (XSSF out of memory between 50,000 and 100,000 rows and at 1,000,000 rows even with 3 GB), 1,000,000 rows streamed with `StreamingResponseBody` under a 64 MB heap, sheet rolling at 1,048,576 rows, `compressTempFiles` cutting the spill from 475 MB to 56 MB, OpenCSV swallowing write errors, and an SXSSF file LibreOffice will not open | | [`thymeleaf-htmx/`](thymeleaf-htmx) | [Spring Boot 4 + Thymeleaf + htmx: Server-Side CRUD UI in 2026](https://ankurm.com/spring-boot-thymeleaf-htmx-crud-ui/) | one URL answering as full page or fragment on `HX-Request`, a Thymeleaf layout, 422 validation re-render, pagination, inline edit and delete, 10 MockMvc tests plus a headless-Chromium check that observed htmx 2 ignoring a 204 and (without a config tag) a 422 | | [`rate-limit/`](rate-limit) | [Rate Limiting Spring Boot APIs with Bucket4j and Redis: Per-User, Per-Key and Global Limits](https://ankurm.com/spring-boot-rate-limiting-bucket4j-redis/) | per-user, per-API-key and global token buckets in a servlet filter on Bucket4j 8.21 (`bucket4j_jdk17-*`), `429` with `Retry-After` and `X-RateLimit-*` headers, two application instances sharing one Redis bucket and 200 concurrent requests letting exactly 20 through, a limit raised in `application.yml` that does not change buckets already stored in Redis until a config version is bumped, a client clock 30 s ahead handing out tokens another instance already spent, and fail-open versus fail-closed when Redis dies | | [`sbom/`](sbom) | [Spring Boot SBOM in Practice: the tools jar-mode sbom Command, CycloneDX and Dependency Scanning](https://ankurm.com/spring-boot-sbom-tools-jarmode-cyclonedx-dependency-scanning/) | the 4.2.0-RC1 `java -Djarmode=tools -jar app.jar sbom` command (absent in 4.1.2), the CycloneDX document the build puts in the jar, `/actuator/sbom`, an SBOM that lists 8 starters not in the jar and misses the jar mode's own library, a licence rule that wrongly flags dual-licensed logback, and an OSV scan that finds Log4Shell in a deliberately old build | diff --git a/excel-export/README.md b/excel-export/README.md new file mode 100644 index 0000000..4952b24 --- /dev/null +++ b/excel-export/README.md @@ -0,0 +1,25 @@ +# excel-export + +Companion code for **[Export Excel and CSV from Spring Boot: Streaming 1 Million Rows with Apache POI SXSSF](https://ankurm.com/spring-boot-export-excel-csv-streaming-poi-sxssf/)**. + +Spring Boot 4.1.2, Spring Framework 7, JDK 25, Apache POI 6.0.0 (`poi-ooxml`), OpenCSV 5.12.0. No database, no Docker. + +| Piece | Code | What it shows | +|---|---|---| +| XSSF endpoint | `ExportController#xssf` | whole workbook in heap; the baseline that runs out of memory | +| SXSSF endpoint | `ExportController#sxssf` | `SXSSFWorkbook(100)` + `StreamingResponseBody`, `dispose()` in `finally`, optional `compressTempFiles` | +| CSV endpoint | `ExportController#csv`, `CsvExporter` | OpenCSV streaming, UTF-8 BOM for Excel, formula-injection guard, swallowed write errors | +| Writer shared by XSSF and SXSSF | `XlsxExporter` | styles created once, freeze pane, auto filter, lakh/crore number format, sheet rolling at 1,048,576 rows | +| Data | `SaleSource` | rows computed from their index (stands in for a database cursor) | +| Diagnostics | `ProbeController` | peak used heap and peak RSS; delete before shipping | + +```bash +./scripts/run-all.sh # tests + every measurement; rewrites output/*.txt (15-25 minutes) +curl -o sales.xlsx 'http://localhost:8080/export/sxssf?rows=1000000' # after: java -jar target/excel-export-1.0.0.jar +``` + +`output/01..04` come from the tests, `output/05..09` from `scripts/measure.sh`, `output/10` from `scripts/libreoffice-check.sh`, which starts a fresh JVM with a fixed +`-Xmx` for every line. The numbers are indicative: they were taken on a shared 2-vCPU sandbox, one run per line. + +Not run here: a real database cursor (rows are generated) and Microsoft Excel itself. LibreOffice 24.2.7.2 refuses to open +the SXSSF file because its ZIP entries carry a zero timestamp (same with POI 5.5.1); see `output/10-libreoffice.txt`. diff --git a/excel-export/output/01-xlsx-structure.txt b/excel-export/output/01-xlsx-structure.txt new file mode 100644 index 0000000..498bd17 --- /dev/null +++ b/excel-export/output/01-xlsx-structure.txt @@ -0,0 +1,25 @@ +sheetLimit=1000 (header included), 2500 data rows + sheet 1 "Sales": last row index 999, first data cell 1 + sheet 2 "Sales 2": last row index 999, first data cell 1000 + sheet 3 "Sales 3": last row index 502, first data cell 1999 +XSSF and SXSSF files: same sheets, same formatted cell values on every 97th row + +styling of a 5,000-row SXSSF file, read back with XSSF + header cell A1 text=Order id bold=true + freeze pane: row split at 1 + auto filter: A1:I1 + cell styles in the workbook: 5 (for 5,000 rows x 9 cells) + B2 as Excel shows it: 30-May-2025 + amount number format: [>=10000000]##\,##\,##\,##0.00;[>=100000]##\,##\,##0.00;##,##0.00 + +the sheet limit + createRow(1048575) ok + createRow(1048576) -> IllegalArgumentException: Invalid row number (1048576) outside allowable range (0..1048575) + +lakh/crore number format, rendered by POI's DataFormatter (not by Excel) + 999.50 -> 999.50 + 99,999.50 -> 99,999.50 + 123,456.78 -> 1,23,456.78 + 1,234,567.89 -> 12,34,567.89 + 12,345,678.90 -> 1,23,45,678.90 + 123,456,789.10 -> 12,34,56,789.10 diff --git a/excel-export/output/02-temp-files.txt b/excel-export/output/02-temp-files.txt new file mode 100644 index 0000000..050ac76 --- /dev/null +++ b/excel-export/output/02-temp-files.txt @@ -0,0 +1,7 @@ +after write(), no dispose(): 1 temp file(s) left: [poi-sxssf-sheet2787588911680201252.xml] +after close() only: 0 temp file(s) left +after dispose(): 0 temp file(s) left + +temp file size after writing 50,000 rows (9 columns), before dispose(): + compressTempFiles=false: 24,339,585 bytes + compressTempFiles=true: 2,974,461 bytes diff --git a/excel-export/output/03-csv.txt b/excel-export/output/03-csv.txt new file mode 100644 index 0000000..e5ad2ac --- /dev/null +++ b/excel-export/output/03-csv.txt @@ -0,0 +1,18 @@ +first bytes: EF BB BF (UTF-8 BOM so Excel reads UTF-8) +rows written: 3 + Order id,Order date,Region,Product,Qty,Unit price,Amount,Status,Note + 1,2025-05-30,Chennai,Product-384,20,464.83,9296.60,RETURNED,INV/2025-26/0000001 dispatched via hub 71 + 2,2025-07-29,Pune,Product-121,20,596.20,11924.00,RETURNED,INV/2025-26/0000002 dispatched via hub 46 + 3,2025-11-22,Kolkata,Product-6,11,61.05,671.55,SHIPPED,INV/2025-26/0000003 dispatched via hub 51 + +comma, quote and newline inside fields: "Pune, MH","said ""hi""","line1\nline2"\n + + =HYPERLINK("http://x","pay") -> '=HYPERLINK("http://x","pay") + +91 98200 00000 -> '+91 98200 00000 + -2+3 -> '-2+3 + @SUM(A1) -> '@SUM(A1) + Mumbai -> Mumbai + +raw CSVWriter into a broken stream: writeNext called 10,000 times, none threw + checkError() = true, getException() = java.io.IOException: Broken pipe +CsvExporter into the same stream: IOException: CSV write failed (cause: java.io.IOException: Broken pipe) diff --git a/excel-export/output/04-endpoints.txt b/excel-export/output/04-endpoints.txt new file mode 100644 index 0000000..3c4a127 --- /dev/null +++ b/excel-export/output/04-endpoints.txt @@ -0,0 +1,3 @@ +GET /export/xssf?rows=1500 -> 200, application/vnd.openxmlformats-officedocument.spreadsheetml.sheet, attachment; filename="sales.xlsx", content-length header: none (chunked: chunked), 91,345 bytes +GET /export/sxssf?rows=1500 -> 200, application/vnd.openxmlformats-officedocument.spreadsheetml.sheet, attachment; filename="sales.xlsx", content-length header: none (chunked: chunked), 93,481 bytes +GET /export/csv?rows=1500 -> 200, text/csv;charset=UTF-8, attachment; filename="sales.csv", content-length header: none (chunked: chunked), 152,833 bytes diff --git a/excel-export/output/05-xssf-sweep.txt b/excel-export/output/05-xssf-sweep.txt new file mode 100644 index 0000000..19d1367 --- /dev/null +++ b/excel-export/output/05-xssf-sweep.txt @@ -0,0 +1,7 @@ +# XSSF under a fixed 512 MB heap: rows until it dies +# openjdk version "25.0.5" 2026-10-20 LTS | 2 vCPU | 8027 MB RAM +# indicative numbers from a shared sandbox, one run per line; see README +XSSF /export/xssf?rows=10000 Xmx=512m http=200 curl=0 bytes=576961 10.1s | peak used heap 135 MB | max after GC 112 MB | 18 GC pauses, longest 495 ms | peak RSS 397 MB | temp dir peak 0 MB, files left 0 | OOM=no +XSSF /export/xssf?rows=25000 Xmx=512m http=200 curl=0 bytes=1430086 13.8s | peak used heap 307 MB | max after GC 255 MB | 18 GC pauses, longest 327 ms | peak RSS 671 MB | temp dir peak 0 MB, files left 0 | OOM=no +XSSF /export/xssf?rows=50000 Xmx=512m http=200 curl=0 bytes=2851566 14.3s | peak used heap 494 MB | max after GC 474 MB | 26 GC pauses, longest 315 ms | peak RSS 674 MB | temp dir peak 0 MB, files left 0 | OOM=no +XSSF /export/xssf?rows=100000 Xmx=512m http=500 curl=0 bytes=107 68.3s | peak used heap - MB | max after GC 512 MB | 106 GC pauses, longest 2412 ms | peak RSS 667 MB | temp dir peak 0 MB, files left 0 | OOM=YES | body: {"timestamp":"2026-10-29T06:03:25.226Z","status":500,"error":"Internal Server Error","path":"/export/xssf"} diff --git a/excel-export/output/06-sxssf-sweep.txt b/excel-export/output/06-sxssf-sweep.txt new file mode 100644 index 0000000..3d6b113 --- /dev/null +++ b/excel-export/output/06-sxssf-sweep.txt @@ -0,0 +1,8 @@ +# SXSSF under the same 512 MB heap, same rows, plus 1,000,000 +# openjdk version "25.0.5" 2026-10-20 LTS | 2 vCPU | 8027 MB RAM +# indicative numbers from a shared sandbox, one run per line; see README +SXSSF /export/sxssf?rows=10000 Xmx=512m http=200 curl=0 bytes=598239 10.6s | peak used heap 165 MB | max after GC 18 MB | 10 GC pauses, longest 57 ms | peak RSS 333 MB | temp dir peak 4 MB, files left 0 | OOM=no +SXSSF /export/sxssf?rows=25000 Xmx=512m http=200 curl=0 bytes=1490887 3.0s | peak used heap 211 MB | max after GC 18 MB | 11 GC pauses, longest 42 ms | peak RSS 366 MB | temp dir peak 11 MB, files left 0 | OOM=no +SXSSF /export/sxssf?rows=50000 Xmx=512m http=200 curl=0 bytes=2978433 6.5s | peak used heap 307 MB | max after GC 17 MB | 11 GC pauses, longest 82 ms | peak RSS 477 MB | temp dir peak 23 MB, files left 0 | OOM=no +SXSSF /export/sxssf?rows=100000 Xmx=512m http=200 curl=0 bytes=5954167 9.9s | peak used heap 306 MB | max after GC 22 MB | 12 GC pauses, longest 78 ms | peak RSS 484 MB | temp dir peak 46 MB, files left 0 | OOM=no +SXSSF /export/sxssf?rows=1000000 Xmx=512m http=200 curl=0 bytes=59408766 36.0s | peak used heap 321 MB | max after GC 19 MB | 29 GC pauses, longest 79 ms | peak RSS 504 MB | temp dir peak 475 MB, files left 0 | OOM=no diff --git a/excel-export/output/07-million-rows.txt b/excel-export/output/07-million-rows.txt new file mode 100644 index 0000000..ea4f4a7 --- /dev/null +++ b/excel-export/output/07-million-rows.txt @@ -0,0 +1,6 @@ +# 1,000,000 rows: SXSSF and CSV under a 64 MB heap, XSSF given a large heap +# openjdk version "25.0.5" 2026-10-20 LTS | 2 vCPU | 8027 MB RAM +# indicative numbers from a shared sandbox, one run per line; see README +SXSSF /export/sxssf?rows=1000000 Xmx=64m http=200 curl=0 bytes=59408766 37.5s | peak used heap 56 MB | max after GC 20 MB | 163 GC pauses, longest 43 ms | peak RSS 208 MB | temp dir peak 475 MB, files left 0 | OOM=no +CSV /export/csv?rows=1000000 Xmx=64m http=200 curl=0 bytes=104476584 4.3s | peak used heap 52 MB | max after GC 15 MB | 82 GC pauses, longest 40 ms | peak RSS 175 MB | temp dir peak 0 MB, files left 0 | OOM=no +XSSF /export/xssf?rows=1000000 Xmx=3g http=500 curl=0 bytes=107 273.6s | peak used heap - MB | max after GC 3072 MB | 132 GC pauses, longest 12794 ms | peak RSS 3295 MB | temp dir peak 0 MB, files left 0 | OOM=YES | body: {"timestamp":"2026-10-29T06:11:02.454Z","status":500,"error":"Internal Server Error","path":"/export/xssf"} diff --git a/excel-export/output/08-sheet-rolling.txt b/excel-export/output/08-sheet-rolling.txt new file mode 100644 index 0000000..1c71f36 --- /dev/null +++ b/excel-export/output/08-sheet-rolling.txt @@ -0,0 +1,9 @@ +# Sheet rolling: 1,100,000 rows, SXSSF, 64 MB heap, default sheet limit 1,048,576 +# openjdk version "25.0.5" 2026-10-20 LTS | 2 vCPU | 8027 MB RAM +# indicative numbers from a shared sandbox, one run per line; see README +SXSSF /export/sxssf?rows=1100000 Xmx=64m http=200 curl=0 bytes=65376880 39.0s | peak used heap 56 MB | max after GC 25 MB | 178 GC pauses, longest 49 ms | peak RSS 209 MB | temp dir peak 523 MB, files left 0 | OOM=no +file size: 63 MB + 1,05/30/2025,Chennai,Product-384,20,464.83,9296.6,RETURNED,INV/2025-26/0000001 dispatched via hub 71 +sxssf.xlsx: LibreOffice: source file could not be loaded +rezipped.xlsx: LibreOffice loads it; row 2 -> 1,05/30/2025,Chennai,Product-384,20,464.83,9296.6,RETURNED,INV/2025-26/0000001 dispatched via hub 71 diff --git a/excel-export/pom.xml b/excel-export/pom.xml new file mode 100644 index 0000000..1cda920 --- /dev/null +++ b/excel-export/pom.xml @@ -0,0 +1,58 @@ + + + 4.0.0 + + + org.springframework.boot + spring-boot-starter-parent + 4.1.2 + + + + com.ankurm + excel-export + 1.0.0 + excel-export + Export Excel and CSV from Spring Boot 4.1: Apache POI XSSF vs SXSSF measured under a fixed heap, + StreamingResponseBody, OpenCSV, sheet rolling at Excel's row limit, and temp-file cleanup. + jar + + + 25 + 6.0.0 + 5.12.0 + + + + + org.springframework.boot + spring-boot-starter-webmvc + + + org.apache.poi + poi-ooxml + ${poi.version} + + + com.opencsv + opencsv + ${opencsv.version} + + + org.springframework.boot + spring-boot-starter-webmvc-test + test + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + diff --git a/excel-export/scripts/libreoffice-check.sh b/excel-export/scripts/libreoffice-check.sh new file mode 100755 index 0000000..da05fec --- /dev/null +++ b/excel-export/scripts/libreoffice-check.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Opens the same 40-row report from the XSSF and the SXSSF endpoint in headless LibreOffice, and reports what it does. +# Needs: soffice, python3, curl, a built jar. Microsoft Excel itself was NOT available for this check. +cd "$(dirname "$0")/.." +d=$(mktemp -d); P="file://$d/profile" +java -jar target/excel-export-1.0.0.jar --server.port=38091 >/dev/null 2>&1 & p=$! +for _ in $(seq 1 120); do [ "$(curl -s localhost:38091/probe/ping)" = ok ] && break; sleep 1; done +curl -s -o "$d/xssf.xlsx" "localhost:38091/export/xssf?rows=40" +curl -s -o "$d/sxssf.xlsx" "localhost:38091/export/sxssf?rows=40" +kill $p; wait $p 2>/dev/null || true +python3 - "$d" <<'PY' +import sys, zipfile +d = sys.argv[1] +with zipfile.ZipFile(d + "/sxssf.xlsx") as z, zipfile.ZipFile(d + "/rezipped.xlsx", "w", zipfile.ZIP_DEFLATED) as out: + for n in z.namelist(): + out.writestr(n, z.read(n)) +for f in ("xssf", "sxssf"): + with zipfile.ZipFile(f"{d}/{f}.xlsx") as z: + print(f"{f}.xlsx ZIP entry timestamps: {sorted({i.date_time for i in z.infolist()})}") +PY +echo "$(soffice --version 2>/dev/null | head -1)" +for f in xssf sxssf rezipped; do + (cd "$d" && timeout 120 soffice --headless "-env:UserInstallation=$P" --convert-to csv "$f.xlsx" >/dev/null 2>&1) + if [ -f "$d/$f.csv" ]; then echo "$f.xlsx: LibreOffice loads it; row 2 -> $(sed -n 2p "$d/$f.csv")"; else echo "$f.xlsx: LibreOffice: source file could not be loaded"; fi +done +rm -rf "$d" diff --git a/excel-export/scripts/measure.sh b/excel-export/scripts/measure.sh new file mode 100755 index 0000000..e2d7288 --- /dev/null +++ b/excel-export/scripts/measure.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Library: run ONE export in a fresh JVM with a fixed heap and print one result line. +# measure LABEL URL_PATH XMX [SAVE_TO_FILE] +# Every call starts its own JVM, so the numbers of one approach never contaminate another. +# Needs: curl, awk, a built jar (mvn package), Linux (/proc). +set -u +JAR="${JAR:-target/excel-export-1.0.0.jar}" +PORT="${PORT:-$((30000 + RANDOM % 20000))}" +WORK="${WORK:-/tmp/excel-export-measure}" + +measure() { + local label="$1" path="$2" xmx="$3" save="${4:-/dev/null}" + rm -rf "$WORK"; mkdir -p "$WORK/tmp" + java -Xms"$xmx" -Xmx"$xmx" -XX:+UseG1GC -Xlog:gc:file="$WORK/gc.log" -XX:OnOutOfMemoryError="touch $WORK/oom" \ + -Djava.io.tmpdir="$WORK/tmp" -jar "$JAR" --server.port="$PORT" > "$WORK/app.log" 2>&1 & + local pid=$! + for _ in $(seq 1 400); do [ "$(curl -s "http://localhost:$PORT/probe/ping")" = ok ] && break; sleep 0.5; done + + # sample the size of the temp directory while the download runs + ( peak=0; while kill -0 "$pid" 2>/dev/null && [ ! -f "$WORK/done" ]; do + s=$(du -sk "$WORK/tmp" 2>/dev/null | cut -f1); [ "${s:-0}" -gt "$peak" ] && peak=$s; echo "$peak" > "$WORK/tmp.peak"; sleep 0.2 + done ) & + local sampler=$! + + local res + local body="$WORK/body.txt"; [ "$save" = /dev/null ] && save="$body" + res=$(curl -s --max-time "${MAX_TIME:-300}" -o "$save" -w "%{http_code} %{size_download} %{time_total}" "http://localhost:$PORT$path"; echo " curl_exit=$?") + touch "$WORK/done"; wait "$sampler" 2>/dev/null || true + + local http bytes errbody="" secs cexit probe peakUsed rss + read -r http bytes secs _ <<< "$res" + cexit=$(echo "$res" | sed 's/.*curl_exit=//') + probe=$(curl -s --max-time 5 "http://localhost:$PORT/probe" || true) + peakUsed=$(echo "$probe" | sed -n 's/.*"peakUsedHeapMb":\([0-9]*\).*/\1/p') + rss=$(awk '/VmHWM/ {printf "%d", $2/1024}' "/proc/$pid/status" 2>/dev/null) + local tmpafter; tmpafter=$(find "$WORK/tmp" -type f 2>/dev/null | wc -l) + local oom=no; [ -f "$WORK/oom" ] && oom=YES + + [ "$http" != 200 ] && [ -f "$body" ] && errbody=" | body: $(head -c 120 "$body" | tr -d '\n')" + kill "$pid" 2>/dev/null || true; wait "$pid" 2>/dev/null || true + + # largest heap occupancy right after a collection, number of pauses, longest pause + local gc + gc=$(grep -oE '[0-9]+M->[0-9]+M\([0-9]+M\) [0-9.]+ms' "$WORK/gc.log" | tr '>()M-' ' ' | sed 's/ms//' \ + | awk '{ if ($2>a) a=$2; n++; if ($4>p) p=$4 } END { printf "%d %d %.0f", a, n, p }') + local maxafter pauses maxpause; read -r maxafter pauses maxpause <<< "$gc" + local tmppeak; tmppeak=$(( $(cat "$WORK/tmp.peak" 2>/dev/null || echo 0) / 1024 )) + + printf '%-7s %-34s Xmx=%-5s http=%-3s curl=%-2s bytes=%-10s %6.1fs | peak used heap %5s MB | max after GC %5s MB | %4s GC pauses, longest %4s ms | peak RSS %5s MB | temp dir peak %4s MB, files left %s | OOM=%s%s\n' \ + "$label" "$path" "$xmx" "$http" "$cexit" "$bytes" "$secs" "${peakUsed:--}" "$maxafter" "$pauses" "$maxpause" "${rss:--}" "$tmppeak" "$tmpafter" "$oom" "$errbody" +} diff --git a/excel-export/scripts/run-all.sh b/excel-export/scripts/run-all.sh new file mode 100755 index 0000000..c867981 --- /dev/null +++ b/excel-export/scripts/run-all.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Regenerates every file in output/. Takes 15-25 minutes on a shared 2-vCPU box, mostly the XSSF runs. +# Needs JDK 25, Maven, curl, awk, unzip. LibreOffice is optional (step 10 is skipped without it). +set -euo pipefail +cd "$(dirname "$0")/.." +source scripts/measure.sh +mkdir -p output +HOST="$(java -version 2>&1 | grep -v JAVA_TOOL | head -1) | $(nproc) vCPU | $(awk '/MemTotal/ {printf "%d MB RAM", $2/1024}' /proc/meminfo)" + +if [ -z "${SKIP_BUILD:-}" ]; then + mvn -q -B test # writes output/01..04 (structure, temp files, csv, endpoints) + mvn -q -B package -DskipTests +fi + +hdr() { { echo "# $1"; echo "# $HOST"; echo "# indicative numbers from a shared sandbox, one run per line; see README"; } > "$2"; } + +hdr "XSSF under a fixed 512 MB heap: rows until it dies" output/05-xssf-sweep.txt +for n in ${XSSF_ROWS:-10000 25000 50000 100000}; do + measure XSSF "/export/xssf?rows=$n" 512m >> output/05-xssf-sweep.txt +done + +hdr "SXSSF under the same 512 MB heap, same rows, plus 1,000,000" output/06-sxssf-sweep.txt +for n in ${XSSF_ROWS:-10000 25000 50000 100000} 1000000; do + measure SXSSF "/export/sxssf?rows=$n" 512m >> output/06-sxssf-sweep.txt +done + +hdr "1,000,000 rows: SXSSF and CSV under a 64 MB heap, XSSF given a large heap" output/07-million-rows.txt +measure SXSSF "/export/sxssf?rows=1000000" 64m >> output/07-million-rows.txt +measure CSV "/export/csv?rows=1000000" 64m >> output/07-million-rows.txt +measure XSSF "/export/xssf?rows=1000000" "${XSSF_BIG_HEAP:-3g}" >> output/07-million-rows.txt + +hdr "Sheet rolling: 1,100,000 rows, SXSSF, 64 MB heap, default sheet limit 1,048,576" output/08-sheet-rolling.txt +measure SXSSF "/export/sxssf?rows=1100000" 64m /tmp/excel-export-big.xlsx >> output/08-sheet-rolling.txt +{ + echo "file size: $(du -m /tmp/excel-export-big.xlsx | cut -f1) MB" + unzip -p /tmp/excel-export-big.xlsx xl/workbook.xml | grep -oE ']*name="[^"]*"' | sed 's/ r:id.*//' + for s in 1 2; do + echo "sheet$s.xml: $(unzip -p /tmp/excel-export-big.xlsx xl/worksheets/sheet$s.xml | grep -o '> output/08-sheet-rolling.txt +rm -f /tmp/excel-export-big.xlsx + +hdr "compressTempFiles on a 1,000,000-row SXSSF export, 64 MB heap" output/09-compress-temp.txt +measure SXSSF "/export/sxssf?rows=1000000&compress=false" 64m >> output/09-compress-temp.txt +measure SXSSF "/export/sxssf?rows=1000000&compress=true" 64m >> output/09-compress-temp.txt + +if command -v soffice >/dev/null; then + bash scripts/libreoffice-check.sh > output/10-libreoffice.txt 2>&1 || true +fi +echo "done" diff --git a/excel-export/src/main/java/com/ankurm/excelexport/CsvExporter.java b/excel-export/src/main/java/com/ankurm/excelexport/CsvExporter.java new file mode 100644 index 0000000..9eb4156 --- /dev/null +++ b/excel-export/src/main/java/com/ankurm/excelexport/CsvExporter.java @@ -0,0 +1,60 @@ +package com.ankurm.excelexport; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.util.Iterator; + +import com.opencsv.CSVWriter; +import com.opencsv.ICSVWriter; + +/** Streams {@link SaleRow}s as CSV with OpenCSV. Memory use does not depend on the row count. */ +public final class CsvExporter { + + private static final String[] HEADERS = {"Order id", "Order date", "Region", "Product", "Qty", "Unit price", "Amount", "Status", "Note"}; + + private CsvExporter() { } + + /** + * @param excelBom write a UTF-8 byte-order mark so Excel opens the file as UTF-8 instead of the Windows code page + * @return the number of data rows written + */ + public static long write(Iterator rows, OutputStream out, boolean excelBom) throws IOException { + if (excelBom) { + out.write(new byte[] {(byte) 0xEF, (byte) 0xBB, (byte) 0xBF}); + } + Writer w = new OutputStreamWriter(out, StandardCharsets.UTF_8); + ICSVWriter csv = new CSVWriter(w); + csv.writeNext(HEADERS, false); + long n = 0; + while (rows.hasNext()) { + SaleRow r = rows.next(); + csv.writeNext(new String[] { + Long.toString(r.id()), r.orderDate().toString(), safe(r.region()), safe(r.product()), + Integer.toString(r.quantity()), r.unitPrice().toPlainString(), r.amount().toPlainString(), + safe(r.status()), safe(r.note())}, false); + n++; + } + if (csv.checkError()) { // writeNext() swallows IOExceptions and only records them; see CsvTest + throw new IOException("CSV write failed", csv.getException()); + } + csv.flush(); + return n; + } + + /** + * CSV injection guard: a text cell that starts with = + - @ (or a tab or CR) is evaluated as a formula by + * Excel when the CSV is opened. Prefix a single quote so it is shown as text. + */ + static String safe(String s) { + if (s != null && !s.isEmpty()) { + char c = s.charAt(0); + if (c == '=' || c == '+' || c == '-' || c == '@' || c == '\t' || c == '\r') { + return "'" + s; + } + } + return s; + } +} diff --git a/excel-export/src/main/java/com/ankurm/excelexport/ExcelExportApplication.java b/excel-export/src/main/java/com/ankurm/excelexport/ExcelExportApplication.java new file mode 100644 index 0000000..9e6b934 --- /dev/null +++ b/excel-export/src/main/java/com/ankurm/excelexport/ExcelExportApplication.java @@ -0,0 +1,12 @@ +package com.ankurm.excelexport; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class ExcelExportApplication { + + public static void main(String[] args) { + SpringApplication.run(ExcelExportApplication.class, args); + } +} diff --git a/excel-export/src/main/java/com/ankurm/excelexport/ExportController.java b/excel-export/src/main/java/com/ankurm/excelexport/ExportController.java new file mode 100644 index 0000000..62ebdf6 --- /dev/null +++ b/excel-export/src/main/java/com/ankurm/excelexport/ExportController.java @@ -0,0 +1,70 @@ +package com.ankurm.excelexport; + +import org.apache.poi.xssf.streaming.SXSSFWorkbook; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.springframework.http.ContentDisposition; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; + +@RestController +public class ExportController { + + static final MediaType XLSX = MediaType.parseMediaType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + static final MediaType CSV = MediaType.parseMediaType("text/csv;charset=UTF-8"); + + /** + * The naive version: XSSFWorkbook keeps every cell as an object until write(). StreamingResponseBody does + * not help here: the response is "streamed" only after the workbook has been built in full. + */ + @GetMapping("/export/xssf") + public ResponseEntity xssf(@RequestParam(defaultValue = "100000") int rows, + @RequestParam(defaultValue = "" + XlsxExporter.EXCEL_MAX_ROWS) int sheetLimit) { + StreamingResponseBody body = out -> { + try (XSSFWorkbook wb = new XSSFWorkbook()) { + XlsxExporter.write(wb, SaleSource.rows(rows), sheetLimit, out); + } + }; + return download("sales.xlsx", XLSX, body); + } + + /** + * Streaming version: SXSSFWorkbook(100) keeps 100 rows per sheet in memory and writes older rows to a temp + * file. dispose() deletes that temp file; it runs in finally so a failed or aborted download cleans up too. + */ + @GetMapping("/export/sxssf") + public ResponseEntity sxssf(@RequestParam(defaultValue = "100000") int rows, + @RequestParam(defaultValue = "false") boolean compress, + @RequestParam(defaultValue = "100") int window, + @RequestParam(defaultValue = "" + XlsxExporter.EXCEL_MAX_ROWS) int sheetLimit) { + StreamingResponseBody body = out -> { + SXSSFWorkbook wb = new SXSSFWorkbook(window); + wb.setCompressTempFiles(compress); + try { + XlsxExporter.write(wb, SaleSource.rows(rows), sheetLimit, out); + } finally { + wb.dispose(); + wb.close(); + } + }; + return download("sales.xlsx", XLSX, body); + } + + @GetMapping("/export/csv") + public ResponseEntity csv(@RequestParam(defaultValue = "100000") int rows, + @RequestParam(defaultValue = "true") boolean bom) { + StreamingResponseBody body = out -> CsvExporter.write(SaleSource.rows(rows), out, bom); + return download("sales.csv", CSV, body); + } + + private static ResponseEntity download(String filename, MediaType type, StreamingResponseBody body) { + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_DISPOSITION, ContentDisposition.attachment().filename(filename).build().toString()) + .contentType(type) + .body(body); + } +} diff --git a/excel-export/src/main/java/com/ankurm/excelexport/ProbeController.java b/excel-export/src/main/java/com/ankurm/excelexport/ProbeController.java new file mode 100644 index 0000000..742c796 --- /dev/null +++ b/excel-export/src/main/java/com/ankurm/excelexport/ProbeController.java @@ -0,0 +1,60 @@ +package com.ankurm.excelexport; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; + +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * Diagnostic endpoint for the measurements. It samples used heap every 20 ms and reports the peak, plus the + * process's peak resident set size from /proc (Linux only). Delete it before shipping anything. + */ +@RestController +public class ProbeController { + + private final AtomicLong peakUsed = new AtomicLong(); + + public ProbeController() { + Thread t = new Thread(() -> { + Runtime rt = Runtime.getRuntime(); + while (true) { + peakUsed.accumulateAndGet(rt.totalMemory() - rt.freeMemory(), Math::max); + try { + Thread.sleep(20); + } catch (InterruptedException e) { + return; + } + } + }, "heap-peak-sampler"); + t.setDaemon(true); + t.start(); + } + + @GetMapping("/probe/ping") + public String ping() { + return "ok"; + } + + @GetMapping("/probe") + public Map probe() throws Exception { + Map m = new LinkedHashMap<>(); + Runtime rt = Runtime.getRuntime(); + m.put("maxHeapMb", rt.maxMemory() >> 20); + m.put("peakUsedHeapMb", peakUsed.get() >> 20); + m.put("peakRssMb", procKb("VmHWM") >> 10); + return m; + } + + private static long procKb(String key) throws Exception { + try (Stream lines = Files.lines(Path.of("/proc/self/status"))) { + return lines.filter(l -> l.startsWith(key + ":")) + .map(l -> Long.parseLong(l.replaceAll("\\D+", ""))) + .findFirst().orElse(-1L); + } + } +} diff --git a/excel-export/src/main/java/com/ankurm/excelexport/SaleRow.java b/excel-export/src/main/java/com/ankurm/excelexport/SaleRow.java new file mode 100644 index 0000000..b565775 --- /dev/null +++ b/excel-export/src/main/java/com/ankurm/excelexport/SaleRow.java @@ -0,0 +1,13 @@ +package com.ankurm.excelexport; + +import java.math.BigDecimal; +import java.time.LocalDate; + +/** One line of the report. Nine columns once the computed amount is added; the note is unique per row on purpose. */ +public record SaleRow(long id, LocalDate orderDate, String region, String product, int quantity, + BigDecimal unitPrice, String status, String note) { + + public BigDecimal amount() { + return unitPrice.multiply(BigDecimal.valueOf(quantity)); + } +} diff --git a/excel-export/src/main/java/com/ankurm/excelexport/SaleSource.java b/excel-export/src/main/java/com/ankurm/excelexport/SaleSource.java new file mode 100644 index 0000000..535732c --- /dev/null +++ b/excel-export/src/main/java/com/ankurm/excelexport/SaleSource.java @@ -0,0 +1,54 @@ +package com.ankurm.excelexport; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.util.Iterator; +import java.util.NoSuchElementException; + +/** + * A deterministic stand-in for a database cursor: row n is computed from n, so nothing is stored and the + * same call always yields the same rows. In a real service this is a JDBC {@code ResultSet} or a JPA + * {@code Stream} with a fetch size; the exporters only ever see an {@link Iterator}. + */ +public final class SaleSource { + + private static final String[] REGIONS = {"Mumbai", "Pune", "Delhi", "Bengaluru", "Chennai", "Hyderabad", "Kolkata", "Ahmedabad"}; + private static final String[] STATUSES = {"PAID", "PENDING", "SHIPPED", "RETURNED"}; + private static final LocalDate BASE = LocalDate.of(2025, 4, 1); + + private SaleSource() { } + + public static Iterator rows(int count) { + return new Iterator<>() { + private long next = 1; + + @Override public boolean hasNext() { + return next <= count; + } + + @Override public SaleRow next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return row(next++); + } + }; + } + + public static SaleRow row(long id) { + long h = id * 0x9E3779B97F4A7C15L; // cheap deterministic scrambler + int a = (int) ((h >>> 33) & 0x7fffffff); + int b = (int) ((h >>> 11) & 0x7fffffff); + String product = "Product-" + (a % 500); + int qty = 1 + b % 20; + BigDecimal price = BigDecimal.valueOf(99 + (a % 99_900), 2); + return new SaleRow(id, + BASE.plusDays(b % 365), + REGIONS[a % REGIONS.length], + product, + qty, + price, + STATUSES[b % STATUSES.length], + "INV/2025-26/" + String.format("%07d", id) + " dispatched via hub " + (b % 97)); + } +} diff --git a/excel-export/src/main/java/com/ankurm/excelexport/XlsxExporter.java b/excel-export/src/main/java/com/ankurm/excelexport/XlsxExporter.java new file mode 100644 index 0000000..0093eb8 --- /dev/null +++ b/excel-export/src/main/java/com/ankurm/excelexport/XlsxExporter.java @@ -0,0 +1,128 @@ +package com.ankurm.excelexport; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Iterator; + +import org.apache.poi.ss.usermodel.BorderStyle; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellStyle; +import org.apache.poi.ss.usermodel.FillPatternType; +import org.apache.poi.ss.usermodel.Font; +import org.apache.poi.ss.usermodel.HorizontalAlignment; +import org.apache.poi.ss.usermodel.IndexedColors; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.util.CellRangeAddress; + +/** + * Writes {@link SaleRow}s into any POI {@link Workbook}. The same code drives {@code XSSFWorkbook} (everything + * in heap) and {@code SXSSFWorkbook} (a sliding window of rows in heap, the rest on disk), so the only thing + * that differs between the two measured endpoints is the constructor call. + * + *

Explained in the post section "Styling and the 64,000 cell-style limit". + */ +public final class XlsxExporter { + + /** Excel's hard limit per worksheet, header row included (rows 0..1,048,575). */ + public static final int EXCEL_MAX_ROWS = 1_048_576; + + private static final String[] HEADERS = {"Order id", "Order date", "Region", "Product", "Qty", "Unit price", "Amount", "Status", "Note"}; + private static final int[] WIDTH_CHARS = {12, 13, 12, 14, 7, 14, 16, 11, 42}; + + /** Lakh/crore grouping (12,34,567.89) as an Excel number format: two conditions plus a default section. */ + static final String INR_FORMAT = "[>=10000000]##\\,##\\,##\\,##0.00;[>=100000]##\\,##\\,##0.00;##,##0.00"; + + /** The styles are created once per workbook. A style per cell would hit Excel's limit of about 64,000 styles. */ + private record Styles(CellStyle header, CellStyle date, CellStyle integer, CellStyle money) { } + + private XlsxExporter() { } + + /** + * @param maxRowsPerSheet rows per sheet including the header row; {@link #EXCEL_MAX_ROWS} in production, + * smaller in tests so rolling can be seen without a million rows + * @return the number of sheets written + */ + public static int write(Workbook wb, Iterator rows, int maxRowsPerSheet, OutputStream out) throws IOException { + if (maxRowsPerSheet < 2 || maxRowsPerSheet > EXCEL_MAX_ROWS) { + throw new IllegalArgumentException("maxRowsPerSheet must be 2.." + EXCEL_MAX_ROWS); + } + Styles st = styles(wb); + int sheets = 0; + Sheet sheet = null; + int rowIdx = 0; + while (rows.hasNext()) { + if (sheet == null || rowIdx >= maxRowsPerSheet) { + sheet = newSheet(wb, ++sheets, st); + rowIdx = 1; // row 0 is the header + } + SaleRow r = rows.next(); + Row row = sheet.createRow(rowIdx++); + row.createCell(0).setCellValue(r.id()); + Cell d = row.createCell(1); + d.setCellValue(r.orderDate()); + d.setCellStyle(st.date()); + row.createCell(2).setCellValue(r.region()); + row.createCell(3).setCellValue(r.product()); + Cell q = row.createCell(4); + q.setCellValue(r.quantity()); + q.setCellStyle(st.integer()); + Cell p = row.createCell(5); + p.setCellValue(r.unitPrice().doubleValue()); + p.setCellStyle(st.money()); + Cell a = row.createCell(6); + a.setCellValue(r.amount().doubleValue()); + a.setCellStyle(st.money()); + row.createCell(7).setCellValue(r.status()); + row.createCell(8).setCellValue(r.note()); + } + if (sheets == 0) { + newSheet(wb, 1, st); // an empty report still opens in Excel + sheets = 1; + } + wb.write(out); + return sheets; + } + + private static Sheet newSheet(Workbook wb, int n, Styles st) { + Sheet sheet = wb.createSheet(n == 1 ? "Sales" : "Sales " + n); + for (int c = 0; c < WIDTH_CHARS.length; c++) { + sheet.setColumnWidth(c, WIDTH_CHARS[c] * 256); // fixed widths: autoSizeColumn on SXSSF would need every row kept + } + Row h = sheet.createRow(0); + for (int c = 0; c < HEADERS.length; c++) { + Cell cell = h.createCell(c); + cell.setCellValue(HEADERS[c]); + cell.setCellStyle(st.header()); + } + sheet.createFreezePane(0, 1); + sheet.setAutoFilter(new CellRangeAddress(0, 0, 0, HEADERS.length - 1)); + return sheet; + } + + private static Styles styles(Workbook wb) { + var df = wb.createDataFormat(); + + Font bold = wb.createFont(); + bold.setBold(true); + bold.setColor(IndexedColors.WHITE.getIndex()); + CellStyle header = wb.createCellStyle(); + header.setFont(bold); + header.setFillForegroundColor(IndexedColors.DARK_BLUE.getIndex()); + header.setFillPattern(FillPatternType.SOLID_FOREGROUND); + header.setAlignment(HorizontalAlignment.CENTER); + header.setBorderBottom(BorderStyle.THIN); + + CellStyle date = wb.createCellStyle(); + date.setDataFormat(df.getFormat("dd-mmm-yyyy")); + date.setAlignment(HorizontalAlignment.CENTER); + + CellStyle integer = wb.createCellStyle(); + integer.setDataFormat(df.getFormat("#,##0")); + + CellStyle money = wb.createCellStyle(); + money.setDataFormat(df.getFormat(INR_FORMAT)); + return new Styles(header, date, integer, money); + } +} diff --git a/excel-export/src/main/resources/application.properties b/excel-export/src/main/resources/application.properties new file mode 100644 index 0000000..e437d9a --- /dev/null +++ b/excel-export/src/main/resources/application.properties @@ -0,0 +1,2 @@ +# Streaming downloads can run for minutes on a slow link; the default async timeout would cut them off. +spring.mvc.async.request-timeout=10m diff --git a/excel-export/src/test/java/com/ankurm/excelexport/CsvTest.java b/excel-export/src/test/java/com/ankurm/excelexport/CsvTest.java new file mode 100644 index 0000000..3350d7f --- /dev/null +++ b/excel-export/src/test/java/com/ankurm/excelexport/CsvTest.java @@ -0,0 +1,97 @@ +package com.ankurm.excelexport; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.util.Iterator; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import com.opencsv.CSVWriter; + +@org.junit.jupiter.api.TestMethodOrder(org.junit.jupiter.api.MethodOrderer.MethodName.class) +class CsvTest { + + static final String F = "03-csv.txt"; + + @BeforeAll + static void clean() { + Out.reset(F); + } + + @Test + void a_shapeBomAndQuoting() throws Exception { + var bos = new ByteArrayOutputStream(); + long n = CsvExporter.write(SaleSource.rows(3), bos, true); + byte[] raw = bos.toByteArray(); + String text = new String(raw, StandardCharsets.UTF_8); + Out.say(F, "first bytes: %02X %02X %02X (UTF-8 BOM so Excel reads UTF-8)".formatted(raw[0], raw[1], raw[2])); + Out.say(F, "rows written: " + n); + text.substring(1).lines().forEach(l -> Out.say(F, " " + l)); + assertThat(raw[0]).isEqualTo((byte) 0xEF); + assertThat(n).isEqualTo(3); + + var plain = new ByteArrayOutputStream(); + CsvExporter.write(SaleSource.rows(0), plain, false); + assertThat(plain.toByteArray()[0]).isEqualTo((byte) 'O'); + + var tricky = new ByteArrayOutputStream(); + var w = new CSVWriter(new OutputStreamWriter(tricky, StandardCharsets.UTF_8)); + w.writeNext(new String[] {"Pune, MH", "said \"hi\"", "line1\nline2"}, false); + w.flush(); + Out.say(F, ""); + Out.say(F, "comma, quote and newline inside fields: " + tricky.toString(StandardCharsets.UTF_8).replace("\n", "\\n")); + assertThat(tricky.toString(StandardCharsets.UTF_8)).startsWith("\"Pune, MH\",\"said \"\"hi\"\"\",\"line1\\nline2\"".replace("\\n", "\n")); + } + + @Test + void b_formulaInjectionIsNeutralised() { + Out.say(F, ""); + for (String s : new String[] {"=HYPERLINK(\"http://x\",\"pay\")", "+91 98200 00000", "-2+3", "@SUM(A1)", "Mumbai"}) { + Out.say(F, " %-32s -> %s".formatted(s, CsvExporter.safe(s))); + } + assertThat(CsvExporter.safe("=1+1")).isEqualTo("'=1+1"); + assertThat(CsvExporter.safe("Mumbai")).isEqualTo("Mumbai"); + } + + /** An OutputStream that fails after a few kilobytes, like a client that closed the browser tab. */ + static OutputStream breaksAfter(int bytes) { + return new OutputStream() { + int n; + @Override public void write(int b) throws IOException { + if (++n > bytes) { + throw new IOException("Broken pipe"); + } + } + }; + } + + @Test + void c_openCsvSwallowsWriteErrors() throws Exception { + Out.say(F, ""); + // 1. raw CSVWriter: 10,000 rows into a stream that breaks after 1,000 bytes + var w = new CSVWriter(new OutputStreamWriter(breaksAfter(1_000), StandardCharsets.UTF_8)); + Iterator it = SaleSource.rows(10_000); + int written = 0; + while (it.hasNext()) { + w.writeNext(new String[] {it.next().note()}, false); + written++; + } + Out.say(F, "raw CSVWriter into a broken stream: writeNext called %,d times, none threw".formatted(written)); + Out.say(F, " checkError() = %s, getException() = %s".formatted(w.checkError(), w.getException())); + assertThat(written).isEqualTo(10_000); + assertThat(w.checkError()).isTrue(); + + // 2. our exporter checks and throws + assertThatThrownBy(() -> CsvExporter.write(SaleSource.rows(10_000), breaksAfter(1_000), false)) + .isInstanceOf(IOException.class) + .satisfies(e -> Out.say(F, "CsvExporter into the same stream: " + e.getClass().getSimpleName() + ": " + e.getMessage() + + " (cause: " + e.getCause() + ")")); + } +} diff --git a/excel-export/src/test/java/com/ankurm/excelexport/EndpointTest.java b/excel-export/src/test/java/com/ankurm/excelexport/EndpointTest.java new file mode 100644 index 0000000..a3c14f0 --- /dev/null +++ b/excel-export/src/test/java/com/ankurm/excelexport/EndpointTest.java @@ -0,0 +1,52 @@ +package com.ankurm.excelexport; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.ByteArrayInputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; + +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.server.LocalServerPort; + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +class EndpointTest { + + static final String F = "04-endpoints.txt"; + static final HttpClient HTTP = HttpClient.newHttpClient(); + + @LocalServerPort int port; + + @BeforeAll + static void clean() { + Out.reset(F); + } + + HttpResponse get(String path) throws Exception { + return HTTP.send(HttpRequest.newBuilder(URI.create("http://localhost:" + port + path)).build(), + HttpResponse.BodyHandlers.ofByteArray()); + } + + @Test + void headersAndBodies() throws Exception { + for (String p : new String[] {"/export/xssf?rows=1500", "/export/sxssf?rows=1500", "/export/csv?rows=1500"}) { + HttpResponse r = get(p); + Out.say(F, "GET %s -> %d, %s, %s, content-length header: %s, %,d bytes".formatted(p, r.statusCode(), + r.headers().firstValue("Content-Type").orElse("-"), + r.headers().firstValue("Content-Disposition").orElse("-"), + r.headers().firstValue("Content-Length").orElse("none (chunked: " + r.headers().firstValue("Transfer-Encoding").orElse("-") + ")"), + r.body().length)); + assertThat(r.statusCode()).isEqualTo(200); + if (p.contains("xssf")) { + try (var wb = new XSSFWorkbook(new ByteArrayInputStream(r.body()))) { + assertThat(wb.getSheetAt(0).getLastRowNum()).isEqualTo(1500); + } + } + } + } +} diff --git a/excel-export/src/test/java/com/ankurm/excelexport/Out.java b/excel-export/src/test/java/com/ankurm/excelexport/Out.java new file mode 100644 index 0000000..6ddb652 --- /dev/null +++ b/excel-export/src/test/java/com/ankurm/excelexport/Out.java @@ -0,0 +1,34 @@ +package com.ankurm.excelexport; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +/** Writes the transcript files under output/ that the post quotes. The tests assert the same numbers they print. */ +final class Out { + + static final Path DIR = Path.of("output"); + + private Out() { } + + static void reset(String file) { + try { + Files.createDirectories(DIR); + Files.deleteIfExists(DIR.resolve(file)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + static void say(String file, String line) { + System.out.println(line); + try { + Files.writeString(DIR.resolve(file), line + System.lineSeparator(), + StandardOpenOption.CREATE, StandardOpenOption.APPEND); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/excel-export/src/test/java/com/ankurm/excelexport/TempFileTest.java b/excel-export/src/test/java/com/ankurm/excelexport/TempFileTest.java new file mode 100644 index 0000000..da9165a --- /dev/null +++ b/excel-export/src/test/java/com/ankurm/excelexport/TempFileTest.java @@ -0,0 +1,87 @@ +package com.ankurm.excelexport; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; + +import org.apache.poi.util.DefaultTempFileCreationStrategy; +import org.apache.poi.util.TempFile; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** SXSSF spills rows to temp files. This shows when they are removed and what compressTempFiles changes. */ +@org.junit.jupiter.api.TestMethodOrder(org.junit.jupiter.api.MethodOrderer.MethodName.class) +class TempFileTest { + + static final String F = "02-temp-files.txt"; + + @BeforeAll + static void clean() { + Out.reset(F); + } + + static String[] list(Path dir) { + String[] names = dir.toFile().list(); + Arrays.sort(names); + return names; + } + + static long sizeOf(Path dir) { + long t = 0; + for (File f : dir.toFile().listFiles()) { + t += f.length(); + } + return t; + } + + @Test + void a_whenTheTempFilesGoAway(@TempDir Path dir) throws Exception { + TempFile.setTempFileCreationStrategy(new DefaultTempFileCreationStrategy(dir.toFile())); + + // 1. exported, written, never disposed + SXSSFWorkbook leaked = new SXSSFWorkbook(100); + XlsxExporter.write(leaked, SaleSource.rows(20_000), XlsxExporter.EXCEL_MAX_ROWS, new ByteArrayOutputStream()); + Out.say(F, "after write(), no dispose(): %d temp file(s) left: %s".formatted(list(dir).length, Arrays.toString(list(dir)))); + assertThat(list(dir)).hasSize(1); + + // 2. close() alone + leaked.close(); + Out.say(F, "after close() only: %d temp file(s) left".formatted(list(dir).length)); + int afterClose = list(dir).length; + + // 3. dispose() + leaked.dispose(); + Out.say(F, "after dispose(): %d temp file(s) left".formatted(list(dir).length)); + assertThat(list(dir)).isEmpty(); + assertThat(afterClose).isIn(0, 1); + } + + @Test + void b_compressTempFilesShrinksTheSpill(@TempDir Path dir) throws Exception { + TempFile.setTempFileCreationStrategy(new DefaultTempFileCreationStrategy(dir.toFile())); + long[] sizes = new long[2]; + for (int i = 0; i < 2; i++) { + SXSSFWorkbook wb = new SXSSFWorkbook(100); + wb.setCompressTempFiles(i == 1); + try { + XlsxExporter.write(wb, SaleSource.rows(50_000), XlsxExporter.EXCEL_MAX_ROWS, new ByteArrayOutputStream()); + sizes[i] = sizeOf(dir); // measured after write(), before dispose() + } finally { + wb.dispose(); + wb.close(); + } + } + Out.say(F, ""); + Out.say(F, "temp file size after writing 50,000 rows (9 columns), before dispose():"); + Out.say(F, " compressTempFiles=false: %,d bytes".formatted(sizes[0])); + Out.say(F, " compressTempFiles=true: %,d bytes".formatted(sizes[1])); + assertThat(sizes[1]).isLessThan(sizes[0]); + assertThat(Files.list(dir).count()).isZero(); + } +} diff --git a/excel-export/src/test/java/com/ankurm/excelexport/XlsxStructureTest.java b/excel-export/src/test/java/com/ankurm/excelexport/XlsxStructureTest.java new file mode 100644 index 0000000..f3dd62c --- /dev/null +++ b/excel-export/src/test/java/com/ankurm/excelexport/XlsxStructureTest.java @@ -0,0 +1,139 @@ +package com.ankurm.excelexport; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; + +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; +import org.apache.poi.xssf.usermodel.XSSFSheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +@org.junit.jupiter.api.TestMethodOrder(org.junit.jupiter.api.MethodOrderer.MethodName.class) +class XlsxStructureTest { + + static final String F = "01-xlsx-structure.txt"; + + @BeforeAll + static void clean() { + Out.reset(F); + } + + static byte[] xssf(int rows, int limit) throws Exception { + var bos = new ByteArrayOutputStream(); + try (XSSFWorkbook wb = new XSSFWorkbook()) { + XlsxExporter.write(wb, SaleSource.rows(rows), limit, bos); + } + return bos.toByteArray(); + } + + static byte[] sxssf(int rows, int limit) throws Exception { + var bos = new ByteArrayOutputStream(); + SXSSFWorkbook wb = new SXSSFWorkbook(100); + try { + XlsxExporter.write(wb, SaleSource.rows(rows), limit, bos); + } finally { + wb.dispose(); + wb.close(); + } + return bos.toByteArray(); + } + + @Test + void a_sheetsRollAndBothWorkbookTypesAgree() throws Exception { + // 2,500 data rows with a pretend limit of 1,000 rows per sheet (header included) = 999 data rows per sheet. + byte[] a = xssf(2_500, 1_000); + byte[] b = sxssf(2_500, 1_000); + DataFormatter fmt = new DataFormatter(); + try (var x = new XSSFWorkbook(new ByteArrayInputStream(a)); var s = new XSSFWorkbook(new ByteArrayInputStream(b))) { + Out.say(F, "sheetLimit=1000 (header included), 2500 data rows"); + assertThat(x.getNumberOfSheets()).isEqualTo(3).isEqualTo(s.getNumberOfSheets()); + for (int i = 0; i < 3; i++) { + Sheet sx = x.getSheetAt(i); + Sheet ss = s.getSheetAt(i); + Out.say(F, " sheet %d \"%s\": last row index %d, first data cell %s".formatted( + i + 1, sx.getSheetName(), sx.getLastRowNum(), fmt.formatCellValue(sx.getRow(1).getCell(0)))); + assertThat(ss.getSheetName()).isEqualTo(sx.getSheetName()); + assertThat(ss.getLastRowNum()).isEqualTo(sx.getLastRowNum()); + for (int r = 0; r <= sx.getLastRowNum(); r += 97) { + for (int c = 0; c < 9; c++) { + assertThat(fmt.formatCellValue(ss.getRow(r).getCell(c))) + .isEqualTo(fmt.formatCellValue(sx.getRow(r).getCell(c))); + } + } + } + assertThat(x.getSheetAt(0).getLastRowNum()).isEqualTo(999); + assertThat(x.getSheetAt(2).getLastRowNum()).isEqualTo(2_500 - 2 * 999); + Out.say(F, "XSSF and SXSSF files: same sheets, same formatted cell values on every 97th row"); + } + } + + @Test + void b_stylingSurvivesAndStyleCountStaysSmall() throws Exception { + byte[] b = sxssf(5_000, XlsxExporter.EXCEL_MAX_ROWS); + DataFormatter fmt = new DataFormatter(); + try (var wb = new XSSFWorkbook(new ByteArrayInputStream(b))) { + XSSFSheet sh = wb.getSheetAt(0); + Row h = sh.getRow(0); + Out.say(F, ""); + Out.say(F, "styling of a 5,000-row SXSSF file, read back with XSSF"); + Out.say(F, " header cell A1 text=%s bold=%s".formatted(h.getCell(0).getStringCellValue(), + wb.getFontAt(h.getCell(0).getCellStyle().getFontIndex()).getBold())); + Out.say(F, " freeze pane: row split at %d".formatted(sh.getPaneInformation().getHorizontalSplitPosition())); + Out.say(F, " auto filter: " + sh.getCTWorksheet().getAutoFilter().getRef()); + Out.say(F, " cell styles in the workbook: %d (for 5,000 rows x 9 cells)".formatted(wb.getNumCellStyles())); + Out.say(F, " B2 as Excel shows it: " + fmt.formatCellValue(sh.getRow(1).getCell(1))); + Out.say(F, " amount number format: " + sh.getRow(1).getCell(6).getCellStyle().getDataFormatString()); + assertThat(h.getCell(0).getCellStyle().getFontIndex()).isNotZero(); + assertThat(sh.getPaneInformation().getHorizontalSplitPosition()).isEqualTo((short) 1); + assertThat(wb.getNumCellStyles()).isLessThan(10); + assertThat(fmt.formatCellValue(sh.getRow(1).getCell(1))).matches("\\d{2}-\\w{3}-2025|\\d{2}-\\w{3}-2026"); + } + } + + @Test + void c_excelRowLimitIsReal() throws Exception { + Out.say(F, ""); + Out.say(F, "the sheet limit"); + try (SXSSFWorkbook wb = new SXSSFWorkbook(100)) { + Sheet s = wb.createSheet("limit"); + s.createRow(XlsxExporter.EXCEL_MAX_ROWS - 1); // last legal row index 1,048,575 + Out.say(F, " createRow(1048575) ok"); + assertThatThrownBy(() -> s.createRow(XlsxExporter.EXCEL_MAX_ROWS)) + .isInstanceOf(IllegalArgumentException.class) + .satisfies(e -> Out.say(F, " createRow(1048576) -> " + e.getClass().getSimpleName() + ": " + e.getMessage())); + } + try (SXSSFWorkbook wb = new SXSSFWorkbook(100)) { + assertThatThrownBy(() -> XlsxExporter.write(wb, SaleSource.rows(1), XlsxExporter.EXCEL_MAX_ROWS + 1, new ByteArrayOutputStream())) + .isInstanceOf(IllegalArgumentException.class); + } + } + + @Test + void d_indianGroupingFormatRenders() throws Exception { + Out.say(F, ""); + Out.say(F, "lakh/crore number format, rendered by POI's DataFormatter (not by Excel)"); + try (XSSFWorkbook wb = new XSSFWorkbook()) { + var style = wb.createCellStyle(); + style.setDataFormat(wb.createDataFormat().getFormat(XlsxExporter.INR_FORMAT)); + Sheet sh = wb.createSheet("fmt"); + DataFormatter fmt = new DataFormatter(); + double[] values = {999.5, 99_999.5, 123_456.78, 1_234_567.89, 12_345_678.9, 123_456_789.1}; + String[] expected = {"999.50", "99,999.50", "1,23,456.78", "12,34,567.89", "1,23,45,678.90", "12,34,56,789.10"}; + for (int i = 0; i < values.length; i++) { + var c = sh.createRow(i).createCell(0); + c.setCellValue(values[i]); + c.setCellStyle(style); + String shown = fmt.formatCellValue(c); + Out.say(F, " %,.2f -> %s".formatted(values[i], shown)); + assertThat(shown).isEqualTo(expected[i]); + } + } + } +}