Add excel-export module: Apache POI 6.0.0 XSSF vs SXSSF under a fixed heap, 1,000,000 rows streamed with StreamingResponseBody, OpenCSV, sheet rolling, temp-file cleanup
Co-Authored-By: Claude Sonnet 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_017ic4b4VQVEJWetTT1dYzyJ
This commit is contained in:
Claude Sonnet 5.5
Executable
+26
@@ -0,0 +1,26 @@
|
||||
#!/usr/bin/env bash
|
||||
# Opens the same 40-row report from the XSSF and the SXSSF endpoint in headless LibreOffice, and reports what it does.
|
||||
# Needs: soffice, python3, curl, a built jar. Microsoft Excel itself was NOT available for this check.
|
||||
cd "$(dirname "$0")/.."
|
||||
d=$(mktemp -d); P="file://$d/profile"
|
||||
java -jar target/excel-export-1.0.0.jar --server.port=38091 >/dev/null 2>&1 & p=$!
|
||||
for _ in $(seq 1 120); do [ "$(curl -s localhost:38091/probe/ping)" = ok ] && break; sleep 1; done
|
||||
curl -s -o "$d/xssf.xlsx" "localhost:38091/export/xssf?rows=40"
|
||||
curl -s -o "$d/sxssf.xlsx" "localhost:38091/export/sxssf?rows=40"
|
||||
kill $p; wait $p 2>/dev/null || true
|
||||
python3 - "$d" <<'PY'
|
||||
import sys, zipfile
|
||||
d = sys.argv[1]
|
||||
with zipfile.ZipFile(d + "/sxssf.xlsx") as z, zipfile.ZipFile(d + "/rezipped.xlsx", "w", zipfile.ZIP_DEFLATED) as out:
|
||||
for n in z.namelist():
|
||||
out.writestr(n, z.read(n))
|
||||
for f in ("xssf", "sxssf"):
|
||||
with zipfile.ZipFile(f"{d}/{f}.xlsx") as z:
|
||||
print(f"{f}.xlsx ZIP entry timestamps: {sorted({i.date_time for i in z.infolist()})}")
|
||||
PY
|
||||
echo "$(soffice --version 2>/dev/null | head -1)"
|
||||
for f in xssf sxssf rezipped; do
|
||||
(cd "$d" && timeout 120 soffice --headless "-env:UserInstallation=$P" --convert-to csv "$f.xlsx" >/dev/null 2>&1)
|
||||
if [ -f "$d/$f.csv" ]; then echo "$f.xlsx: LibreOffice loads it; row 2 -> $(sed -n 2p "$d/$f.csv")"; else echo "$f.xlsx: LibreOffice: source file could not be loaded"; fi
|
||||
done
|
||||
rm -rf "$d"
|
||||
Executable
+51
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env bash
|
||||
# Library: run ONE export in a fresh JVM with a fixed heap and print one result line.
|
||||
# measure LABEL URL_PATH XMX [SAVE_TO_FILE]
|
||||
# Every call starts its own JVM, so the numbers of one approach never contaminate another.
|
||||
# Needs: curl, awk, a built jar (mvn package), Linux (/proc).
|
||||
set -u
|
||||
JAR="${JAR:-target/excel-export-1.0.0.jar}"
|
||||
PORT="${PORT:-$((30000 + RANDOM % 20000))}"
|
||||
WORK="${WORK:-/tmp/excel-export-measure}"
|
||||
|
||||
measure() {
|
||||
local label="$1" path="$2" xmx="$3" save="${4:-/dev/null}"
|
||||
rm -rf "$WORK"; mkdir -p "$WORK/tmp"
|
||||
java -Xms"$xmx" -Xmx"$xmx" -XX:+UseG1GC -Xlog:gc:file="$WORK/gc.log" -XX:OnOutOfMemoryError="touch $WORK/oom" \
|
||||
-Djava.io.tmpdir="$WORK/tmp" -jar "$JAR" --server.port="$PORT" > "$WORK/app.log" 2>&1 &
|
||||
local pid=$!
|
||||
for _ in $(seq 1 400); do [ "$(curl -s "http://localhost:$PORT/probe/ping")" = ok ] && break; sleep 0.5; done
|
||||
|
||||
# sample the size of the temp directory while the download runs
|
||||
( peak=0; while kill -0 "$pid" 2>/dev/null && [ ! -f "$WORK/done" ]; do
|
||||
s=$(du -sk "$WORK/tmp" 2>/dev/null | cut -f1); [ "${s:-0}" -gt "$peak" ] && peak=$s; echo "$peak" > "$WORK/tmp.peak"; sleep 0.2
|
||||
done ) &
|
||||
local sampler=$!
|
||||
|
||||
local res
|
||||
local body="$WORK/body.txt"; [ "$save" = /dev/null ] && save="$body"
|
||||
res=$(curl -s --max-time "${MAX_TIME:-300}" -o "$save" -w "%{http_code} %{size_download} %{time_total}" "http://localhost:$PORT$path"; echo " curl_exit=$?")
|
||||
touch "$WORK/done"; wait "$sampler" 2>/dev/null || true
|
||||
|
||||
local http bytes errbody="" secs cexit probe peakUsed rss
|
||||
read -r http bytes secs _ <<< "$res"
|
||||
cexit=$(echo "$res" | sed 's/.*curl_exit=//')
|
||||
probe=$(curl -s --max-time 5 "http://localhost:$PORT/probe" || true)
|
||||
peakUsed=$(echo "$probe" | sed -n 's/.*"peakUsedHeapMb":\([0-9]*\).*/\1/p')
|
||||
rss=$(awk '/VmHWM/ {printf "%d", $2/1024}' "/proc/$pid/status" 2>/dev/null)
|
||||
local tmpafter; tmpafter=$(find "$WORK/tmp" -type f 2>/dev/null | wc -l)
|
||||
local oom=no; [ -f "$WORK/oom" ] && oom=YES
|
||||
|
||||
[ "$http" != 200 ] && [ -f "$body" ] && errbody=" | body: $(head -c 120 "$body" | tr -d '\n')"
|
||||
kill "$pid" 2>/dev/null || true; wait "$pid" 2>/dev/null || true
|
||||
|
||||
# largest heap occupancy right after a collection, number of pauses, longest pause
|
||||
local gc
|
||||
gc=$(grep -oE '[0-9]+M->[0-9]+M\([0-9]+M\) [0-9.]+ms' "$WORK/gc.log" | tr '>()M-' ' ' | sed 's/ms//' \
|
||||
| awk '{ if ($2>a) a=$2; n++; if ($4>p) p=$4 } END { printf "%d %d %.0f", a, n, p }')
|
||||
local maxafter pauses maxpause; read -r maxafter pauses maxpause <<< "$gc"
|
||||
local tmppeak; tmppeak=$(( $(cat "$WORK/tmp.peak" 2>/dev/null || echo 0) / 1024 ))
|
||||
|
||||
printf '%-7s %-34s Xmx=%-5s http=%-3s curl=%-2s bytes=%-10s %6.1fs | peak used heap %5s MB | max after GC %5s MB | %4s GC pauses, longest %4s ms | peak RSS %5s MB | temp dir peak %4s MB, files left %s | OOM=%s%s\n' \
|
||||
"$label" "$path" "$xmx" "$http" "$cexit" "$bytes" "$secs" "${peakUsed:--}" "$maxafter" "$pauses" "$maxpause" "${rss:--}" "$tmppeak" "$tmpafter" "$oom" "$errbody"
|
||||
}
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
# Regenerates every file in output/. Takes 15-25 minutes on a shared 2-vCPU box, mostly the XSSF runs.
|
||||
# Needs JDK 25, Maven, curl, awk, unzip. LibreOffice is optional (step 10 is skipped without it).
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
source scripts/measure.sh
|
||||
mkdir -p output
|
||||
HOST="$(java -version 2>&1 | grep -v JAVA_TOOL | head -1) | $(nproc) vCPU | $(awk '/MemTotal/ {printf "%d MB RAM", $2/1024}' /proc/meminfo)"
|
||||
|
||||
if [ -z "${SKIP_BUILD:-}" ]; then
|
||||
mvn -q -B test # writes output/01..04 (structure, temp files, csv, endpoints)
|
||||
mvn -q -B package -DskipTests
|
||||
fi
|
||||
|
||||
hdr() { { echo "# $1"; echo "# $HOST"; echo "# indicative numbers from a shared sandbox, one run per line; see README"; } > "$2"; }
|
||||
|
||||
hdr "XSSF under a fixed 512 MB heap: rows until it dies" output/05-xssf-sweep.txt
|
||||
for n in ${XSSF_ROWS:-10000 25000 50000 100000}; do
|
||||
measure XSSF "/export/xssf?rows=$n" 512m >> output/05-xssf-sweep.txt
|
||||
done
|
||||
|
||||
hdr "SXSSF under the same 512 MB heap, same rows, plus 1,000,000" output/06-sxssf-sweep.txt
|
||||
for n in ${XSSF_ROWS:-10000 25000 50000 100000} 1000000; do
|
||||
measure SXSSF "/export/sxssf?rows=$n" 512m >> output/06-sxssf-sweep.txt
|
||||
done
|
||||
|
||||
hdr "1,000,000 rows: SXSSF and CSV under a 64 MB heap, XSSF given a large heap" output/07-million-rows.txt
|
||||
measure SXSSF "/export/sxssf?rows=1000000" 64m >> output/07-million-rows.txt
|
||||
measure CSV "/export/csv?rows=1000000" 64m >> output/07-million-rows.txt
|
||||
measure XSSF "/export/xssf?rows=1000000" "${XSSF_BIG_HEAP:-3g}" >> output/07-million-rows.txt
|
||||
|
||||
hdr "Sheet rolling: 1,100,000 rows, SXSSF, 64 MB heap, default sheet limit 1,048,576" output/08-sheet-rolling.txt
|
||||
measure SXSSF "/export/sxssf?rows=1100000" 64m /tmp/excel-export-big.xlsx >> output/08-sheet-rolling.txt
|
||||
{
|
||||
echo "file size: $(du -m /tmp/excel-export-big.xlsx | cut -f1) MB"
|
||||
unzip -p /tmp/excel-export-big.xlsx xl/workbook.xml | grep -oE '<sheet [^>]*name="[^"]*"' | sed 's/ r:id.*//'
|
||||
for s in 1 2; do
|
||||
echo "sheet$s.xml: $(unzip -p /tmp/excel-export-big.xlsx xl/worksheets/sheet$s.xml | grep -o '<row ' | wc -l) rows (header included)"
|
||||
done
|
||||
} >> output/08-sheet-rolling.txt
|
||||
rm -f /tmp/excel-export-big.xlsx
|
||||
|
||||
hdr "compressTempFiles on a 1,000,000-row SXSSF export, 64 MB heap" output/09-compress-temp.txt
|
||||
measure SXSSF "/export/sxssf?rows=1000000&compress=false" 64m >> output/09-compress-temp.txt
|
||||
measure SXSSF "/export/sxssf?rows=1000000&compress=true" 64m >> output/09-compress-temp.txt
|
||||
|
||||
if command -v soffice >/dev/null; then
|
||||
bash scripts/libreoffice-check.sh > output/10-libreoffice.txt 2>&1 || true
|
||||
fi
|
||||
echo "done"
|
||||
Reference in New Issue
Block a user