Add excel-export module: Apache POI 6.0.0 XSSF vs SXSSF under a fixed heap, 1,000,000 rows streamed with StreamingResponseBody, OpenCSV, sheet rolling, temp-file cleanup
Co-Authored-By: Claude Sonnet 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_017ic4b4VQVEJWetTT1dYzyJ
This commit is contained in:
Claude Sonnet 5.5
@@ -0,0 +1,60 @@
|
||||
package com.ankurm.excelexport;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.io.Writer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Iterator;
|
||||
|
||||
import com.opencsv.CSVWriter;
|
||||
import com.opencsv.ICSVWriter;
|
||||
|
||||
/** Streams {@link SaleRow}s as CSV with OpenCSV. Memory use does not depend on the row count. */
|
||||
public final class CsvExporter {
|
||||
|
||||
private static final String[] HEADERS = {"Order id", "Order date", "Region", "Product", "Qty", "Unit price", "Amount", "Status", "Note"};
|
||||
|
||||
private CsvExporter() { }
|
||||
|
||||
/**
|
||||
* @param excelBom write a UTF-8 byte-order mark so Excel opens the file as UTF-8 instead of the Windows code page
|
||||
* @return the number of data rows written
|
||||
*/
|
||||
public static long write(Iterator<SaleRow> rows, OutputStream out, boolean excelBom) throws IOException {
|
||||
if (excelBom) {
|
||||
out.write(new byte[] {(byte) 0xEF, (byte) 0xBB, (byte) 0xBF});
|
||||
}
|
||||
Writer w = new OutputStreamWriter(out, StandardCharsets.UTF_8);
|
||||
ICSVWriter csv = new CSVWriter(w);
|
||||
csv.writeNext(HEADERS, false);
|
||||
long n = 0;
|
||||
while (rows.hasNext()) {
|
||||
SaleRow r = rows.next();
|
||||
csv.writeNext(new String[] {
|
||||
Long.toString(r.id()), r.orderDate().toString(), safe(r.region()), safe(r.product()),
|
||||
Integer.toString(r.quantity()), r.unitPrice().toPlainString(), r.amount().toPlainString(),
|
||||
safe(r.status()), safe(r.note())}, false);
|
||||
n++;
|
||||
}
|
||||
if (csv.checkError()) { // writeNext() swallows IOExceptions and only records them; see CsvTest
|
||||
throw new IOException("CSV write failed", csv.getException());
|
||||
}
|
||||
csv.flush();
|
||||
return n;
|
||||
}
|
||||
|
||||
/**
|
||||
* CSV injection guard: a text cell that starts with = + - @ (or a tab or CR) is evaluated as a formula by
|
||||
* Excel when the CSV is opened. Prefix a single quote so it is shown as text.
|
||||
*/
|
||||
static String safe(String s) {
|
||||
if (s != null && !s.isEmpty()) {
|
||||
char c = s.charAt(0);
|
||||
if (c == '=' || c == '+' || c == '-' || c == '@' || c == '\t' || c == '\r') {
|
||||
return "'" + s;
|
||||
}
|
||||
}
|
||||
return s;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package com.ankurm.excelexport;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
@SpringBootApplication
|
||||
public class ExcelExportApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(ExcelExportApplication.class, args);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package com.ankurm.excelexport;
|
||||
|
||||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.springframework.http.ContentDisposition;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
|
||||
|
||||
@RestController
|
||||
public class ExportController {
|
||||
|
||||
static final MediaType XLSX = MediaType.parseMediaType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
|
||||
static final MediaType CSV = MediaType.parseMediaType("text/csv;charset=UTF-8");
|
||||
|
||||
/**
|
||||
* The naive version: XSSFWorkbook keeps every cell as an object until write(). StreamingResponseBody does
|
||||
* not help here: the response is "streamed" only after the workbook has been built in full.
|
||||
*/
|
||||
@GetMapping("/export/xssf")
|
||||
public ResponseEntity<StreamingResponseBody> xssf(@RequestParam(defaultValue = "100000") int rows,
|
||||
@RequestParam(defaultValue = "" + XlsxExporter.EXCEL_MAX_ROWS) int sheetLimit) {
|
||||
StreamingResponseBody body = out -> {
|
||||
try (XSSFWorkbook wb = new XSSFWorkbook()) {
|
||||
XlsxExporter.write(wb, SaleSource.rows(rows), sheetLimit, out);
|
||||
}
|
||||
};
|
||||
return download("sales.xlsx", XLSX, body);
|
||||
}
|
||||
|
||||
/**
|
||||
* Streaming version: SXSSFWorkbook(100) keeps 100 rows per sheet in memory and writes older rows to a temp
|
||||
* file. dispose() deletes that temp file; it runs in finally so a failed or aborted download cleans up too.
|
||||
*/
|
||||
@GetMapping("/export/sxssf")
|
||||
public ResponseEntity<StreamingResponseBody> sxssf(@RequestParam(defaultValue = "100000") int rows,
|
||||
@RequestParam(defaultValue = "false") boolean compress,
|
||||
@RequestParam(defaultValue = "100") int window,
|
||||
@RequestParam(defaultValue = "" + XlsxExporter.EXCEL_MAX_ROWS) int sheetLimit) {
|
||||
StreamingResponseBody body = out -> {
|
||||
SXSSFWorkbook wb = new SXSSFWorkbook(window);
|
||||
wb.setCompressTempFiles(compress);
|
||||
try {
|
||||
XlsxExporter.write(wb, SaleSource.rows(rows), sheetLimit, out);
|
||||
} finally {
|
||||
wb.dispose();
|
||||
wb.close();
|
||||
}
|
||||
};
|
||||
return download("sales.xlsx", XLSX, body);
|
||||
}
|
||||
|
||||
@GetMapping("/export/csv")
|
||||
public ResponseEntity<StreamingResponseBody> csv(@RequestParam(defaultValue = "100000") int rows,
|
||||
@RequestParam(defaultValue = "true") boolean bom) {
|
||||
StreamingResponseBody body = out -> CsvExporter.write(SaleSource.rows(rows), out, bom);
|
||||
return download("sales.csv", CSV, body);
|
||||
}
|
||||
|
||||
private static ResponseEntity<StreamingResponseBody> download(String filename, MediaType type, StreamingResponseBody body) {
|
||||
return ResponseEntity.ok()
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION, ContentDisposition.attachment().filename(filename).build().toString())
|
||||
.contentType(type)
|
||||
.body(body);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package com.ankurm.excelexport;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* Diagnostic endpoint for the measurements. It samples used heap every 20 ms and reports the peak, plus the
|
||||
* process's peak resident set size from /proc (Linux only). Delete it before shipping anything.
|
||||
*/
|
||||
@RestController
|
||||
public class ProbeController {
|
||||
|
||||
private final AtomicLong peakUsed = new AtomicLong();
|
||||
|
||||
public ProbeController() {
|
||||
Thread t = new Thread(() -> {
|
||||
Runtime rt = Runtime.getRuntime();
|
||||
while (true) {
|
||||
peakUsed.accumulateAndGet(rt.totalMemory() - rt.freeMemory(), Math::max);
|
||||
try {
|
||||
Thread.sleep(20);
|
||||
} catch (InterruptedException e) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}, "heap-peak-sampler");
|
||||
t.setDaemon(true);
|
||||
t.start();
|
||||
}
|
||||
|
||||
@GetMapping("/probe/ping")
|
||||
public String ping() {
|
||||
return "ok";
|
||||
}
|
||||
|
||||
@GetMapping("/probe")
|
||||
public Map<String, Object> probe() throws Exception {
|
||||
Map<String, Object> m = new LinkedHashMap<>();
|
||||
Runtime rt = Runtime.getRuntime();
|
||||
m.put("maxHeapMb", rt.maxMemory() >> 20);
|
||||
m.put("peakUsedHeapMb", peakUsed.get() >> 20);
|
||||
m.put("peakRssMb", procKb("VmHWM") >> 10);
|
||||
return m;
|
||||
}
|
||||
|
||||
private static long procKb(String key) throws Exception {
|
||||
try (Stream<String> lines = Files.lines(Path.of("/proc/self/status"))) {
|
||||
return lines.filter(l -> l.startsWith(key + ":"))
|
||||
.map(l -> Long.parseLong(l.replaceAll("\\D+", "")))
|
||||
.findFirst().orElse(-1L);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package com.ankurm.excelexport;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.time.LocalDate;
|
||||
|
||||
/** One line of the report. Nine columns once the computed amount is added; the note is unique per row on purpose. */
|
||||
public record SaleRow(long id, LocalDate orderDate, String region, String product, int quantity,
|
||||
BigDecimal unitPrice, String status, String note) {
|
||||
|
||||
public BigDecimal amount() {
|
||||
return unitPrice.multiply(BigDecimal.valueOf(quantity));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package com.ankurm.excelexport;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.time.LocalDate;
|
||||
import java.util.Iterator;
|
||||
import java.util.NoSuchElementException;
|
||||
|
||||
/**
|
||||
* A deterministic stand-in for a database cursor: row n is computed from n, so nothing is stored and the
|
||||
* same call always yields the same rows. In a real service this is a JDBC {@code ResultSet} or a JPA
|
||||
* {@code Stream<Entity>} with a fetch size; the exporters only ever see an {@link Iterator}.
|
||||
*/
|
||||
public final class SaleSource {
|
||||
|
||||
private static final String[] REGIONS = {"Mumbai", "Pune", "Delhi", "Bengaluru", "Chennai", "Hyderabad", "Kolkata", "Ahmedabad"};
|
||||
private static final String[] STATUSES = {"PAID", "PENDING", "SHIPPED", "RETURNED"};
|
||||
private static final LocalDate BASE = LocalDate.of(2025, 4, 1);
|
||||
|
||||
private SaleSource() { }
|
||||
|
||||
public static Iterator<SaleRow> rows(int count) {
|
||||
return new Iterator<>() {
|
||||
private long next = 1;
|
||||
|
||||
@Override public boolean hasNext() {
|
||||
return next <= count;
|
||||
}
|
||||
|
||||
@Override public SaleRow next() {
|
||||
if (!hasNext()) {
|
||||
throw new NoSuchElementException();
|
||||
}
|
||||
return row(next++);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public static SaleRow row(long id) {
|
||||
long h = id * 0x9E3779B97F4A7C15L; // cheap deterministic scrambler
|
||||
int a = (int) ((h >>> 33) & 0x7fffffff);
|
||||
int b = (int) ((h >>> 11) & 0x7fffffff);
|
||||
String product = "Product-" + (a % 500);
|
||||
int qty = 1 + b % 20;
|
||||
BigDecimal price = BigDecimal.valueOf(99 + (a % 99_900), 2);
|
||||
return new SaleRow(id,
|
||||
BASE.plusDays(b % 365),
|
||||
REGIONS[a % REGIONS.length],
|
||||
product,
|
||||
qty,
|
||||
price,
|
||||
STATUSES[b % STATUSES.length],
|
||||
"INV/2025-26/" + String.format("%07d", id) + " dispatched via hub " + (b % 97));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package com.ankurm.excelexport;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.util.Iterator;
|
||||
|
||||
import org.apache.poi.ss.usermodel.BorderStyle;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.CellStyle;
|
||||
import org.apache.poi.ss.usermodel.FillPatternType;
|
||||
import org.apache.poi.ss.usermodel.Font;
|
||||
import org.apache.poi.ss.usermodel.HorizontalAlignment;
|
||||
import org.apache.poi.ss.usermodel.IndexedColors;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.ss.util.CellRangeAddress;
|
||||
|
||||
/**
|
||||
* Writes {@link SaleRow}s into any POI {@link Workbook}. The same code drives {@code XSSFWorkbook} (everything
|
||||
* in heap) and {@code SXSSFWorkbook} (a sliding window of rows in heap, the rest on disk), so the only thing
|
||||
* that differs between the two measured endpoints is the constructor call.
|
||||
*
|
||||
* <p>Explained in the post section "Styling and the 64,000 cell-style limit".
|
||||
*/
|
||||
public final class XlsxExporter {
|
||||
|
||||
/** Excel's hard limit per worksheet, header row included (rows 0..1,048,575). */
|
||||
public static final int EXCEL_MAX_ROWS = 1_048_576;
|
||||
|
||||
private static final String[] HEADERS = {"Order id", "Order date", "Region", "Product", "Qty", "Unit price", "Amount", "Status", "Note"};
|
||||
private static final int[] WIDTH_CHARS = {12, 13, 12, 14, 7, 14, 16, 11, 42};
|
||||
|
||||
/** Lakh/crore grouping (12,34,567.89) as an Excel number format: two conditions plus a default section. */
|
||||
static final String INR_FORMAT = "[>=10000000]##\\,##\\,##\\,##0.00;[>=100000]##\\,##\\,##0.00;##,##0.00";
|
||||
|
||||
/** The styles are created once per workbook. A style per cell would hit Excel's limit of about 64,000 styles. */
|
||||
private record Styles(CellStyle header, CellStyle date, CellStyle integer, CellStyle money) { }
|
||||
|
||||
private XlsxExporter() { }
|
||||
|
||||
/**
|
||||
* @param maxRowsPerSheet rows per sheet including the header row; {@link #EXCEL_MAX_ROWS} in production,
|
||||
* smaller in tests so rolling can be seen without a million rows
|
||||
* @return the number of sheets written
|
||||
*/
|
||||
public static int write(Workbook wb, Iterator<SaleRow> rows, int maxRowsPerSheet, OutputStream out) throws IOException {
|
||||
if (maxRowsPerSheet < 2 || maxRowsPerSheet > EXCEL_MAX_ROWS) {
|
||||
throw new IllegalArgumentException("maxRowsPerSheet must be 2.." + EXCEL_MAX_ROWS);
|
||||
}
|
||||
Styles st = styles(wb);
|
||||
int sheets = 0;
|
||||
Sheet sheet = null;
|
||||
int rowIdx = 0;
|
||||
while (rows.hasNext()) {
|
||||
if (sheet == null || rowIdx >= maxRowsPerSheet) {
|
||||
sheet = newSheet(wb, ++sheets, st);
|
||||
rowIdx = 1; // row 0 is the header
|
||||
}
|
||||
SaleRow r = rows.next();
|
||||
Row row = sheet.createRow(rowIdx++);
|
||||
row.createCell(0).setCellValue(r.id());
|
||||
Cell d = row.createCell(1);
|
||||
d.setCellValue(r.orderDate());
|
||||
d.setCellStyle(st.date());
|
||||
row.createCell(2).setCellValue(r.region());
|
||||
row.createCell(3).setCellValue(r.product());
|
||||
Cell q = row.createCell(4);
|
||||
q.setCellValue(r.quantity());
|
||||
q.setCellStyle(st.integer());
|
||||
Cell p = row.createCell(5);
|
||||
p.setCellValue(r.unitPrice().doubleValue());
|
||||
p.setCellStyle(st.money());
|
||||
Cell a = row.createCell(6);
|
||||
a.setCellValue(r.amount().doubleValue());
|
||||
a.setCellStyle(st.money());
|
||||
row.createCell(7).setCellValue(r.status());
|
||||
row.createCell(8).setCellValue(r.note());
|
||||
}
|
||||
if (sheets == 0) {
|
||||
newSheet(wb, 1, st); // an empty report still opens in Excel
|
||||
sheets = 1;
|
||||
}
|
||||
wb.write(out);
|
||||
return sheets;
|
||||
}
|
||||
|
||||
private static Sheet newSheet(Workbook wb, int n, Styles st) {
|
||||
Sheet sheet = wb.createSheet(n == 1 ? "Sales" : "Sales " + n);
|
||||
for (int c = 0; c < WIDTH_CHARS.length; c++) {
|
||||
sheet.setColumnWidth(c, WIDTH_CHARS[c] * 256); // fixed widths: autoSizeColumn on SXSSF would need every row kept
|
||||
}
|
||||
Row h = sheet.createRow(0);
|
||||
for (int c = 0; c < HEADERS.length; c++) {
|
||||
Cell cell = h.createCell(c);
|
||||
cell.setCellValue(HEADERS[c]);
|
||||
cell.setCellStyle(st.header());
|
||||
}
|
||||
sheet.createFreezePane(0, 1);
|
||||
sheet.setAutoFilter(new CellRangeAddress(0, 0, 0, HEADERS.length - 1));
|
||||
return sheet;
|
||||
}
|
||||
|
||||
private static Styles styles(Workbook wb) {
|
||||
var df = wb.createDataFormat();
|
||||
|
||||
Font bold = wb.createFont();
|
||||
bold.setBold(true);
|
||||
bold.setColor(IndexedColors.WHITE.getIndex());
|
||||
CellStyle header = wb.createCellStyle();
|
||||
header.setFont(bold);
|
||||
header.setFillForegroundColor(IndexedColors.DARK_BLUE.getIndex());
|
||||
header.setFillPattern(FillPatternType.SOLID_FOREGROUND);
|
||||
header.setAlignment(HorizontalAlignment.CENTER);
|
||||
header.setBorderBottom(BorderStyle.THIN);
|
||||
|
||||
CellStyle date = wb.createCellStyle();
|
||||
date.setDataFormat(df.getFormat("dd-mmm-yyyy"));
|
||||
date.setAlignment(HorizontalAlignment.CENTER);
|
||||
|
||||
CellStyle integer = wb.createCellStyle();
|
||||
integer.setDataFormat(df.getFormat("#,##0"));
|
||||
|
||||
CellStyle money = wb.createCellStyle();
|
||||
money.setDataFormat(df.getFormat(INR_FORMAT));
|
||||
return new Styles(header, date, integer, money);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
# Streaming downloads can run for minutes on a slow link; the default async timeout would cut them off.
|
||||
spring.mvc.async.request-timeout=10m
|
||||
@@ -0,0 +1,97 @@
|
||||
package com.ankurm.excelexport;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.io.OutputStreamWriter;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Iterator;
|
||||
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import com.opencsv.CSVWriter;
|
||||
|
||||
@org.junit.jupiter.api.TestMethodOrder(org.junit.jupiter.api.MethodOrderer.MethodName.class)
|
||||
class CsvTest {
|
||||
|
||||
static final String F = "03-csv.txt";
|
||||
|
||||
@BeforeAll
|
||||
static void clean() {
|
||||
Out.reset(F);
|
||||
}
|
||||
|
||||
@Test
|
||||
void a_shapeBomAndQuoting() throws Exception {
|
||||
var bos = new ByteArrayOutputStream();
|
||||
long n = CsvExporter.write(SaleSource.rows(3), bos, true);
|
||||
byte[] raw = bos.toByteArray();
|
||||
String text = new String(raw, StandardCharsets.UTF_8);
|
||||
Out.say(F, "first bytes: %02X %02X %02X (UTF-8 BOM so Excel reads UTF-8)".formatted(raw[0], raw[1], raw[2]));
|
||||
Out.say(F, "rows written: " + n);
|
||||
text.substring(1).lines().forEach(l -> Out.say(F, " " + l));
|
||||
assertThat(raw[0]).isEqualTo((byte) 0xEF);
|
||||
assertThat(n).isEqualTo(3);
|
||||
|
||||
var plain = new ByteArrayOutputStream();
|
||||
CsvExporter.write(SaleSource.rows(0), plain, false);
|
||||
assertThat(plain.toByteArray()[0]).isEqualTo((byte) 'O');
|
||||
|
||||
var tricky = new ByteArrayOutputStream();
|
||||
var w = new CSVWriter(new OutputStreamWriter(tricky, StandardCharsets.UTF_8));
|
||||
w.writeNext(new String[] {"Pune, MH", "said \"hi\"", "line1\nline2"}, false);
|
||||
w.flush();
|
||||
Out.say(F, "");
|
||||
Out.say(F, "comma, quote and newline inside fields: " + tricky.toString(StandardCharsets.UTF_8).replace("\n", "\\n"));
|
||||
assertThat(tricky.toString(StandardCharsets.UTF_8)).startsWith("\"Pune, MH\",\"said \"\"hi\"\"\",\"line1\\nline2\"".replace("\\n", "\n"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void b_formulaInjectionIsNeutralised() {
|
||||
Out.say(F, "");
|
||||
for (String s : new String[] {"=HYPERLINK(\"http://x\",\"pay\")", "+91 98200 00000", "-2+3", "@SUM(A1)", "Mumbai"}) {
|
||||
Out.say(F, " %-32s -> %s".formatted(s, CsvExporter.safe(s)));
|
||||
}
|
||||
assertThat(CsvExporter.safe("=1+1")).isEqualTo("'=1+1");
|
||||
assertThat(CsvExporter.safe("Mumbai")).isEqualTo("Mumbai");
|
||||
}
|
||||
|
||||
/** An OutputStream that fails after a few kilobytes, like a client that closed the browser tab. */
|
||||
static OutputStream breaksAfter(int bytes) {
|
||||
return new OutputStream() {
|
||||
int n;
|
||||
@Override public void write(int b) throws IOException {
|
||||
if (++n > bytes) {
|
||||
throw new IOException("Broken pipe");
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Test
|
||||
void c_openCsvSwallowsWriteErrors() throws Exception {
|
||||
Out.say(F, "");
|
||||
// 1. raw CSVWriter: 10,000 rows into a stream that breaks after 1,000 bytes
|
||||
var w = new CSVWriter(new OutputStreamWriter(breaksAfter(1_000), StandardCharsets.UTF_8));
|
||||
Iterator<SaleRow> it = SaleSource.rows(10_000);
|
||||
int written = 0;
|
||||
while (it.hasNext()) {
|
||||
w.writeNext(new String[] {it.next().note()}, false);
|
||||
written++;
|
||||
}
|
||||
Out.say(F, "raw CSVWriter into a broken stream: writeNext called %,d times, none threw".formatted(written));
|
||||
Out.say(F, " checkError() = %s, getException() = %s".formatted(w.checkError(), w.getException()));
|
||||
assertThat(written).isEqualTo(10_000);
|
||||
assertThat(w.checkError()).isTrue();
|
||||
|
||||
// 2. our exporter checks and throws
|
||||
assertThatThrownBy(() -> CsvExporter.write(SaleSource.rows(10_000), breaksAfter(1_000), false))
|
||||
.isInstanceOf(IOException.class)
|
||||
.satisfies(e -> Out.say(F, "CsvExporter into the same stream: " + e.getClass().getSimpleName() + ": " + e.getMessage()
|
||||
+ " (cause: " + e.getCause() + ")"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package com.ankurm.excelexport;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.test.web.server.LocalServerPort;
|
||||
|
||||
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
class EndpointTest {
|
||||
|
||||
static final String F = "04-endpoints.txt";
|
||||
static final HttpClient HTTP = HttpClient.newHttpClient();
|
||||
|
||||
@LocalServerPort int port;
|
||||
|
||||
@BeforeAll
|
||||
static void clean() {
|
||||
Out.reset(F);
|
||||
}
|
||||
|
||||
HttpResponse<byte[]> get(String path) throws Exception {
|
||||
return HTTP.send(HttpRequest.newBuilder(URI.create("http://localhost:" + port + path)).build(),
|
||||
HttpResponse.BodyHandlers.ofByteArray());
|
||||
}
|
||||
|
||||
@Test
|
||||
void headersAndBodies() throws Exception {
|
||||
for (String p : new String[] {"/export/xssf?rows=1500", "/export/sxssf?rows=1500", "/export/csv?rows=1500"}) {
|
||||
HttpResponse<byte[]> r = get(p);
|
||||
Out.say(F, "GET %s -> %d, %s, %s, content-length header: %s, %,d bytes".formatted(p, r.statusCode(),
|
||||
r.headers().firstValue("Content-Type").orElse("-"),
|
||||
r.headers().firstValue("Content-Disposition").orElse("-"),
|
||||
r.headers().firstValue("Content-Length").orElse("none (chunked: " + r.headers().firstValue("Transfer-Encoding").orElse("-") + ")"),
|
||||
r.body().length));
|
||||
assertThat(r.statusCode()).isEqualTo(200);
|
||||
if (p.contains("xssf")) {
|
||||
try (var wb = new XSSFWorkbook(new ByteArrayInputStream(r.body()))) {
|
||||
assertThat(wb.getSheetAt(0).getLastRowNum()).isEqualTo(1500);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package com.ankurm.excelexport;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
|
||||
/** Writes the transcript files under output/ that the post quotes. The tests assert the same numbers they print. */
|
||||
final class Out {
|
||||
|
||||
static final Path DIR = Path.of("output");
|
||||
|
||||
private Out() { }
|
||||
|
||||
static void reset(String file) {
|
||||
try {
|
||||
Files.createDirectories(DIR);
|
||||
Files.deleteIfExists(DIR.resolve(file));
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
}
|
||||
|
||||
static void say(String file, String line) {
|
||||
System.out.println(line);
|
||||
try {
|
||||
Files.writeString(DIR.resolve(file), line + System.lineSeparator(),
|
||||
StandardOpenOption.CREATE, StandardOpenOption.APPEND);
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package com.ankurm.excelexport;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Arrays;
|
||||
|
||||
import org.apache.poi.util.DefaultTempFileCreationStrategy;
|
||||
import org.apache.poi.util.TempFile;
|
||||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
/** SXSSF spills rows to temp files. This shows when they are removed and what compressTempFiles changes. */
|
||||
@org.junit.jupiter.api.TestMethodOrder(org.junit.jupiter.api.MethodOrderer.MethodName.class)
|
||||
class TempFileTest {
|
||||
|
||||
static final String F = "02-temp-files.txt";
|
||||
|
||||
@BeforeAll
|
||||
static void clean() {
|
||||
Out.reset(F);
|
||||
}
|
||||
|
||||
static String[] list(Path dir) {
|
||||
String[] names = dir.toFile().list();
|
||||
Arrays.sort(names);
|
||||
return names;
|
||||
}
|
||||
|
||||
static long sizeOf(Path dir) {
|
||||
long t = 0;
|
||||
for (File f : dir.toFile().listFiles()) {
|
||||
t += f.length();
|
||||
}
|
||||
return t;
|
||||
}
|
||||
|
||||
@Test
|
||||
void a_whenTheTempFilesGoAway(@TempDir Path dir) throws Exception {
|
||||
TempFile.setTempFileCreationStrategy(new DefaultTempFileCreationStrategy(dir.toFile()));
|
||||
|
||||
// 1. exported, written, never disposed
|
||||
SXSSFWorkbook leaked = new SXSSFWorkbook(100);
|
||||
XlsxExporter.write(leaked, SaleSource.rows(20_000), XlsxExporter.EXCEL_MAX_ROWS, new ByteArrayOutputStream());
|
||||
Out.say(F, "after write(), no dispose(): %d temp file(s) left: %s".formatted(list(dir).length, Arrays.toString(list(dir))));
|
||||
assertThat(list(dir)).hasSize(1);
|
||||
|
||||
// 2. close() alone
|
||||
leaked.close();
|
||||
Out.say(F, "after close() only: %d temp file(s) left".formatted(list(dir).length));
|
||||
int afterClose = list(dir).length;
|
||||
|
||||
// 3. dispose()
|
||||
leaked.dispose();
|
||||
Out.say(F, "after dispose(): %d temp file(s) left".formatted(list(dir).length));
|
||||
assertThat(list(dir)).isEmpty();
|
||||
assertThat(afterClose).isIn(0, 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
void b_compressTempFilesShrinksTheSpill(@TempDir Path dir) throws Exception {
|
||||
TempFile.setTempFileCreationStrategy(new DefaultTempFileCreationStrategy(dir.toFile()));
|
||||
long[] sizes = new long[2];
|
||||
for (int i = 0; i < 2; i++) {
|
||||
SXSSFWorkbook wb = new SXSSFWorkbook(100);
|
||||
wb.setCompressTempFiles(i == 1);
|
||||
try {
|
||||
XlsxExporter.write(wb, SaleSource.rows(50_000), XlsxExporter.EXCEL_MAX_ROWS, new ByteArrayOutputStream());
|
||||
sizes[i] = sizeOf(dir); // measured after write(), before dispose()
|
||||
} finally {
|
||||
wb.dispose();
|
||||
wb.close();
|
||||
}
|
||||
}
|
||||
Out.say(F, "");
|
||||
Out.say(F, "temp file size after writing 50,000 rows (9 columns), before dispose():");
|
||||
Out.say(F, " compressTempFiles=false: %,d bytes".formatted(sizes[0]));
|
||||
Out.say(F, " compressTempFiles=true: %,d bytes".formatted(sizes[1]));
|
||||
assertThat(sizes[1]).isLessThan(sizes[0]);
|
||||
assertThat(Files.list(dir).count()).isZero();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package com.ankurm.excelexport;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
|
||||
import org.apache.poi.ss.usermodel.DataFormatter;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
|
||||
import org.apache.poi.xssf.usermodel.XSSFSheet;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
@org.junit.jupiter.api.TestMethodOrder(org.junit.jupiter.api.MethodOrderer.MethodName.class)
|
||||
class XlsxStructureTest {
|
||||
|
||||
static final String F = "01-xlsx-structure.txt";
|
||||
|
||||
@BeforeAll
|
||||
static void clean() {
|
||||
Out.reset(F);
|
||||
}
|
||||
|
||||
static byte[] xssf(int rows, int limit) throws Exception {
|
||||
var bos = new ByteArrayOutputStream();
|
||||
try (XSSFWorkbook wb = new XSSFWorkbook()) {
|
||||
XlsxExporter.write(wb, SaleSource.rows(rows), limit, bos);
|
||||
}
|
||||
return bos.toByteArray();
|
||||
}
|
||||
|
||||
static byte[] sxssf(int rows, int limit) throws Exception {
|
||||
var bos = new ByteArrayOutputStream();
|
||||
SXSSFWorkbook wb = new SXSSFWorkbook(100);
|
||||
try {
|
||||
XlsxExporter.write(wb, SaleSource.rows(rows), limit, bos);
|
||||
} finally {
|
||||
wb.dispose();
|
||||
wb.close();
|
||||
}
|
||||
return bos.toByteArray();
|
||||
}
|
||||
|
||||
@Test
|
||||
void a_sheetsRollAndBothWorkbookTypesAgree() throws Exception {
|
||||
// 2,500 data rows with a pretend limit of 1,000 rows per sheet (header included) = 999 data rows per sheet.
|
||||
byte[] a = xssf(2_500, 1_000);
|
||||
byte[] b = sxssf(2_500, 1_000);
|
||||
DataFormatter fmt = new DataFormatter();
|
||||
try (var x = new XSSFWorkbook(new ByteArrayInputStream(a)); var s = new XSSFWorkbook(new ByteArrayInputStream(b))) {
|
||||
Out.say(F, "sheetLimit=1000 (header included), 2500 data rows");
|
||||
assertThat(x.getNumberOfSheets()).isEqualTo(3).isEqualTo(s.getNumberOfSheets());
|
||||
for (int i = 0; i < 3; i++) {
|
||||
Sheet sx = x.getSheetAt(i);
|
||||
Sheet ss = s.getSheetAt(i);
|
||||
Out.say(F, " sheet %d \"%s\": last row index %d, first data cell %s".formatted(
|
||||
i + 1, sx.getSheetName(), sx.getLastRowNum(), fmt.formatCellValue(sx.getRow(1).getCell(0))));
|
||||
assertThat(ss.getSheetName()).isEqualTo(sx.getSheetName());
|
||||
assertThat(ss.getLastRowNum()).isEqualTo(sx.getLastRowNum());
|
||||
for (int r = 0; r <= sx.getLastRowNum(); r += 97) {
|
||||
for (int c = 0; c < 9; c++) {
|
||||
assertThat(fmt.formatCellValue(ss.getRow(r).getCell(c)))
|
||||
.isEqualTo(fmt.formatCellValue(sx.getRow(r).getCell(c)));
|
||||
}
|
||||
}
|
||||
}
|
||||
assertThat(x.getSheetAt(0).getLastRowNum()).isEqualTo(999);
|
||||
assertThat(x.getSheetAt(2).getLastRowNum()).isEqualTo(2_500 - 2 * 999);
|
||||
Out.say(F, "XSSF and SXSSF files: same sheets, same formatted cell values on every 97th row");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void b_stylingSurvivesAndStyleCountStaysSmall() throws Exception {
|
||||
byte[] b = sxssf(5_000, XlsxExporter.EXCEL_MAX_ROWS);
|
||||
DataFormatter fmt = new DataFormatter();
|
||||
try (var wb = new XSSFWorkbook(new ByteArrayInputStream(b))) {
|
||||
XSSFSheet sh = wb.getSheetAt(0);
|
||||
Row h = sh.getRow(0);
|
||||
Out.say(F, "");
|
||||
Out.say(F, "styling of a 5,000-row SXSSF file, read back with XSSF");
|
||||
Out.say(F, " header cell A1 text=%s bold=%s".formatted(h.getCell(0).getStringCellValue(),
|
||||
wb.getFontAt(h.getCell(0).getCellStyle().getFontIndex()).getBold()));
|
||||
Out.say(F, " freeze pane: row split at %d".formatted(sh.getPaneInformation().getHorizontalSplitPosition()));
|
||||
Out.say(F, " auto filter: " + sh.getCTWorksheet().getAutoFilter().getRef());
|
||||
Out.say(F, " cell styles in the workbook: %d (for 5,000 rows x 9 cells)".formatted(wb.getNumCellStyles()));
|
||||
Out.say(F, " B2 as Excel shows it: " + fmt.formatCellValue(sh.getRow(1).getCell(1)));
|
||||
Out.say(F, " amount number format: " + sh.getRow(1).getCell(6).getCellStyle().getDataFormatString());
|
||||
assertThat(h.getCell(0).getCellStyle().getFontIndex()).isNotZero();
|
||||
assertThat(sh.getPaneInformation().getHorizontalSplitPosition()).isEqualTo((short) 1);
|
||||
assertThat(wb.getNumCellStyles()).isLessThan(10);
|
||||
assertThat(fmt.formatCellValue(sh.getRow(1).getCell(1))).matches("\\d{2}-\\w{3}-2025|\\d{2}-\\w{3}-2026");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void c_excelRowLimitIsReal() throws Exception {
|
||||
Out.say(F, "");
|
||||
Out.say(F, "the sheet limit");
|
||||
try (SXSSFWorkbook wb = new SXSSFWorkbook(100)) {
|
||||
Sheet s = wb.createSheet("limit");
|
||||
s.createRow(XlsxExporter.EXCEL_MAX_ROWS - 1); // last legal row index 1,048,575
|
||||
Out.say(F, " createRow(1048575) ok");
|
||||
assertThatThrownBy(() -> s.createRow(XlsxExporter.EXCEL_MAX_ROWS))
|
||||
.isInstanceOf(IllegalArgumentException.class)
|
||||
.satisfies(e -> Out.say(F, " createRow(1048576) -> " + e.getClass().getSimpleName() + ": " + e.getMessage()));
|
||||
}
|
||||
try (SXSSFWorkbook wb = new SXSSFWorkbook(100)) {
|
||||
assertThatThrownBy(() -> XlsxExporter.write(wb, SaleSource.rows(1), XlsxExporter.EXCEL_MAX_ROWS + 1, new ByteArrayOutputStream()))
|
||||
.isInstanceOf(IllegalArgumentException.class);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void d_indianGroupingFormatRenders() throws Exception {
|
||||
Out.say(F, "");
|
||||
Out.say(F, "lakh/crore number format, rendered by POI's DataFormatter (not by Excel)");
|
||||
try (XSSFWorkbook wb = new XSSFWorkbook()) {
|
||||
var style = wb.createCellStyle();
|
||||
style.setDataFormat(wb.createDataFormat().getFormat(XlsxExporter.INR_FORMAT));
|
||||
Sheet sh = wb.createSheet("fmt");
|
||||
DataFormatter fmt = new DataFormatter();
|
||||
double[] values = {999.5, 99_999.5, 123_456.78, 1_234_567.89, 12_345_678.9, 123_456_789.1};
|
||||
String[] expected = {"999.50", "99,999.50", "1,23,456.78", "12,34,567.89", "1,23,45,678.90", "12,34,56,789.10"};
|
||||
for (int i = 0; i < values.length; i++) {
|
||||
var c = sh.createRow(i).createCell(0);
|
||||
c.setCellValue(values[i]);
|
||||
c.setCellStyle(style);
|
||||
String shown = fmt.formatCellValue(c);
|
||||
Out.say(F, " %,.2f -> %s".formatted(values[i], shown));
|
||||
assertThat(shown).isEqualTo(expected[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user