Add spring-batch-partitioning: manager/worker partitioning, gridSize semantics, rejected-partition recovery, and a real 10M-row scaling sweep

2026-09-14 09:36:35 +00:00
parent b81af72bc3
commit 34e9f5b243
40 changed files with 2071 additions and 0 deletions
@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""Generates the sharded order CSVs this module partitions over.
+
+Usage:
+  generate-shards.py <output-dir> <total-rows> <num-shards> [--corrupt-shard N] [--seed S]
+
+Each shard is orders-shard-NN.csv with header "orderId,customerId,amountCents,region".
+Row counts are split as evenly as possible across shards (the last shard absorbs the remainder).
+--corrupt-shard N replaces one row near the middle of shard N (0-indexed) with a non-numeric
+amountCents field, so FlatFileItemReader's FieldSetMapper throws NumberFormatException wrapped in
+FlatFileParseException when that shard is read -- a real, reproducible parse failure, not a
+simulated one.
+"""
+import argparse
+import os
+import random
+import sys
+
+REGIONS = ["NORTH", "SOUTH", "EAST", "WEST", "CENTRAL"]
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("output_dir")
+    ap.add_argument("total_rows", type=int)
+    ap.add_argument("num_shards", type=int)
+    ap.add_argument("--corrupt-shard", type=int, default=-1)
+    ap.add_argument("--seed", type=int, default=42)
+    args = ap.parse_args()
+
+    os.makedirs(args.output_dir, exist_ok=True)
+    rnd = random.Random(args.seed)
+
+    base = args.total_rows // args.num_shards
+    counts = [base] * args.num_shards
+    counts[-1] += args.total_rows - base * args.num_shards
+
+    order_id = 1
+    for shard_idx, count in enumerate(counts):
+        path = os.path.join(args.output_dir, f"orders-shard-{shard_idx:02d}.csv")
+        corrupt_at = -1
+        if shard_idx == args.corrupt_shard:
+            corrupt_at = count // 2
+        with open(path, "w", newline="") as f:
+            f.write("orderId,customerId,amountCents,region\n")
+            for i in range(count):
+                customer_id = rnd.randint(1, 2_000_000)
+                amount_cents = rnd.randint(500, 9_999_999)
+                region = REGIONS[rnd.randint(0, len(REGIONS) - 1)]
+                if i == corrupt_at:
+                    f.write(f"{order_id},{customer_id},NOT_A_NUMBER,{region}\n")
+                else:
+                    f.write(f"{order_id},{customer_id},{amount_cents},{region}\n")
+                order_id += 1
+        print(f"wrote {path}: {count} rows" + (" (1 corrupt row)" if corrupt_at >= 0 else ""))
+
+    print(f"total: {order_id - 1} rows across {args.num_shards} shards in {args.output_dir}")
+
+
+if __name__ == "__main__":
+    main()