Add spring-batch-partitioning: manager/worker partitioning, gridSize semantics, rejected-partition recovery, and a real 10M-row scaling sweep

spring-batch-partitioning/scripts/generate-shards.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+"""Generates the sharded order CSVs this module partitions over.
+
+Usage:
+  generate-shards.py <output-dir> <total-rows> <num-shards> [--corrupt-shard N] [--seed S]
+
+Each shard is orders-shard-NN.csv with header "orderId,customerId,amountCents,region".
+Row counts are split as evenly as possible across shards (the last shard absorbs the remainder).
+--corrupt-shard N replaces one row near the middle of shard N (0-indexed) with a non-numeric
+amountCents field, so FlatFileItemReader's FieldSetMapper throws NumberFormatException wrapped in
+FlatFileParseException when that shard is read -- a real, reproducible parse failure, not a
+simulated one.
+"""
+import argparse
+import os
+import random
+import sys
+
+REGIONS = ["NORTH", "SOUTH", "EAST", "WEST", "CENTRAL"]
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("output_dir")
+    ap.add_argument("total_rows", type=int)
+    ap.add_argument("num_shards", type=int)
+    ap.add_argument("--corrupt-shard", type=int, default=-1)
+    ap.add_argument("--seed", type=int, default=42)
+    args = ap.parse_args()
+
+    os.makedirs(args.output_dir, exist_ok=True)
+    rnd = random.Random(args.seed)
+
+    base = args.total_rows // args.num_shards
+    counts = [base] * args.num_shards
+    counts[-1] += args.total_rows - base * args.num_shards
+
+    order_id = 1
+    for shard_idx, count in enumerate(counts):
+        path = os.path.join(args.output_dir, f"orders-shard-{shard_idx:02d}.csv")
+        corrupt_at = -1
+        if shard_idx == args.corrupt_shard:
+            corrupt_at = count // 2
+        with open(path, "w", newline="") as f:
+            f.write("orderId,customerId,amountCents,region\n")
+            for i in range(count):
+                customer_id = rnd.randint(1, 2_000_000)
+                amount_cents = rnd.randint(500, 9_999_999)
+                region = REGIONS[rnd.randint(0, len(REGIONS) - 1)]
+                if i == corrupt_at:
+                    f.write(f"{order_id},{customer_id},NOT_A_NUMBER,{region}\n")
+                else:
+                    f.write(f"{order_id},{customer_id},{amount_cents},{region}\n")
+                order_id += 1
+        print(f"wrote {path}: {count} rows" + (" (1 corrupt row)" if corrupt_at >= 0 else ""))
+
+    print(f"total: {order_id - 1} rows across {args.num_shards} shards in {args.output_dir}")
+
+
+if __name__ == "__main__":
+    main()

spring-batch-partitioning/scripts/run.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Runs the demo once, in the foreground, streaming to both stdout and a log file.
+#
+#   scripts/run.sh <db-name> <shards-dir> [grid-size] [pool-size] [extra java -D or --args...]
+#
+# Never `pkill -f 'spring-boot'` here -- the pattern matches this script's own invocation line
+# under some shells and can kill the wrong process. Kill by main class instead.
+set -eu
+DB_NAME="${1:?usage: run.sh <db-name> <shards-dir> [grid-size] [pool-size] [extra args...]}"
+SHARDS_DIR="${2:?usage: run.sh <db-name> <shards-dir> [grid-size] [pool-size] [extra args...]}"
+GRID_SIZE="${3:-4}"
+POOL_SIZE="${4:-4}"
+shift $(( $# >= 4 ? 4 : $# ))
+
+for p in $(ps -eo pid,cmd | grep '[P]artitioningDemoApplication' | awk '{print $1}'); do
+  kill -9 "$p" 2>/dev/null || true
+done
+
+JAR="$(dirname "$0")/../target/spring-batch-partitioning-1.0.0.jar"
+LOG="/tmp/run-${DB_NAME}.log"
+
+"${JAVA_HOME:?set JAVA_HOME}/bin/java" -jar "$JAR" \
+  --spring.datasource.url="jdbc:h2:file:./data/${DB_NAME};AUTO_SERVER=TRUE" \
+  --partition.shards-dir="$SHARDS_DIR" \
+  --partition.grid-size="$GRID_SIZE" \
+  --partition.pool-core-size="$POOL_SIZE" \
+  --partition.pool-max-size="$POOL_SIZE" \
+  "$@" 2>&1 | tee "$LOG"