Add db-migrations-expand-contract: zero-downtime schema migrations proven with a real 4-deploy rolling run

Companion code for Zero-Downtime Database Migrations: Expand-Contract in Practice with Spring Boot: a full expand/migrate-writes/migrate-reads/contract sequence run as an actual rolling deploy across two live replicas, with a load generator sending continuous HTTP traffic through all four deploys (99.98% success, every residual error traced to a root cause rather than left unexplained). Findings include a real NOT NULL constraint trap in the expand migration, a backfill-window bug in the read switch, H2's AUTO_SERVER=TRUE single-point-of-failure behavior under a rolling restart, the drain-before-SIGTERM fix needed to close a health-check gap during graceful shutdown, and H2 silently discarding a concurrently committed INSERT during an ALTER TABLE ADD/DROP COLUMN rebuild - confirmed, by primary source, to be an H2-specific behavior rather than a property of the technique itself. Co-Authored-By: Claude Sonnet 5 <[email protected]> Claude-Session: https://claude.ai/code/session_019Fb7vW8vLyLKngBc4R3huA
2026-09-16 19:22:48 +00:00
parent 22c30d4a5b
commit e478eafda3
60 changed files with 3561 additions and 0 deletions
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# Shared paths and settings for every script in this directory. Source this, don't run it.
+export EC_DB_TCP_PORT="${EC_DB_TCP_PORT:-9092}"
+export EC_DB_NAME="${EC_DB_NAME:-expand-contract}"
+EC_DB_BASE_DIR="${EC_DB_BASE_DIR:-/tmp/ec-demo/db}"
+EC_RUN_DIR="${EC_RUN_DIR:-/tmp/ec-demo}"
+EC_LOG_DIR="$EC_RUN_DIR/logs"
+EC_PID_DIR="$EC_RUN_DIR/pids"
+EC_PHASE_FILE="$EC_RUN_DIR/phase.txt"
+EC_LOAD_SUMMARY="$EC_RUN_DIR/load-summary.txt"
+PORT_A=8081
+PORT_B=8082
+
+MODULE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+JAR="$MODULE_DIR/target/db-migrations-expand-contract-1.0.0.jar"
+CP_FILE="$MODULE_DIR/cp.txt"
+
+mkdir -p "$EC_LOG_DIR" "$EC_PID_DIR" "$EC_DB_BASE_DIR"
+
+wait_db_server() {
+    local tries="${1:-40}"
+    for i in $(seq 1 "$tries"); do
+        if (echo > "/dev/tcp/localhost/$EC_DB_TCP_PORT") >/dev/null 2>&1; then
+            return 0
+        fi
+        sleep 0.25
+    done
+    echo "H2 TCP server on port $EC_DB_TCP_PORT never came up" >&2
+    return 1
+}
+
+set_phase() {
+    echo -n "$1" > "$EC_PHASE_FILE"
+    echo "[phase] $1"
+}
+
+wait_healthy() {
+    local port="$1"
+    local tries="${2:-40}"
+    for i in $(seq 1 "$tries"); do
+        code=$(curl -s -o /dev/null -w '%{http_code}' "http://localhost:$port/actuator/health" || true)
+        if [ "$code" = "200" ]; then
+            return 0
+        fi
+        sleep 0.5
+    done
+    echo "instance on port $port never became healthy" >&2
+    return 1
+}
+
+wait_down() {
+    local port="$1"
+    local tries="${2:-20}"
+    for i in $(seq 1 "$tries"); do
+        code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 1 "http://localhost:$port/actuator/health" || true)
+        if [ "$code" != "200" ]; then
+            return 0
+        fi
+        sleep 0.3
+    done
+    return 1
+}