Add db-migrations-expand-contract: zero-downtime schema migrations proven with a real 4-deploy rolling run
Companion code for Zero-Downtime Database Migrations: Expand-Contract in Practice with Spring Boot: a full expand/migrate-writes/migrate-reads/contract sequence run as an actual rolling deploy across two live replicas, with a load generator sending continuous HTTP traffic through all four deploys (99.98% success, every residual error traced to a root cause rather than left unexplained). Findings include a real NOT NULL constraint trap in the expand migration, a backfill-window bug in the read switch, H2's AUTO_SERVER=TRUE single-point-of-failure behavior under a rolling restart, the drain-before-SIGTERM fix needed to close a health-check gap during graceful shutdown, and H2 silently discarding a concurrently committed INSERT during an ALTER TABLE ADD/DROP COLUMN rebuild - confirmed, by primary source, to be an H2-specific behavior rather than a property of the technique itself. Co-Authored-By: Claude Sonnet 5 <[email protected]> Claude-Session: https://claude.ai/code/session_019Fb7vW8vLyLKngBc4R3huA
This commit is contained in:
Claude Sonnet 5
Executable
+62
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env bash
|
||||
# Shared paths and settings for every script in this directory. Source this, don't run it.
|
||||
export EC_DB_TCP_PORT="${EC_DB_TCP_PORT:-9092}"
|
||||
export EC_DB_NAME="${EC_DB_NAME:-expand-contract}"
|
||||
EC_DB_BASE_DIR="${EC_DB_BASE_DIR:-/tmp/ec-demo/db}"
|
||||
EC_RUN_DIR="${EC_RUN_DIR:-/tmp/ec-demo}"
|
||||
EC_LOG_DIR="$EC_RUN_DIR/logs"
|
||||
EC_PID_DIR="$EC_RUN_DIR/pids"
|
||||
EC_PHASE_FILE="$EC_RUN_DIR/phase.txt"
|
||||
EC_LOAD_SUMMARY="$EC_RUN_DIR/load-summary.txt"
|
||||
PORT_A=8081
|
||||
PORT_B=8082
|
||||
|
||||
MODULE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
JAR="$MODULE_DIR/target/db-migrations-expand-contract-1.0.0.jar"
|
||||
CP_FILE="$MODULE_DIR/cp.txt"
|
||||
|
||||
mkdir -p "$EC_LOG_DIR" "$EC_PID_DIR" "$EC_DB_BASE_DIR"
|
||||
|
||||
wait_db_server() {
|
||||
local tries="${1:-40}"
|
||||
for i in $(seq 1 "$tries"); do
|
||||
if (echo > "/dev/tcp/localhost/$EC_DB_TCP_PORT") >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
echo "H2 TCP server on port $EC_DB_TCP_PORT never came up" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
set_phase() {
|
||||
echo -n "$1" > "$EC_PHASE_FILE"
|
||||
echo "[phase] $1"
|
||||
}
|
||||
|
||||
wait_healthy() {
|
||||
local port="$1"
|
||||
local tries="${2:-40}"
|
||||
for i in $(seq 1 "$tries"); do
|
||||
code=$(curl -s -o /dev/null -w '%{http_code}' "http://localhost:$port/actuator/health" || true)
|
||||
if [ "$code" = "200" ]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 0.5
|
||||
done
|
||||
echo "instance on port $port never became healthy" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_down() {
|
||||
local port="$1"
|
||||
local tries="${2:-20}"
|
||||
for i in $(seq 1 "$tries"); do
|
||||
code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 1 "http://localhost:$port/actuator/health" || true)
|
||||
if [ "$code" != "200" ]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 0.3
|
||||
done
|
||||
return 1
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runs the standalone migration.MigrationCli against the live database - no app
|
||||
# restart, no app deploy. Usage: migrate.sh <target|latest>
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$DIR/env.sh"
|
||||
|
||||
TARGET="${1:-latest}"
|
||||
java -cp "$MODULE_DIR/target/classes:$(cat "$CP_FILE")" \
|
||||
com.ankurm.expandcontract.migration.MigrationCli --target="$TARGET"
|
||||
+145
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env bash
|
||||
# The live exhibit: two replicas, a shared H2 database, and a load generator that
|
||||
# never stops sending traffic while this script performs all four deploys of the
|
||||
# expand-contract sequence as an actual rolling deploy - one replica at a time.
|
||||
#
|
||||
# Regenerates:
|
||||
# docs/output/11-live-deploy-sequence.txt - the deploy log, phase by phase
|
||||
# docs/output/12-load-generator-summary.txt - total/ok/error counts, by phase
|
||||
# docs/output/13-schema-diagnostics-timeline.txt - /diag/schema after each deploy
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$DIR/env.sh"
|
||||
|
||||
OUTPUT_DIR="$MODULE_DIR/docs/output"
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
DEPLOY_LOG="$OUTPUT_DIR/11-live-deploy-sequence.txt"
|
||||
DIAG_LOG="$OUTPUT_DIR/13-schema-diagnostics-timeline.txt"
|
||||
|
||||
log() {
|
||||
echo "$(date -u +%H:%M:%S) $*" | tee -a "$DEPLOY_LOG"
|
||||
}
|
||||
|
||||
diag() {
|
||||
local label="$1"
|
||||
{
|
||||
echo ""
|
||||
echo "-- $label --"
|
||||
curl -s "http://localhost:$PORT_A/diag/schema"
|
||||
echo ""
|
||||
} | tee -a "$DIAG_LOG"
|
||||
}
|
||||
|
||||
# --- clean slate ---
|
||||
for p in "$PORT_A" "$PORT_B"; do
|
||||
"$DIR/stop-instance.sh" "$p" 2>/dev/null || true
|
||||
done
|
||||
if [ -f "$EC_PID_DIR/loadgen.pid" ]; then
|
||||
kill -9 "$(cat "$EC_PID_DIR/loadgen.pid")" 2>/dev/null || true
|
||||
rm -f "$EC_PID_DIR/loadgen.pid"
|
||||
fi
|
||||
"$DIR/stop-db-server.sh" 2>/dev/null || true
|
||||
rm -rf "$EC_DB_BASE_DIR"
|
||||
mkdir -p "$EC_DB_BASE_DIR"
|
||||
rm -f "$DEPLOY_LOG" "$DIAG_LOG"
|
||||
|
||||
echo "=====================================================================" > "$DEPLOY_LOG"
|
||||
echo "Zero-downtime expand-contract: live 4-deploy sequence" >> "$DEPLOY_LOG"
|
||||
echo "=====================================================================" >> "$DEPLOY_LOG"
|
||||
log "captured: $(date -u +%FT%TZ)"
|
||||
|
||||
# --- Deploy -1: the database itself. This process is never restarted for the rest
|
||||
# of this script - it is the one thing every deploy below has to treat as always up.
|
||||
log "Starting the database as its own standalone process (not owned by either replica)"
|
||||
"$DIR/start-db-server.sh" | tee -a "$DEPLOY_LOG"
|
||||
|
||||
# --- Deploy 0: baseline schema + two Stage 1 replicas ---
|
||||
log "Deploy 0: create schema (V1), start two Stage 1 replicas"
|
||||
"$DIR/migrate.sh" 1 | tee -a "$DEPLOY_LOG" > /dev/null
|
||||
"$DIR/start-instance.sh" "$PORT_A" 1 | tee -a "$DEPLOY_LOG"
|
||||
"$DIR/start-instance.sh" "$PORT_B" 1 | tee -a "$DEPLOY_LOG"
|
||||
diag "after Deploy 0 (Stage 1 / Stage 1)"
|
||||
|
||||
"$DIR/run-load-generator.sh" 120
|
||||
log "load generator running against both replicas"
|
||||
sleep 10
|
||||
log "baseline soak complete (10s, both replicas on Stage 1)"
|
||||
|
||||
# --- Deploy 1 (EXPAND): schema only, zero app restarts ---
|
||||
set_phase "01-expand-migration"
|
||||
log "Deploy 1 (EXPAND): migrating to V2 live - zero app restarts"
|
||||
"$DIR/migrate.sh" 2 | tee -a "$DEPLOY_LOG" > /dev/null
|
||||
diag "after Deploy 1 (schema expanded, both replicas still Stage 1)"
|
||||
sleep 5
|
||||
|
||||
# --- Deploy 2 (MIGRATE WRITES): rolling restart to Stage 2 ---
|
||||
set_phase "02-deploy-stage2-rollout"
|
||||
log "Deploy 2 (MIGRATE WRITES): rolling restart to Stage 2, replica A first"
|
||||
"$DIR/stop-instance.sh" "$PORT_A" | tee -a "$DEPLOY_LOG"
|
||||
"$DIR/start-instance.sh" "$PORT_A" 2 | tee -a "$DEPLOY_LOG"
|
||||
sleep 2
|
||||
log "Deploy 2: replica B"
|
||||
"$DIR/stop-instance.sh" "$PORT_B" | tee -a "$DEPLOY_LOG"
|
||||
"$DIR/start-instance.sh" "$PORT_B" 2 | tee -a "$DEPLOY_LOG"
|
||||
set_phase "02-stage2-soak"
|
||||
diag "after Deploy 2 (both replicas Stage 2, dual-write live)"
|
||||
sleep 8
|
||||
|
||||
# --- Deploy 3 (MIGRATE READS): rolling restart to Stage 3 ---
|
||||
set_phase "03-deploy-stage3-rollout"
|
||||
log "Deploy 3 (MIGRATE READS): rolling restart to Stage 3, replica A first"
|
||||
"$DIR/stop-instance.sh" "$PORT_A" | tee -a "$DEPLOY_LOG"
|
||||
"$DIR/start-instance.sh" "$PORT_A" 3 | tee -a "$DEPLOY_LOG"
|
||||
sleep 2
|
||||
log "Deploy 3: replica B"
|
||||
"$DIR/stop-instance.sh" "$PORT_B" | tee -a "$DEPLOY_LOG"
|
||||
"$DIR/start-instance.sh" "$PORT_B" 3 | tee -a "$DEPLOY_LOG"
|
||||
set_phase "03-stage3-soak"
|
||||
diag "after Deploy 3 (both replicas Stage 3, reading email_address)"
|
||||
sleep 8
|
||||
|
||||
# --- Deploy 4a (CONTRACT, code): rolling restart to Stage 4 ---
|
||||
set_phase "04a-deploy-stage4-rollout"
|
||||
log "Deploy 4a (CONTRACT code): rolling restart to Stage 4, replica A first"
|
||||
"$DIR/stop-instance.sh" "$PORT_A" | tee -a "$DEPLOY_LOG"
|
||||
"$DIR/start-instance.sh" "$PORT_A" 4 | tee -a "$DEPLOY_LOG"
|
||||
sleep 2
|
||||
log "Deploy 4a: replica B"
|
||||
"$DIR/stop-instance.sh" "$PORT_B" | tee -a "$DEPLOY_LOG"
|
||||
"$DIR/start-instance.sh" "$PORT_B" 4 | tee -a "$DEPLOY_LOG"
|
||||
set_phase "04a-stage4-soak"
|
||||
diag "after Deploy 4a (both replicas Stage 4, email column still present but unused)"
|
||||
sleep 8
|
||||
|
||||
# --- Deploy 4b (CONTRACT, schema): drop the old column, live ---
|
||||
set_phase "04b-contract-migration"
|
||||
log "Deploy 4b (CONTRACT schema): migrating to V3 live - drops \"email\", zero app restarts"
|
||||
"$DIR/migrate.sh" latest | tee -a "$DEPLOY_LOG" > /dev/null
|
||||
diag "after Deploy 4b (email column dropped)"
|
||||
|
||||
set_phase "05-final-soak"
|
||||
sleep 10
|
||||
log "final soak complete"
|
||||
|
||||
log "waiting for the load generator to finish its run..."
|
||||
if [ -f "$EC_PID_DIR/loadgen.pid" ]; then
|
||||
LOADGEN_PID="$(cat "$EC_PID_DIR/loadgen.pid")"
|
||||
while kill -0 "$LOADGEN_PID" 2>/dev/null; do
|
||||
sleep 1
|
||||
done
|
||||
fi
|
||||
log "load generator finished"
|
||||
|
||||
cp "$EC_LOAD_SUMMARY" "$OUTPUT_DIR/12-load-generator-summary.txt"
|
||||
|
||||
for p in "$PORT_A" "$PORT_B"; do
|
||||
"$DIR/stop-instance.sh" "$p" || true
|
||||
done
|
||||
"$DIR/stop-db-server.sh" || true
|
||||
|
||||
log "=== deploy sequence complete ==="
|
||||
echo ""
|
||||
echo "Deploy log: $DEPLOY_LOG"
|
||||
echo "Load summary: $OUTPUT_DIR/12-load-generator-summary.txt"
|
||||
echo "Schema timeline: $DIAG_LOG"
|
||||
cat "$OUTPUT_DIR/12-load-generator-summary.txt"
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env bash
|
||||
# Starts the load generator against both replica ports for a fixed duration and
|
||||
# returns immediately - it runs in the background for the rest of run-all.sh.
|
||||
# Usage: run-load-generator.sh <durationSeconds>
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$DIR/env.sh"
|
||||
|
||||
DURATION="${1:-170}"
|
||||
rm -f "$EC_PHASE_FILE" "$EC_LOAD_SUMMARY"
|
||||
set_phase "00-baseline-soak"
|
||||
|
||||
nohup java -cp "$MODULE_DIR/target/classes:$(cat "$CP_FILE")" \
|
||||
com.ankurm.expandcontract.loadgen.LoadGenerator \
|
||||
--ports="$PORT_A,$PORT_B" \
|
||||
--durationSeconds="$DURATION" \
|
||||
--phaseFile="$EC_PHASE_FILE" \
|
||||
--outFile="$EC_LOAD_SUMMARY" \
|
||||
--threads=8 \
|
||||
> "$EC_LOG_DIR/load-generator.log" 2>&1 < /dev/null &
|
||||
echo $! > "$EC_PID_DIR/loadgen.pid"
|
||||
echo "load generator started (pid $(cat "$EC_PID_DIR/loadgen.pid")), running for ${DURATION}s"
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
# Starts H2 as its own standalone TCP server process - not owned by, or co-located
|
||||
# inside, either app replica. This is what a real production database is: a process
|
||||
# that outlives every app deploy. See docs/12-the-auto-server-trap.md for what went
|
||||
# wrong the first time this demo shared a database file directly between the two
|
||||
# replicas instead.
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$DIR/env.sh"
|
||||
|
||||
nohup java -cp "$(cat "$CP_FILE")" org.h2.tools.Server \
|
||||
-tcp -tcpPort "$EC_DB_TCP_PORT" -baseDir "$EC_DB_BASE_DIR" -ifNotExists \
|
||||
> "$EC_LOG_DIR/db-server.log" 2>&1 < /dev/null &
|
||||
echo $! > "$EC_PID_DIR/db-server.pid"
|
||||
wait_db_server
|
||||
echo "H2 TCP server up on port $EC_DB_TCP_PORT (pid $(cat "$EC_PID_DIR/db-server.pid")), baseDir $EC_DB_BASE_DIR"
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env bash
|
||||
# Starts one replica on a given port and stage, and waits for it to report healthy.
|
||||
# Usage: start-instance.sh <port> <stage>
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$DIR/env.sh"
|
||||
|
||||
PORT="$1"
|
||||
STAGE="$2"
|
||||
|
||||
# -XX:TieredStopAtLevel=1 and a small fixed heap cut this JVM's own startup CPU
|
||||
# burst dramatically (C2 compilation and heap sizing are the two biggest costs of a
|
||||
# cold Spring Boot start). That matters here specifically because two replicas share
|
||||
# a 2-core sandbox: a slow-starting replica can starve its own sibling's health
|
||||
# checks long enough to look like an outage that never actually happened - see
|
||||
# docs/11-the-load-generator.md.
|
||||
nohup java -XX:TieredStopAtLevel=1 -XX:+UseSerialGC -Xms128m -Xmx256m \
|
||||
-jar "$JAR" --server.port="$PORT" --app.stage="$STAGE" \
|
||||
> "$EC_LOG_DIR/instance-$PORT.log" 2>&1 < /dev/null &
|
||||
echo $! > "$EC_PID_DIR/$PORT.pid"
|
||||
echo "started stage $STAGE on port $PORT (pid $(cat "$EC_PID_DIR/$PORT.pid"))"
|
||||
wait_healthy "$PORT"
|
||||
echo "port $PORT healthy"
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$DIR/env.sh"
|
||||
|
||||
PID_FILE="$EC_PID_DIR/db-server.pid"
|
||||
if [ -f "$PID_FILE" ]; then
|
||||
PID="$(cat "$PID_FILE")"
|
||||
if kill -0 "$PID" 2>/dev/null; then
|
||||
kill -9 "$PID"
|
||||
echo "stopped H2 TCP server (pid $PID)"
|
||||
fi
|
||||
rm -f "$PID_FILE"
|
||||
fi
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env bash
|
||||
# Stops one replica by the PID file start-instance.sh wrote for it - never by matching
|
||||
# the process name or command line, which risks matching the wrong process (including
|
||||
# this very script's own shell). Usage: stop-instance.sh <port>
|
||||
#
|
||||
# Sends SIGTERM, not SIGKILL. "server.shutdown: graceful" in application.yml only
|
||||
# does anything on SIGTERM: it stops accepting new connections but lets in-flight
|
||||
# requests finish first. An earlier version used `kill -9` here, which bypasses that
|
||||
# entirely, and requests that were in flight the instant the process vanished showed
|
||||
# up in the load generator's summary as ConnectException/IOException - a real
|
||||
# artifact of skipping the drain step, not a defect in the migration itself. See
|
||||
# docs/13-graceful-shutdown-vs-kill-9.md.
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$DIR/env.sh"
|
||||
|
||||
PORT="$1"
|
||||
PID_FILE="$EC_PID_DIR/$PORT.pid"
|
||||
if [ -f "$PID_FILE" ]; then
|
||||
PID="$(cat "$PID_FILE")"
|
||||
if kill -0 "$PID" 2>/dev/null; then
|
||||
# Deregister BEFORE terminating: tell the load balancer to stop sending new
|
||||
# traffic here, then give its health check a couple of poll cycles to notice,
|
||||
# THEN stop the process. Skipping this drain window and going straight to
|
||||
# SIGTERM is what produced the ConnectException bursts in an earlier run.
|
||||
curl -s -X POST "http://localhost:$PORT/admin/drain" -o /dev/null || true
|
||||
sleep 1.5
|
||||
kill -15 "$PID"
|
||||
for i in $(seq 1 40); do
|
||||
kill -0 "$PID" 2>/dev/null || break
|
||||
sleep 0.25
|
||||
done
|
||||
if kill -0 "$PID" 2>/dev/null; then
|
||||
echo "port $PORT (pid $PID) did not exit gracefully in 10s, sending SIGKILL" >&2
|
||||
kill -9 "$PID"
|
||||
fi
|
||||
echo "stopped port $PORT (pid $PID)"
|
||||
fi
|
||||
rm -f "$PID_FILE"
|
||||
fi
|
||||
wait_down "$PORT" || echo "warning: port $PORT still answering after stop" >&2
|
||||
Reference in New Issue
Block a user