asmhatre/spring-boot-demo
1
0
Fork 0
Code Activity
Files
main
spring-boot-demo/db-migrations-expand-contract/scripts/run-all.sh
T
asmhatreandClaude Sonnet 5 e478eafda3 Add db-migrations-expand-contract: zero-downtime schema migrations proven with a real 4-deploy rolling run 
Companion code for Zero-Downtime Database Migrations: Expand-Contract in Practice
with Spring Boot: a full expand/migrate-writes/migrate-reads/contract sequence run
as an actual rolling deploy across two live replicas, with a load generator sending
continuous HTTP traffic through all four deploys (99.98% success, every residual
error traced to a root cause rather than left unexplained). Findings include a real
NOT NULL constraint trap in the expand migration, a backfill-window bug in the read
switch, H2's AUTO_SERVER=TRUE single-point-of-failure behavior under a rolling
restart, the drain-before-SIGTERM fix needed to close a health-check gap during
graceful shutdown, and H2 silently discarding a concurrently committed INSERT during
an ALTER TABLE ADD/DROP COLUMN rebuild - confirmed, by primary source, to be an
H2-specific behavior rather than a property of the technique itself.

Co-Authored-By: Claude Sonnet 5 <[email protected]>
Claude-Session: https://claude.ai/code/session_019Fb7vW8vLyLKngBc4R3huA
2026-09-16 19:22:48 +00:00

146 lines
5.4 KiB
Bash
Executable File
Raw Permalink Blame History

#!/usr/bin/env bash
# The live exhibit: two replicas, a shared H2 database, and a load generator that
# never stops sending traffic while this script performs all four deploys of the
# expand-contract sequence as an actual rolling deploy - one replica at a time.
#
# Regenerates:
# docs/output/11-live-deploy-sequence.txt - the deploy log, phase by phase
# docs/output/12-load-generator-summary.txt - total/ok/error counts, by phase
# docs/output/13-schema-diagnostics-timeline.txt - /diag/schema after each deploy
set -euo pipefail
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$DIR/env.sh"
OUTPUT_DIR="$MODULE_DIR/docs/output"
mkdir -p "$OUTPUT_DIR"
DEPLOY_LOG="$OUTPUT_DIR/11-live-deploy-sequence.txt"
DIAG_LOG="$OUTPUT_DIR/13-schema-diagnostics-timeline.txt"
log() {
echo "$(date -u +%H:%M:%S) $*" | tee -a "$DEPLOY_LOG"
}
diag() {
local label="$1"
{
echo ""
echo "-- $label --"
curl -s "http://localhost:$PORT_A/diag/schema"
echo ""
} | tee -a "$DIAG_LOG"
}
# --- clean slate ---
for p in "$PORT_A" "$PORT_B"; do
"$DIR/stop-instance.sh" "$p" 2>/dev/null || true
done
if [ -f "$EC_PID_DIR/loadgen.pid" ]; then
kill -9 "$(cat "$EC_PID_DIR/loadgen.pid")" 2>/dev/null || true
rm -f "$EC_PID_DIR/loadgen.pid"
fi
"$DIR/stop-db-server.sh" 2>/dev/null || true
rm -rf "$EC_DB_BASE_DIR"
mkdir -p "$EC_DB_BASE_DIR"
rm -f "$DEPLOY_LOG" "$DIAG_LOG"
echo "=====================================================================" > "$DEPLOY_LOG"
echo "Zero-downtime expand-contract: live 4-deploy sequence" >> "$DEPLOY_LOG"
echo "=====================================================================" >> "$DEPLOY_LOG"
log "captured: $(date -u +%FT%TZ)"
# --- Deploy -1: the database itself. This process is never restarted for the rest
# of this script - it is the one thing every deploy below has to treat as always up.
log "Starting the database as its own standalone process (not owned by either replica)"
"$DIR/start-db-server.sh" | tee -a "$DEPLOY_LOG"
# --- Deploy 0: baseline schema + two Stage 1 replicas ---
log "Deploy 0: create schema (V1), start two Stage 1 replicas"
"$DIR/migrate.sh" 1 | tee -a "$DEPLOY_LOG" > /dev/null
"$DIR/start-instance.sh" "$PORT_A" 1 | tee -a "$DEPLOY_LOG"
"$DIR/start-instance.sh" "$PORT_B" 1 | tee -a "$DEPLOY_LOG"
diag "after Deploy 0 (Stage 1 / Stage 1)"
"$DIR/run-load-generator.sh" 120
log "load generator running against both replicas"
sleep 10
log "baseline soak complete (10s, both replicas on Stage 1)"
# --- Deploy 1 (EXPAND): schema only, zero app restarts ---
set_phase "01-expand-migration"
log "Deploy 1 (EXPAND): migrating to V2 live - zero app restarts"
"$DIR/migrate.sh" 2 | tee -a "$DEPLOY_LOG" > /dev/null
diag "after Deploy 1 (schema expanded, both replicas still Stage 1)"
sleep 5
# --- Deploy 2 (MIGRATE WRITES): rolling restart to Stage 2 ---
set_phase "02-deploy-stage2-rollout"
log "Deploy 2 (MIGRATE WRITES): rolling restart to Stage 2, replica A first"
"$DIR/stop-instance.sh" "$PORT_A" | tee -a "$DEPLOY_LOG"
"$DIR/start-instance.sh" "$PORT_A" 2 | tee -a "$DEPLOY_LOG"
sleep 2
log "Deploy 2: replica B"
"$DIR/stop-instance.sh" "$PORT_B" | tee -a "$DEPLOY_LOG"
"$DIR/start-instance.sh" "$PORT_B" 2 | tee -a "$DEPLOY_LOG"
set_phase "02-stage2-soak"
diag "after Deploy 2 (both replicas Stage 2, dual-write live)"
sleep 8
# --- Deploy 3 (MIGRATE READS): rolling restart to Stage 3 ---
set_phase "03-deploy-stage3-rollout"
log "Deploy 3 (MIGRATE READS): rolling restart to Stage 3, replica A first"
"$DIR/stop-instance.sh" "$PORT_A" | tee -a "$DEPLOY_LOG"
"$DIR/start-instance.sh" "$PORT_A" 3 | tee -a "$DEPLOY_LOG"
sleep 2
log "Deploy 3: replica B"
"$DIR/stop-instance.sh" "$PORT_B" | tee -a "$DEPLOY_LOG"
"$DIR/start-instance.sh" "$PORT_B" 3 | tee -a "$DEPLOY_LOG"
set_phase "03-stage3-soak"
diag "after Deploy 3 (both replicas Stage 3, reading email_address)"
sleep 8
# --- Deploy 4a (CONTRACT, code): rolling restart to Stage 4 ---
set_phase "04a-deploy-stage4-rollout"
log "Deploy 4a (CONTRACT code): rolling restart to Stage 4, replica A first"
"$DIR/stop-instance.sh" "$PORT_A" | tee -a "$DEPLOY_LOG"
"$DIR/start-instance.sh" "$PORT_A" 4 | tee -a "$DEPLOY_LOG"
sleep 2
log "Deploy 4a: replica B"
"$DIR/stop-instance.sh" "$PORT_B" | tee -a "$DEPLOY_LOG"
"$DIR/start-instance.sh" "$PORT_B" 4 | tee -a "$DEPLOY_LOG"
set_phase "04a-stage4-soak"
diag "after Deploy 4a (both replicas Stage 4, email column still present but unused)"
sleep 8
# --- Deploy 4b (CONTRACT, schema): drop the old column, live ---
set_phase "04b-contract-migration"
log "Deploy 4b (CONTRACT schema): migrating to V3 live - drops \"email\", zero app restarts"
"$DIR/migrate.sh" latest | tee -a "$DEPLOY_LOG" > /dev/null
diag "after Deploy 4b (email column dropped)"
set_phase "05-final-soak"
sleep 10
log "final soak complete"
log "waiting for the load generator to finish its run..."
if [ -f "$EC_PID_DIR/loadgen.pid" ]; then
LOADGEN_PID="$(cat "$EC_PID_DIR/loadgen.pid")"
while kill -0 "$LOADGEN_PID" 2>/dev/null; do
sleep 1
done
fi
log "load generator finished"
cp "$EC_LOAD_SUMMARY" "$OUTPUT_DIR/12-load-generator-summary.txt"
for p in "$PORT_A" "$PORT_B"; do
"$DIR/stop-instance.sh" "$p" || true
done
"$DIR/stop-db-server.sh" || true
log "=== deploy sequence complete ==="
echo ""
echo "Deploy log: $DEPLOY_LOG"
echo "Load summary: $OUTPUT_DIR/12-load-generator-summary.txt"
echo "Schema timeline: $DIAG_LOG"
cat "$OUTPUT_DIR/12-load-generator-summary.txt"
View Git Blame Copy Permalink