#!/usr/bin/env bash # The live exhibit: two replicas, a shared H2 database, and a load generator that # never stops sending traffic while this script performs all four deploys of the # expand-contract sequence as an actual rolling deploy - one replica at a time. # # Regenerates: # docs/output/11-live-deploy-sequence.txt - the deploy log, phase by phase # docs/output/12-load-generator-summary.txt - total/ok/error counts, by phase # docs/output/13-schema-diagnostics-timeline.txt - /diag/schema after each deploy set -euo pipefail DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$DIR/env.sh" OUTPUT_DIR="$MODULE_DIR/docs/output" mkdir -p "$OUTPUT_DIR" DEPLOY_LOG="$OUTPUT_DIR/11-live-deploy-sequence.txt" DIAG_LOG="$OUTPUT_DIR/13-schema-diagnostics-timeline.txt" log() { echo "$(date -u +%H:%M:%S) $*" | tee -a "$DEPLOY_LOG" } diag() { local label="$1" { echo "" echo "-- $label --" curl -s "http://localhost:$PORT_A/diag/schema" echo "" } | tee -a "$DIAG_LOG" } # --- clean slate --- for p in "$PORT_A" "$PORT_B"; do "$DIR/stop-instance.sh" "$p" 2>/dev/null || true done if [ -f "$EC_PID_DIR/loadgen.pid" ]; then kill -9 "$(cat "$EC_PID_DIR/loadgen.pid")" 2>/dev/null || true rm -f "$EC_PID_DIR/loadgen.pid" fi "$DIR/stop-db-server.sh" 2>/dev/null || true rm -rf "$EC_DB_BASE_DIR" mkdir -p "$EC_DB_BASE_DIR" rm -f "$DEPLOY_LOG" "$DIAG_LOG" echo "=====================================================================" > "$DEPLOY_LOG" echo "Zero-downtime expand-contract: live 4-deploy sequence" >> "$DEPLOY_LOG" echo "=====================================================================" >> "$DEPLOY_LOG" log "captured: $(date -u +%FT%TZ)" # --- Deploy -1: the database itself. This process is never restarted for the rest # of this script - it is the one thing every deploy below has to treat as always up. log "Starting the database as its own standalone process (not owned by either replica)" "$DIR/start-db-server.sh" | tee -a "$DEPLOY_LOG" # --- Deploy 0: baseline schema + two Stage 1 replicas --- log "Deploy 0: create schema (V1), start two Stage 1 replicas" "$DIR/migrate.sh" 1 | tee -a "$DEPLOY_LOG" > /dev/null "$DIR/start-instance.sh" "$PORT_A" 1 | tee -a "$DEPLOY_LOG" "$DIR/start-instance.sh" "$PORT_B" 1 | tee -a "$DEPLOY_LOG" diag "after Deploy 0 (Stage 1 / Stage 1)" "$DIR/run-load-generator.sh" 120 log "load generator running against both replicas" sleep 10 log "baseline soak complete (10s, both replicas on Stage 1)" # --- Deploy 1 (EXPAND): schema only, zero app restarts --- set_phase "01-expand-migration" log "Deploy 1 (EXPAND): migrating to V2 live - zero app restarts" "$DIR/migrate.sh" 2 | tee -a "$DEPLOY_LOG" > /dev/null diag "after Deploy 1 (schema expanded, both replicas still Stage 1)" sleep 5 # --- Deploy 2 (MIGRATE WRITES): rolling restart to Stage 2 --- set_phase "02-deploy-stage2-rollout" log "Deploy 2 (MIGRATE WRITES): rolling restart to Stage 2, replica A first" "$DIR/stop-instance.sh" "$PORT_A" | tee -a "$DEPLOY_LOG" "$DIR/start-instance.sh" "$PORT_A" 2 | tee -a "$DEPLOY_LOG" sleep 2 log "Deploy 2: replica B" "$DIR/stop-instance.sh" "$PORT_B" | tee -a "$DEPLOY_LOG" "$DIR/start-instance.sh" "$PORT_B" 2 | tee -a "$DEPLOY_LOG" set_phase "02-stage2-soak" diag "after Deploy 2 (both replicas Stage 2, dual-write live)" sleep 8 # --- Deploy 3 (MIGRATE READS): rolling restart to Stage 3 --- set_phase "03-deploy-stage3-rollout" log "Deploy 3 (MIGRATE READS): rolling restart to Stage 3, replica A first" "$DIR/stop-instance.sh" "$PORT_A" | tee -a "$DEPLOY_LOG" "$DIR/start-instance.sh" "$PORT_A" 3 | tee -a "$DEPLOY_LOG" sleep 2 log "Deploy 3: replica B" "$DIR/stop-instance.sh" "$PORT_B" | tee -a "$DEPLOY_LOG" "$DIR/start-instance.sh" "$PORT_B" 3 | tee -a "$DEPLOY_LOG" set_phase "03-stage3-soak" diag "after Deploy 3 (both replicas Stage 3, reading email_address)" sleep 8 # --- Deploy 4a (CONTRACT, code): rolling restart to Stage 4 --- set_phase "04a-deploy-stage4-rollout" log "Deploy 4a (CONTRACT code): rolling restart to Stage 4, replica A first" "$DIR/stop-instance.sh" "$PORT_A" | tee -a "$DEPLOY_LOG" "$DIR/start-instance.sh" "$PORT_A" 4 | tee -a "$DEPLOY_LOG" sleep 2 log "Deploy 4a: replica B" "$DIR/stop-instance.sh" "$PORT_B" | tee -a "$DEPLOY_LOG" "$DIR/start-instance.sh" "$PORT_B" 4 | tee -a "$DEPLOY_LOG" set_phase "04a-stage4-soak" diag "after Deploy 4a (both replicas Stage 4, email column still present but unused)" sleep 8 # --- Deploy 4b (CONTRACT, schema): drop the old column, live --- set_phase "04b-contract-migration" log "Deploy 4b (CONTRACT schema): migrating to V3 live - drops \"email\", zero app restarts" "$DIR/migrate.sh" latest | tee -a "$DEPLOY_LOG" > /dev/null diag "after Deploy 4b (email column dropped)" set_phase "05-final-soak" sleep 10 log "final soak complete" log "waiting for the load generator to finish its run..." if [ -f "$EC_PID_DIR/loadgen.pid" ]; then LOADGEN_PID="$(cat "$EC_PID_DIR/loadgen.pid")" while kill -0 "$LOADGEN_PID" 2>/dev/null; do sleep 1 done fi log "load generator finished" cp "$EC_LOAD_SUMMARY" "$OUTPUT_DIR/12-load-generator-summary.txt" for p in "$PORT_A" "$PORT_B"; do "$DIR/stop-instance.sh" "$p" || true done "$DIR/stop-db-server.sh" || true log "=== deploy sequence complete ===" echo "" echo "Deploy log: $DEPLOY_LOG" echo "Load summary: $OUTPUT_DIR/12-load-generator-summary.txt" echo "Schema timeline: $DIAG_LOG" cat "$OUTPUT_DIR/12-load-generator-summary.txt"