Add db-migrations-expand-contract: zero-downtime schema migrations proven with a real 4-deploy rolling run

Companion code for Zero-Downtime Database Migrations: Expand-Contract in Practice with Spring Boot: a full expand/migrate-writes/migrate-reads/contract sequence run as an actual rolling deploy across two live replicas, with a load generator sending continuous HTTP traffic through all four deploys (99.98% success, every residual error traced to a root cause rather than left unexplained). Findings include a real NOT NULL constraint trap in the expand migration, a backfill-window bug in the read switch, H2's AUTO_SERVER=TRUE single-point-of-failure behavior under a rolling restart, the drain-before-SIGTERM fix needed to close a health-check gap during graceful shutdown, and H2 silently discarding a concurrently committed INSERT during an ALTER TABLE ADD/DROP COLUMN rebuild - confirmed, by primary source, to be an H2-specific behavior rather than a property of the technique itself. Co-Authored-By: Claude Sonnet 5 <[email protected]> Claude-Session: https://claude.ai/code/session_019Fb7vW8vLyLKngBc4R3huA
2026-09-16 19:22:48 +00:00
parent 22c30d4a5b
commit e478eafda3
60 changed files with 3561 additions and 0 deletions
@@ -0,0 +1,68 @@
+package com.ankurm.expandcontract;
+
+import com.ankurm.expandcontract.customer.Customer;
+import com.ankurm.expandcontract.customer.CustomerService;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.springframework.jdbc.core.simple.JdbcClient;
+
+import java.nio.file.Path;
+import java.util.Optional;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * The gap between "the expand migration ran" and "every Stage 1 instance in the fleet
+ * has been replaced by Stage 2" is not instantaneous - a real rolling deploy takes
+ * minutes, and every row a lingering Stage 1 instance writes during that window has
+ * {@code email_address = NULL}. The migration's one-time backfill (V2) only ever sees
+ * rows that existed *before* Deploy 1 ran; it cannot see rows a Stage 1 instance writes
+ * *after* that, during its own rollout window.
+ * <p>
+ * This test reproduces the resulting bug with a naive Stage 3 read (email_address alone)
+ * and then shows the fix that ships in
+ * {@link com.ankurm.expandcontract.customer.CustomerService#findById}: a plain
+ * {@code COALESCE(email_address, email)}. See docs/07-the-backfill-window-bug.md.
+ */
+class BackfillWindowBugTest {
+
+    @Test
+    void naiveStage3ReadReturnsNullForARowStage1WroteDuringTheRollout(@TempDir Path tmp) throws Exception {
+        Path db = tmp.resolve("backfill-window");
+        TestSupport.migrateTo(db, "2");
+        Transcript t = Transcript.start("07-backfill-window-bug",
+                "The backfill window: a Stage 1 write after Deploy 1, read by a naive Stage 3");
+
+        // A Stage 1 instance is still serving traffic during the Deploy 2 rollout and
+        // writes a brand new row exactly as it always has - it has never heard of
+        // email_address. This is not a hypothetical; it is guaranteed to happen for
+        // however long the rollout takes.
+        JdbcClient jdbc = TestSupport.jdbcClient(db);
+        jdbc.sql("INSERT INTO customers(name, email) VALUES (?, ?)")
+                .param("Katherine Johnson").param("[email protected]")
+                .update();
+        long insertedId = jdbc.sql("SELECT id FROM customers WHERE name = ?")
+                .param("Katherine Johnson").query(Long.class).single();
+
+        t.section("the row a lingering Stage 1 instance just wrote");
+        String naiveRead = jdbc.sql("SELECT name, email, email_address FROM customers WHERE id = ?")
+                .param(insertedId).query().listOfRows().toString();
+        t.line(naiveRead);
+        assertThat(naiveRead).containsIgnoringCase("email_address=null");
+
+        t.section("a NAIVE Stage 3 read (email_address alone) - the bug");
+        String naiveEmail = jdbc.sql("SELECT email_address FROM customers WHERE id = ?")
+                .param(insertedId).query(String.class).optional().orElse(null);
+        t.line("naive Stage 3 email column value: " + naiveEmail);
+        assertThat(naiveEmail).isNull();
+
+        t.section("the SHIPPED Stage 3 read (CustomerService, COALESCE) - the fix");
+        CustomerService stage3 = new CustomerService(jdbc, 3);
+        Optional<Customer> fixed = stage3.findById(insertedId);
+        t.line("CustomerService (stage 3) result: " + fixed);
+        assertThat(fixed).isPresent();
+        assertThat(fixed.get().email()).isEqualTo("[email protected]");
+
+        t.write();
+    }
+}