Add db-migrations-expand-contract: zero-downtime schema migrations proven with a real 4-deploy rolling run

Companion code for Zero-Downtime Database Migrations: Expand-Contract in Practice with Spring Boot: a full expand/migrate-writes/migrate-reads/contract sequence run as an actual rolling deploy across two live replicas, with a load generator sending continuous HTTP traffic through all four deploys (99.98% success, every residual error traced to a root cause rather than left unexplained). Findings include a real NOT NULL constraint trap in the expand migration, a backfill-window bug in the read switch, H2's AUTO_SERVER=TRUE single-point-of-failure behavior under a rolling restart, the drain-before-SIGTERM fix needed to close a health-check gap during graceful shutdown, and H2 silently discarding a concurrently committed INSERT during an ALTER TABLE ADD/DROP COLUMN rebuild - confirmed, by primary source, to be an H2-specific behavior rather than a property of the technique itself. Co-Authored-By: Claude Sonnet 5 <[email protected]> Claude-Session: https://claude.ai/code/session_019Fb7vW8vLyLKngBc4R3huA
2026-09-16 19:22:48 +00:00
parent 22c30d4a5b
commit e478eafda3
60 changed files with 3561 additions and 0 deletions
@@ -0,0 +1,85 @@
+package com.ankurm.expandcontract;
+
+import com.ankurm.expandcontract.customer.Customer;
+import com.ankurm.expandcontract.customer.CustomerService;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.springframework.jdbc.core.simple.JdbcClient;
+
+import java.nio.file.Path;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Deploy 4 (CONTRACT), in both directions. First the case that must work: once every
+ * replica is confirmed on Stage 4 and the drop migration (V3) has run, Stage 4 code
+ * keeps working exactly as before - it never referenced "email" to begin with. Second
+ * the case that must fail loudly: a Stage 1, 2 or 3 instance that is somehow still
+ * running against the post-drop schema (a rollback gone wrong, a forgotten canary)
+ * gets a real SQL error the moment it tries to touch the column that no longer exists.
+ * That failure is not a bug in this article's design - it is *why* Deploy 4 has to wait
+ * for confirmation that the fleet is 100% on Stage 4 first. See
+ * docs/09-the-contract-migration.md and docs/10-what-happens-if-you-drop-too-soon.md.
+ */
+class ContractSafetyTest {
+
+    @Test
+    void stage4KeepsWorkingAfterTheColumnIsDropped(@TempDir Path tmp) throws Exception {
+        Path db = tmp.resolve("contract-ok");
+        TestSupport.migrateTo(db, "2");
+        JdbcClient jdbc = TestSupport.jdbcClient(db);
+        Transcript t = Transcript.start("09-contract-safety",
+                "Deploy 4 (CONTRACT): Stage 4 after the drop, and what breaks if you drop too soon");
+
+        CustomerService stage4 = new CustomerService(jdbc, 4);
+        long id = stage4.create("Annie Easley", "[email protected]");
+
+        // Deploy 4b: drop the old column, live, with Stage 4 already the only code running.
+        TestSupport.migrateTo(db, "latest");
+
+        Customer afterDrop = stage4.findById(id).orElseThrow();
+        t.section("Stage 4 read, after V3 dropped the email column");
+        t.line(afterDrop.toString());
+        assertThat(afterDrop.email()).isEqualTo("[email protected]");
+
+        long postDropId = stage4.create("Mary Allen Wilkes", "[email protected]");
+        Customer postDrop = stage4.findById(postDropId).orElseThrow();
+        t.section("Stage 4 create + read, entirely after the drop");
+        t.line(postDrop.toString());
+        assertThat(postDrop.email()).isEqualTo("[email protected]");
+
+        t.write();
+    }
+
+    @Test
+    void stage1CodeFailsLoudlyIfItIsStillRunningAfterTheColumnIsDropped(@TempDir Path tmp) throws Exception {
+        Path db = tmp.resolve("contract-too-soon");
+        TestSupport.migrateTo(db, "latest");
+        JdbcClient jdbc = TestSupport.jdbcClient(db);
+        Transcript t = Transcript.start("10-drop-too-soon",
+                "What a lingering Stage 1 instance sees if the drop runs before it is retired");
+
+        CustomerService stage1 = new CustomerService(jdbc, 1);
+
+        Throwable thrown = catchThrowable(() -> stage1.create("Too Late", "[email protected]"));
+        Throwable root = thrown;
+        while (root.getCause() != null) {
+            root = root.getCause();
+        }
+        t.line("Stage 1 create() after V3 dropped \"email\": " + thrown.getClass().getName());
+        t.line("message: " + thrown.getMessage());
+        t.line("root cause: " + root.getClass().getName() + ": " + root.getMessage());
+        assertThat(root.getMessage()).contains("EMAIL").containsIgnoringCase("not found");
+
+        t.write();
+    }
+
+    private static Throwable catchThrowable(org.assertj.core.api.ThrowableAssert.ThrowingCallable callable) {
+        try {
+            callable.call();
+            return new AssertionError("expected an exception but none was thrown");
+        } catch (Throwable t) {
+            return t;
+        }
+    }
+}