Add db-migrations-expand-contract: zero-downtime schema migrations proven with a real 4-deploy rolling run
Companion code for Zero-Downtime Database Migrations: Expand-Contract in Practice with Spring Boot: a full expand/migrate-writes/migrate-reads/contract sequence run as an actual rolling deploy across two live replicas, with a load generator sending continuous HTTP traffic through all four deploys (99.98% success, every residual error traced to a root cause rather than left unexplained). Findings include a real NOT NULL constraint trap in the expand migration, a backfill-window bug in the read switch, H2's AUTO_SERVER=TRUE single-point-of-failure behavior under a rolling restart, the drain-before-SIGTERM fix needed to close a health-check gap during graceful shutdown, and H2 silently discarding a concurrently committed INSERT during an ALTER TABLE ADD/DROP COLUMN rebuild - confirmed, by primary source, to be an H2-specific behavior rather than a property of the technique itself. Co-Authored-By: Claude Sonnet 5 <[email protected]> Claude-Session: https://claude.ai/code/session_019Fb7vW8vLyLKngBc4R3huA
This commit is contained in:
Claude Sonnet 5
+68
@@ -0,0 +1,68 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import com.ankurm.expandcontract.customer.Customer;
|
||||
import com.ankurm.expandcontract.customer.CustomerService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.jdbc.core.simple.JdbcClient;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* The gap between "the expand migration ran" and "every Stage 1 instance in the fleet
|
||||
* has been replaced by Stage 2" is not instantaneous - a real rolling deploy takes
|
||||
* minutes, and every row a lingering Stage 1 instance writes during that window has
|
||||
* {@code email_address = NULL}. The migration's one-time backfill (V2) only ever sees
|
||||
* rows that existed *before* Deploy 1 ran; it cannot see rows a Stage 1 instance writes
|
||||
* *after* that, during its own rollout window.
|
||||
* <p>
|
||||
* This test reproduces the resulting bug with a naive Stage 3 read (email_address alone)
|
||||
* and then shows the fix that ships in
|
||||
* {@link com.ankurm.expandcontract.customer.CustomerService#findById}: a plain
|
||||
* {@code COALESCE(email_address, email)}. See docs/07-the-backfill-window-bug.md.
|
||||
*/
|
||||
class BackfillWindowBugTest {
|
||||
|
||||
@Test
|
||||
void naiveStage3ReadReturnsNullForARowStage1WroteDuringTheRollout(@TempDir Path tmp) throws Exception {
|
||||
Path db = tmp.resolve("backfill-window");
|
||||
TestSupport.migrateTo(db, "2");
|
||||
Transcript t = Transcript.start("07-backfill-window-bug",
|
||||
"The backfill window: a Stage 1 write after Deploy 1, read by a naive Stage 3");
|
||||
|
||||
// A Stage 1 instance is still serving traffic during the Deploy 2 rollout and
|
||||
// writes a brand new row exactly as it always has - it has never heard of
|
||||
// email_address. This is not a hypothetical; it is guaranteed to happen for
|
||||
// however long the rollout takes.
|
||||
JdbcClient jdbc = TestSupport.jdbcClient(db);
|
||||
jdbc.sql("INSERT INTO customers(name, email) VALUES (?, ?)")
|
||||
.param("Katherine Johnson").param("[email protected]")
|
||||
.update();
|
||||
long insertedId = jdbc.sql("SELECT id FROM customers WHERE name = ?")
|
||||
.param("Katherine Johnson").query(Long.class).single();
|
||||
|
||||
t.section("the row a lingering Stage 1 instance just wrote");
|
||||
String naiveRead = jdbc.sql("SELECT name, email, email_address FROM customers WHERE id = ?")
|
||||
.param(insertedId).query().listOfRows().toString();
|
||||
t.line(naiveRead);
|
||||
assertThat(naiveRead).containsIgnoringCase("email_address=null");
|
||||
|
||||
t.section("a NAIVE Stage 3 read (email_address alone) - the bug");
|
||||
String naiveEmail = jdbc.sql("SELECT email_address FROM customers WHERE id = ?")
|
||||
.param(insertedId).query(String.class).optional().orElse(null);
|
||||
t.line("naive Stage 3 email column value: " + naiveEmail);
|
||||
assertThat(naiveEmail).isNull();
|
||||
|
||||
t.section("the SHIPPED Stage 3 read (CustomerService, COALESCE) - the fix");
|
||||
CustomerService stage3 = new CustomerService(jdbc, 3);
|
||||
Optional<Customer> fixed = stage3.findById(insertedId);
|
||||
t.line("CustomerService (stage 3) result: " + fixed);
|
||||
assertThat(fixed).isPresent();
|
||||
assertThat(fixed.get().email()).isEqualTo("[email protected]");
|
||||
|
||||
t.write();
|
||||
}
|
||||
}
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import com.ankurm.expandcontract.customer.Customer;
|
||||
import com.ankurm.expandcontract.customer.CustomerService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.jdbc.core.simple.JdbcClient;
|
||||
|
||||
import java.nio.file.Path;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* Deploy 4 (CONTRACT), in both directions. First the case that must work: once every
|
||||
* replica is confirmed on Stage 4 and the drop migration (V3) has run, Stage 4 code
|
||||
* keeps working exactly as before - it never referenced "email" to begin with. Second
|
||||
* the case that must fail loudly: a Stage 1, 2 or 3 instance that is somehow still
|
||||
* running against the post-drop schema (a rollback gone wrong, a forgotten canary)
|
||||
* gets a real SQL error the moment it tries to touch the column that no longer exists.
|
||||
* That failure is not a bug in this article's design - it is *why* Deploy 4 has to wait
|
||||
* for confirmation that the fleet is 100% on Stage 4 first. See
|
||||
* docs/09-the-contract-migration.md and docs/10-what-happens-if-you-drop-too-soon.md.
|
||||
*/
|
||||
class ContractSafetyTest {
|
||||
|
||||
@Test
|
||||
void stage4KeepsWorkingAfterTheColumnIsDropped(@TempDir Path tmp) throws Exception {
|
||||
Path db = tmp.resolve("contract-ok");
|
||||
TestSupport.migrateTo(db, "2");
|
||||
JdbcClient jdbc = TestSupport.jdbcClient(db);
|
||||
Transcript t = Transcript.start("09-contract-safety",
|
||||
"Deploy 4 (CONTRACT): Stage 4 after the drop, and what breaks if you drop too soon");
|
||||
|
||||
CustomerService stage4 = new CustomerService(jdbc, 4);
|
||||
long id = stage4.create("Annie Easley", "[email protected]");
|
||||
|
||||
// Deploy 4b: drop the old column, live, with Stage 4 already the only code running.
|
||||
TestSupport.migrateTo(db, "latest");
|
||||
|
||||
Customer afterDrop = stage4.findById(id).orElseThrow();
|
||||
t.section("Stage 4 read, after V3 dropped the email column");
|
||||
t.line(afterDrop.toString());
|
||||
assertThat(afterDrop.email()).isEqualTo("[email protected]");
|
||||
|
||||
long postDropId = stage4.create("Mary Allen Wilkes", "[email protected]");
|
||||
Customer postDrop = stage4.findById(postDropId).orElseThrow();
|
||||
t.section("Stage 4 create + read, entirely after the drop");
|
||||
t.line(postDrop.toString());
|
||||
assertThat(postDrop.email()).isEqualTo("[email protected]");
|
||||
|
||||
t.write();
|
||||
}
|
||||
|
||||
@Test
|
||||
void stage1CodeFailsLoudlyIfItIsStillRunningAfterTheColumnIsDropped(@TempDir Path tmp) throws Exception {
|
||||
Path db = tmp.resolve("contract-too-soon");
|
||||
TestSupport.migrateTo(db, "latest");
|
||||
JdbcClient jdbc = TestSupport.jdbcClient(db);
|
||||
Transcript t = Transcript.start("10-drop-too-soon",
|
||||
"What a lingering Stage 1 instance sees if the drop runs before it is retired");
|
||||
|
||||
CustomerService stage1 = new CustomerService(jdbc, 1);
|
||||
|
||||
Throwable thrown = catchThrowable(() -> stage1.create("Too Late", "[email protected]"));
|
||||
Throwable root = thrown;
|
||||
while (root.getCause() != null) {
|
||||
root = root.getCause();
|
||||
}
|
||||
t.line("Stage 1 create() after V3 dropped \"email\": " + thrown.getClass().getName());
|
||||
t.line("message: " + thrown.getMessage());
|
||||
t.line("root cause: " + root.getClass().getName() + ": " + root.getMessage());
|
||||
assertThat(root.getMessage()).contains("EMAIL").containsIgnoringCase("not found");
|
||||
|
||||
t.write();
|
||||
}
|
||||
|
||||
private static Throwable catchThrowable(org.assertj.core.api.ThrowableAssert.ThrowingCallable callable) {
|
||||
try {
|
||||
callable.call();
|
||||
return new AssertionError("expected an exception but none was thrown");
|
||||
} catch (Throwable t) {
|
||||
return t;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.ResultSetMetaData;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/** Renders a JDBC query as a plain-text table for transcripts - no ORM, no formatting library. */
|
||||
public final class DbDump {
|
||||
|
||||
private DbDump() {
|
||||
}
|
||||
|
||||
public static String table(Connection conn, String sql) {
|
||||
try (Statement st = conn.createStatement(); ResultSet rs = st.executeQuery(sql)) {
|
||||
ResultSetMetaData meta = rs.getMetaData();
|
||||
int cols = meta.getColumnCount();
|
||||
List<String> headers = new ArrayList<>();
|
||||
for (int i = 1; i <= cols; i++) {
|
||||
headers.add(meta.getColumnLabel(i));
|
||||
}
|
||||
List<List<String>> rows = new ArrayList<>();
|
||||
while (rs.next()) {
|
||||
List<String> row = new ArrayList<>();
|
||||
for (int i = 1; i <= cols; i++) {
|
||||
Object v = rs.getObject(i);
|
||||
row.add(v == null ? "NULL" : v.toString());
|
||||
}
|
||||
rows.add(row);
|
||||
}
|
||||
int[] widths = new int[cols];
|
||||
for (int i = 0; i < cols; i++) {
|
||||
widths[i] = headers.get(i).length();
|
||||
}
|
||||
for (List<String> row : rows) {
|
||||
for (int i = 0; i < cols; i++) {
|
||||
widths[i] = Math.max(widths[i], row.get(i).length());
|
||||
}
|
||||
}
|
||||
StringBuilder sb = new StringBuilder();
|
||||
appendRow(sb, headers, widths);
|
||||
StringBuilder sep = new StringBuilder();
|
||||
for (int i = 0; i < cols; i++) {
|
||||
sep.append("-".repeat(widths[i])).append(i < cols - 1 ? "-+-" : "");
|
||||
}
|
||||
sb.append(sep).append(System.lineSeparator());
|
||||
for (List<String> row : rows) {
|
||||
appendRow(sb, row, widths);
|
||||
}
|
||||
sb.append("(").append(rows.size()).append(" row").append(rows.size() == 1 ? "" : "s").append(")");
|
||||
return sb.toString();
|
||||
}
|
||||
catch (SQLException ex) {
|
||||
return "query failed: " + ex.getMessage();
|
||||
}
|
||||
}
|
||||
|
||||
private static void appendRow(StringBuilder sb, List<String> values, int[] widths) {
|
||||
for (int i = 0; i < values.size(); i++) {
|
||||
sb.append(pad(values.get(i), widths[i]));
|
||||
sb.append(i < values.size() - 1 ? " | " : "");
|
||||
}
|
||||
sb.append(System.lineSeparator());
|
||||
}
|
||||
|
||||
private static String pad(String s, int width) {
|
||||
return s + " ".repeat(Math.max(0, width - s.length()));
|
||||
}
|
||||
}
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import com.ankurm.expandcontract.customer.CustomerService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.jdbc.core.simple.JdbcClient;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* The finding that {@link com.ankurm.expandcontract.customer.CustomerService#withRetryForConcurrentDdl}
|
||||
* cannot fix, because nothing throws. The retry helper exists for the case where a concurrent
|
||||
* statement collides with an in-progress {@code ALTER TABLE} and H2 answers with
|
||||
* "table not found" - a real error the caller can see and retry. This test demonstrates a second,
|
||||
* stranger failure mode found the same way the first one was: by running this module's own load
|
||||
* generator against a live rolling deploy and noticing a handful of reads and updates come back
|
||||
* 404 for a customer id that a 201 response had already confirmed existed.
|
||||
* <p>
|
||||
* H2 implements both {@code ALTER TABLE ... ADD COLUMN} and {@code ALTER TABLE ... DROP COLUMN}
|
||||
* by rebuilding the table: copying every row into a new table with the new column layout and
|
||||
* swapping it in. If an ordinary {@code INSERT} on another connection commits - with no error,
|
||||
* with a generated key handed back to the caller - while that rebuild is in flight, the inserted
|
||||
* row can be copied into the new table or left behind in the old one depending on exactly when
|
||||
* the rebuild's internal scan ran relative to the commit. When it is left behind, the row is gone
|
||||
* the instant the rebuild finishes, and nothing on the inserting connection was ever told.
|
||||
* <p>
|
||||
* This is not how every database implements {@code ADD COLUMN} and {@code DROP COLUMN}. PostgreSQL's
|
||||
* reference manual is explicit that both are metadata-only operations on tables like this one -
|
||||
* no non-volatile default and no immediate space reclamation requested - so this is a property of
|
||||
* H2's implementation, not of the expand-contract technique itself. See
|
||||
* docs/14-the-ddl-lock-window.md for the full explanation, the standalone reproduction this test
|
||||
* is built from, and what it implies for choosing a target database for a real migration.
|
||||
*/
|
||||
class DdlSilentDataLossTest {
|
||||
|
||||
// Whether the rebuild's internal scan actually passes a given row before or after that
|
||||
// row's INSERT commits is OS scheduling, not application logic - so a single attempt at
|
||||
// this race can genuinely land on either side of it. A single-attempt version of this test
|
||||
// failed about 1 run in 5 while writing it. Rather than assert on one attempt (flaky either
|
||||
// way) or loosen the assertion to "zero or more" (which would silently stop proving anything
|
||||
// the day this stops reproducing), this test repeats the race, on a fresh table each time,
|
||||
// until it reproduces - the same thing a human would do at a terminal to confirm a suspected
|
||||
// race is real. Twenty attempts reproduced it within the first 4 in every run made while
|
||||
// writing this test.
|
||||
private static final int MAX_ATTEMPTS = 20;
|
||||
|
||||
@Test
|
||||
void concurrentAlterTableAddColumnCanSilentlyDropAnAlreadyCommittedInsert(@TempDir Path tmp) throws Exception {
|
||||
Transcript t = Transcript.start("14-ddl-silent-data-loss",
|
||||
"The failure the retry cannot catch: a committed INSERT that ALTER TABLE loses silently");
|
||||
|
||||
for (int attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
|
||||
Path db = tmp.resolve("silent-data-loss-" + attempt);
|
||||
TestSupport.migrateTo(db, "1");
|
||||
JdbcClient jdbc = TestSupport.jdbcClient(db);
|
||||
CustomerService stage1 = new CustomerService(jdbc, 1);
|
||||
|
||||
AtomicBoolean stop = new AtomicBoolean(false);
|
||||
List<Long> confirmedIds = new CopyOnWriteArrayList<>();
|
||||
List<String> insertErrors = new CopyOnWriteArrayList<>();
|
||||
CountDownLatch started = new CountDownLatch(1);
|
||||
|
||||
Thread inserter = new Thread(() -> {
|
||||
started.countDown();
|
||||
int n = 0;
|
||||
while (!stop.get()) {
|
||||
n++;
|
||||
try {
|
||||
long id = stage1.create("Concurrent " + n, "concurrent" + n + "@example.test");
|
||||
confirmedIds.add(id);
|
||||
} catch (Exception ex) {
|
||||
// The already-documented, already-fixed failure mode: a statement that
|
||||
// collides with the DDL and is told so. Counted here only to show it is
|
||||
// rare and separate from the silent loss this test is isolating.
|
||||
insertErrors.add(ex.getClass().getSimpleName());
|
||||
}
|
||||
}
|
||||
});
|
||||
inserter.start();
|
||||
started.await();
|
||||
|
||||
// The DDL runs on its own connection, exactly as Deploy 1 (EXPAND) runs V2 live
|
||||
// while both replicas keep taking traffic - see scripts/run-all.sh.
|
||||
TestSupport.migrateTo(db, "2");
|
||||
|
||||
stop.set(true);
|
||||
inserter.join();
|
||||
|
||||
int missing = 0;
|
||||
List<Long> missingIds = new java.util.ArrayList<>();
|
||||
for (Long id : confirmedIds) {
|
||||
if (jdbc.sql("SELECT id FROM customers WHERE id = ?").param(id).query().listOfRows().isEmpty()) {
|
||||
missing++;
|
||||
missingIds.add(id);
|
||||
}
|
||||
}
|
||||
|
||||
if (missing == 0 && attempt < MAX_ATTEMPTS) {
|
||||
continue;
|
||||
}
|
||||
|
||||
t.line("attempts needed to reproduce the race: " + attempt + " of " + MAX_ATTEMPTS);
|
||||
t.line("customer creates that returned a generated id with no error: " + confirmedIds.size());
|
||||
t.line("customer creates that got the already-documented, already-fixed DDL-collision error: " + insertErrors.size());
|
||||
t.line("of the ids that came back with no error, missing from the table once V2 finished: " + missing);
|
||||
if (!missingIds.isEmpty()) {
|
||||
t.line("example missing ids: " + missingIds.subList(0, Math.min(5, missingIds.size())));
|
||||
}
|
||||
t.line("");
|
||||
t.line("This is why the retry in CustomerService cannot be the whole fix: these inserts");
|
||||
t.line("never threw anything to retry. The row was committed, then discarded when the");
|
||||
t.line("ADD COLUMN rebuild swapped in a new table that had already been scanned.");
|
||||
t.write();
|
||||
|
||||
// If this fails on the very last attempt, either H2's rebuild strategy changed
|
||||
// (worth its own investigation) or this environment schedules threads differently
|
||||
// enough that this test needs a heavier inserter - not that the finding is wrong.
|
||||
assertThat(missing)
|
||||
.withFailMessage("expected at least one silently-lost row within %d attempts; "
|
||||
+ "either H2's ALTER TABLE implementation changed, or this environment "
|
||||
+ "needs a heavier concurrent inserter to reproduce the race", MAX_ATTEMPTS)
|
||||
.isGreaterThan(0);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import com.ankurm.expandcontract.customer.CustomerService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* Deploy 2 (MIGRATE WRITES): Stage 2 code writes every create and update to both
|
||||
* columns. This is the deploy that makes Deploy 3's read switch safe later - if this
|
||||
* one is wrong, nothing downstream can be trusted. See docs/04-the-dual-write.md.
|
||||
*/
|
||||
class DualWriteConsistencyTest {
|
||||
|
||||
@Test
|
||||
void createAndUpdatePopulateBothColumnsIdentically(@TempDir Path tmp) throws Exception {
|
||||
Path db = tmp.resolve("dual-write");
|
||||
TestSupport.migrateTo(db, "2");
|
||||
Transcript t = Transcript.start("04-dual-write-consistency",
|
||||
"Deploy 2 (MIGRATE WRITES): Stage 2 writes land in both columns");
|
||||
|
||||
CustomerService stage2 = new CustomerService(TestSupport.jdbcClient(db), 2);
|
||||
long id = stage2.create("Margaret Hamilton", "[email protected]");
|
||||
|
||||
try (Connection conn = DriverManager.getConnection(TestSupport.jdbcUrl(db), "sa", "")) {
|
||||
t.section("after create()");
|
||||
String row = DbDump.table(conn, "select name, email, email_address from customers where id = " + id);
|
||||
t.line(row);
|
||||
assertThat(row).contains("[email protected]");
|
||||
assertThat(row.indexOf("[email protected]")).isNotEqualTo(row.lastIndexOf("[email protected]"));
|
||||
}
|
||||
|
||||
stage2.updateEmail(id, "[email protected]");
|
||||
|
||||
try (Connection conn = DriverManager.getConnection(TestSupport.jdbcUrl(db), "sa", "")) {
|
||||
t.section("after updateEmail() - the old value is gone from BOTH columns, not just one");
|
||||
String row = DbDump.table(conn, "select name, email, email_address from customers where id = " + id);
|
||||
t.line(row);
|
||||
assertThat(row).contains("[email protected]").doesNotContain("[email protected]");
|
||||
}
|
||||
|
||||
t.write();
|
||||
}
|
||||
}
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.jdbc.core.simple.JdbcClient;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* Deploy 1 (EXPAND) in isolation: migrate to V2 and confirm two things a real rollout
|
||||
* depends on. First, that every pre-existing row got backfilled in the same migration.
|
||||
* Second - the part that is easy to get wrong - that Stage 1 code, completely unaware
|
||||
* the new column exists, still inserts rows exactly as it always has. If this second
|
||||
* assertion ever failed, the migration would not be additive and expand-contract would
|
||||
* not apply to it. See docs/02-the-expand-migration.md.
|
||||
*/
|
||||
class ExpandMigrationBackwardCompatibleTest {
|
||||
|
||||
@Test
|
||||
void backfillsExistingRowsAndStaysCompatibleWithStage1Code(@TempDir Path tmp) throws Exception {
|
||||
Path db = tmp.resolve("expand");
|
||||
Transcript t = Transcript.start("02-expand-backward-compatible",
|
||||
"Deploy 1 (EXPAND): additive column + backfill, Stage 1 code untouched");
|
||||
|
||||
// Seed one row the way Stage 1 always has, before the expand migration exists.
|
||||
TestSupport.migrateTo(db, "1");
|
||||
JdbcClient preExpand = TestSupport.jdbcClient(db);
|
||||
preExpand.sql("INSERT INTO customers(name, email) VALUES (?, ?)")
|
||||
.param("Ada Lovelace").param("[email protected]").update();
|
||||
|
||||
t.section("schema before Deploy 1");
|
||||
try (Connection conn = DriverManager.getConnection(TestSupport.jdbcUrl(db), "sa", "")) {
|
||||
t.line(DbDump.table(conn, "select column_name from information_schema.columns "
|
||||
+ "where table_name = 'CUSTOMERS' order by ordinal_position"));
|
||||
}
|
||||
|
||||
// Deploy 1: the expand migration runs against the live database. No app restart.
|
||||
TestSupport.migrateTo(db, "2");
|
||||
|
||||
t.section("schema after Deploy 1 (email_address added)");
|
||||
try (Connection conn = DriverManager.getConnection(TestSupport.jdbcUrl(db), "sa", "")) {
|
||||
t.line(DbDump.table(conn, "select column_name from information_schema.columns "
|
||||
+ "where table_name = 'CUSTOMERS' order by ordinal_position"));
|
||||
|
||||
t.section("Ada's row was backfilled by the migration itself");
|
||||
String row = DbDump.table(conn, "select name, email, email_address from customers where name = 'Ada Lovelace'");
|
||||
t.line(row);
|
||||
// Backfilled: both columns hold the same value for a row that predates dual-write code.
|
||||
assertThat(row).contains("[email protected]");
|
||||
assertThat(row.indexOf("[email protected]")).isNotEqualTo(row.lastIndexOf("[email protected]"));
|
||||
}
|
||||
|
||||
// Stage 1 code has not been redeployed and does not know email_address exists.
|
||||
// Its insert statement is byte-for-byte what it was before Deploy 1.
|
||||
JdbcClient postExpand = TestSupport.jdbcClient(db);
|
||||
postExpand.sql("INSERT INTO customers(name, email) VALUES (?, ?)")
|
||||
.param("Grace Hopper").param("[email protected]").update();
|
||||
|
||||
t.section("Stage 1's original INSERT still works, unmodified, after the migration");
|
||||
try (Connection conn = DriverManager.getConnection(TestSupport.jdbcUrl(db), "sa", "")) {
|
||||
String row = DbDump.table(conn, "select name, email, email_address from customers where name = 'Grace Hopper'");
|
||||
t.line(row);
|
||||
assertThat(row).contains("[email protected]").contains("NULL");
|
||||
}
|
||||
|
||||
t.write();
|
||||
}
|
||||
}
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import com.ankurm.expandcontract.customer.Customer;
|
||||
import com.ankurm.expandcontract.customer.CustomerService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.jdbc.core.simple.JdbcClient;
|
||||
|
||||
import java.nio.file.Path;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* The property the whole article rests on: during a rolling deploy, two adjacent
|
||||
* stages are serving traffic against the SAME database at the SAME time, for however
|
||||
* long the rollout takes. If a customer written by one stage cannot be read correctly
|
||||
* by the other, the deploy is not zero-downtime - it is a race with the rollout clock.
|
||||
* <p>
|
||||
* This test builds two {@link CustomerService} instances that share one
|
||||
* {@link JdbcClient} (standing in for one shared database, hit by two replicas on
|
||||
* consecutive stages) and cross-checks every write/read direction for the three
|
||||
* rollouts this article performs: Stage 1↔2, Stage 2↔3, and Stage 3↔4.
|
||||
* See docs/08-the-rolling-window-proof.md - this is the test the live load generator
|
||||
* run in the article is reproducing under real HTTP and real timing.
|
||||
*/
|
||||
class MixedStageRollingWindowTest {
|
||||
|
||||
@Test
|
||||
void everyAdjacentStagePairReadsWhatTheOtherWrote(@TempDir Path tmp) throws Exception {
|
||||
Path db = tmp.resolve("rolling-window");
|
||||
Transcript t = Transcript.start("08-mixed-stage-rolling-window",
|
||||
"Cross-stage consistency during each of the three rolling deploys");
|
||||
|
||||
// Deploy 2's rollout: some replicas still on Stage 1, some already on Stage 2.
|
||||
TestSupport.migrateTo(db, "2");
|
||||
JdbcClient jdbc = TestSupport.jdbcClient(db);
|
||||
CustomerService stage1 = new CustomerService(jdbc, 1);
|
||||
CustomerService stage2 = new CustomerService(jdbc, 2);
|
||||
|
||||
t.section("Stage 1 writes, Stage 2 reads");
|
||||
long a = stage1.create("Radia Perlman", "[email protected]");
|
||||
Customer readByStage2 = stage2.findById(a).orElseThrow();
|
||||
t.line(readByStage2.toString());
|
||||
assertThat(readByStage2.email()).isEqualTo("[email protected]");
|
||||
|
||||
t.section("Stage 2 writes, Stage 1 reads");
|
||||
long b = stage2.create("Barbara Liskov", "[email protected]");
|
||||
Customer readByStage1 = stage1.findById(b).orElseThrow();
|
||||
t.line(readByStage1.toString());
|
||||
assertThat(readByStage1.email()).isEqualTo("[email protected]");
|
||||
|
||||
// Deploy 3's rollout: some replicas on Stage 2, some already on Stage 3.
|
||||
CustomerService stage3 = new CustomerService(jdbc, 3);
|
||||
|
||||
t.section("Stage 2 writes, Stage 3 reads");
|
||||
long c = stage2.create("Shafi Goldwasser", "[email protected]");
|
||||
Customer readByStage3 = stage3.findById(c).orElseThrow();
|
||||
t.line(readByStage3.toString());
|
||||
assertThat(readByStage3.email()).isEqualTo("[email protected]");
|
||||
|
||||
t.section("Stage 3 writes, Stage 2 reads");
|
||||
long d = stage3.create("Frances Allen", "[email protected]");
|
||||
Customer readByStage2Again = stage2.findById(d).orElseThrow();
|
||||
t.line(readByStage2Again.toString());
|
||||
assertThat(readByStage2Again.email()).isEqualTo("[email protected]");
|
||||
|
||||
// Deploy 4a's rollout: some replicas on Stage 3, some already on Stage 4.
|
||||
// email_address has been fully backfilled and dual-written for two whole
|
||||
// deploys by this point - the precondition Stage 4 relies on. The old "email"
|
||||
// column is still physically present (Deploy 4b, the drop, has not run yet)
|
||||
// but Stage 4 code never looks at it.
|
||||
CustomerService stage4 = new CustomerService(jdbc, 4);
|
||||
|
||||
t.section("Stage 3 writes, Stage 4 reads");
|
||||
long e = stage3.create("Adele Goldberg", "[email protected]");
|
||||
Customer readByStage4 = stage4.findById(e).orElseThrow();
|
||||
t.line(readByStage4.toString());
|
||||
assertThat(readByStage4.email()).isEqualTo("[email protected]");
|
||||
|
||||
t.section("Stage 4 writes, Stage 3 reads");
|
||||
long f = stage4.create("Karen Sparck Jones", "[email protected]");
|
||||
Customer readByStage3Again = stage3.findById(f).orElseThrow();
|
||||
t.line(readByStage3Again.toString());
|
||||
assertThat(readByStage3Again.email()).isEqualTo("[email protected]");
|
||||
|
||||
t.write();
|
||||
}
|
||||
}
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import com.ankurm.expandcontract.customer.CustomerService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.jdbc.core.simple.JdbcClient;
|
||||
|
||||
import java.nio.file.Path;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* A trap this article's own first draft of the V2 migration fell into: adding
|
||||
* {@code email_address} is not the whole story of "expand" when the column being
|
||||
* retired is {@code NOT NULL}. Stage 4 code never writes "email" - so if "email" is
|
||||
* still mandatory when Stage 4 starts, every create fails, in production, the moment
|
||||
* that deploy reaches its first replica. The fix is one more line in the SAME
|
||||
* migration: {@code ALTER TABLE customers ALTER COLUMN email DROP NOT NULL}. See
|
||||
* docs/06-the-not-null-trap.md, and compare
|
||||
* {@link com.ankurm.expandcontract.customer.CustomerService#create} - the shipped
|
||||
* V2 migration (with the fix) is what every other test in this module runs against.
|
||||
*/
|
||||
class NotNullConstraintTrapTest {
|
||||
|
||||
@Test
|
||||
void stage4FailsIfTheOldColumnIsStillMandatory(@TempDir Path tmp) throws Exception {
|
||||
Path db = tmp.resolve("not-null-trap");
|
||||
TestSupport.migrateTo(db, "1");
|
||||
JdbcClient jdbc = TestSupport.jdbcClient(db);
|
||||
Transcript t = Transcript.start("06-not-null-trap",
|
||||
"The NOT NULL trap: expand without relaxing the old column's constraint");
|
||||
|
||||
// The NAIVE version of V2: add the column, backfill, stop there. This is
|
||||
// exactly V2 minus its last line.
|
||||
jdbc.sql("ALTER TABLE customers ADD COLUMN email_address VARCHAR(320)").update();
|
||||
jdbc.sql("UPDATE customers SET email_address = email WHERE email_address IS NULL").update();
|
||||
|
||||
CustomerService stage4 = new CustomerService(jdbc, 4);
|
||||
Throwable thrown = null;
|
||||
try {
|
||||
stage4.create("Too Early", "[email protected]");
|
||||
} catch (Throwable ex) {
|
||||
thrown = ex;
|
||||
}
|
||||
t.section("Stage 4 create() against the NAIVE migration (no DROP NOT NULL)");
|
||||
assertThat(thrown).isNotNull();
|
||||
Throwable root = thrown;
|
||||
while (root.getCause() != null) {
|
||||
root = root.getCause();
|
||||
}
|
||||
t.line(thrown.getClass().getName() + ": " + thrown.getMessage());
|
||||
t.line("root cause: " + root.getClass().getName() + ": " + root.getMessage());
|
||||
assertThat(root.getMessage()).contains("NULL not allowed for column \"EMAIL\"");
|
||||
|
||||
t.section("Stage 4 create() against the SHIPPED V2 migration (DROP NOT NULL included)");
|
||||
Path fixedDb = tmp.resolve("not-null-fixed");
|
||||
TestSupport.migrateTo(fixedDb, "2");
|
||||
CustomerService fixedStage4 = new CustomerService(TestSupport.jdbcClient(fixedDb), 4);
|
||||
long id = fixedStage4.create("On Time", "[email protected]");
|
||||
var result = fixedStage4.findById(id).orElseThrow();
|
||||
t.line(result.toString());
|
||||
assertThat(result.email()).isEqualTo("[email protected]");
|
||||
|
||||
t.write();
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import org.springframework.boot.autoconfigure.AutoConfigurations;
|
||||
import org.springframework.boot.flyway.autoconfigure.FlywayAutoConfiguration;
|
||||
import org.springframework.boot.jdbc.autoconfigure.DataSourceAutoConfiguration;
|
||||
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
|
||||
import org.springframework.jdbc.core.simple.JdbcClient;
|
||||
import org.springframework.jdbc.datasource.DriverManagerDataSource;
|
||||
|
||||
import java.nio.file.Path;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* Runs Flyway to a given target version through Boot's own {@code FlywayAutoConfiguration}
|
||||
* (the same {@code spring.flyway.target} property a real deploy pipeline would set), then
|
||||
* hands the test a plain {@link JdbcClient} against the same file - so a test can migrate a
|
||||
* database to "however far Deploy N has gotten" and then exercise
|
||||
* {@link com.ankurm.expandcontract.customer.CustomerService} instances directly, with no
|
||||
* Spring context of their own, exactly as the app constructs them.
|
||||
*/
|
||||
final class TestSupport {
|
||||
|
||||
static void migrateTo(Path dbPath, String target) {
|
||||
new ApplicationContextRunner()
|
||||
.withConfiguration(AutoConfigurations.of(DataSourceAutoConfiguration.class, FlywayAutoConfiguration.class))
|
||||
.withPropertyValues(
|
||||
"spring.datasource.url=" + jdbcUrl(dbPath),
|
||||
"spring.datasource.username=sa",
|
||||
"spring.flyway.locations=classpath:db/migration",
|
||||
"spring.flyway.target=" + target)
|
||||
.run(ctx -> assertThat(ctx).hasNotFailed());
|
||||
}
|
||||
|
||||
static JdbcClient jdbcClient(Path dbPath) {
|
||||
DriverManagerDataSource ds = new DriverManagerDataSource(jdbcUrl(dbPath), "sa", "");
|
||||
return JdbcClient.create(ds);
|
||||
}
|
||||
|
||||
static String jdbcUrl(Path dbPath) {
|
||||
return "jdbc:h2:file:" + dbPath + ";AUTO_SERVER=TRUE";
|
||||
}
|
||||
|
||||
private TestSupport() {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.time.Instant;
|
||||
|
||||
/**
|
||||
* Writes a named transcript under {@code docs/output/} while the test that produced it runs.
|
||||
* Every console line quoted in the companion article for this module comes from one of these
|
||||
* files, and the file is only ever written by a test assertion - never hand-typed.
|
||||
*/
|
||||
public final class Transcript {
|
||||
|
||||
private static final Path OUTPUT_DIR = Paths.get("docs", "output");
|
||||
|
||||
private final StringBuilder buffer = new StringBuilder();
|
||||
private final String name;
|
||||
|
||||
private Transcript(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public static Transcript start(String name, String heading) {
|
||||
Transcript t = new Transcript(name);
|
||||
t.line("=".repeat(Math.min(78, heading.length() + 4)));
|
||||
t.line(heading);
|
||||
t.line("=".repeat(Math.min(78, heading.length() + 4)));
|
||||
t.line("captured: " + Instant.now());
|
||||
t.line("");
|
||||
return t;
|
||||
}
|
||||
|
||||
public Transcript line(String text) {
|
||||
buffer.append(text).append(System.lineSeparator());
|
||||
System.out.println(text);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Transcript section(String title) {
|
||||
line("");
|
||||
line("-- " + title + " --");
|
||||
return this;
|
||||
}
|
||||
|
||||
public void write() {
|
||||
try {
|
||||
Files.createDirectories(OUTPUT_DIR);
|
||||
Path target = OUTPUT_DIR.resolve(name + ".txt");
|
||||
Files.writeString(target, buffer.toString(), StandardOpenOption.CREATE,
|
||||
StandardOpenOption.TRUNCATE_EXISTING);
|
||||
}
|
||||
catch (IOException ex) {
|
||||
throw new UncheckedIOException(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user