asmhatreandClaude Sonnet 5
e478eafda3
Companion code for Zero-Downtime Database Migrations: Expand-Contract in Practice with Spring Boot: a full expand/migrate-writes/migrate-reads/contract sequence run as an actual rolling deploy across two live replicas, with a load generator sending continuous HTTP traffic through all four deploys (99.98% success, every residual error traced to a root cause rather than left unexplained). Findings include a real NOT NULL constraint trap in the expand migration, a backfill-window bug in the read switch, H2's AUTO_SERVER=TRUE single-point-of-failure behavior under a rolling restart, the drain-before-SIGTERM fix needed to close a health-check gap during graceful shutdown, and H2 silently discarding a concurrently committed INSERT during an ALTER TABLE ADD/DROP COLUMN rebuild - confirmed, by primary source, to be an H2-specific behavior rather than a property of the technique itself. Co-Authored-By: Claude Sonnet 5 <[email protected]> Claude-Session: https://claude.ai/code/session_019Fb7vW8vLyLKngBc4R3huA
69 lines
3.3 KiB
Java
69 lines
3.3 KiB
Java
package com.ankurm.expandcontract;
|
|
|
|
import com.ankurm.expandcontract.customer.Customer;
|
|
import com.ankurm.expandcontract.customer.CustomerService;
|
|
import org.junit.jupiter.api.Test;
|
|
import org.junit.jupiter.api.io.TempDir;
|
|
import org.springframework.jdbc.core.simple.JdbcClient;
|
|
|
|
import java.nio.file.Path;
|
|
import java.util.Optional;
|
|
|
|
import static org.assertj.core.api.Assertions.assertThat;
|
|
|
|
/**
|
|
* The gap between "the expand migration ran" and "every Stage 1 instance in the fleet
|
|
* has been replaced by Stage 2" is not instantaneous - a real rolling deploy takes
|
|
* minutes, and every row a lingering Stage 1 instance writes during that window has
|
|
* {@code email_address = NULL}. The migration's one-time backfill (V2) only ever sees
|
|
* rows that existed *before* Deploy 1 ran; it cannot see rows a Stage 1 instance writes
|
|
* *after* that, during its own rollout window.
|
|
* <p>
|
|
* This test reproduces the resulting bug with a naive Stage 3 read (email_address alone)
|
|
* and then shows the fix that ships in
|
|
* {@link com.ankurm.expandcontract.customer.CustomerService#findById}: a plain
|
|
* {@code COALESCE(email_address, email)}. See docs/07-the-backfill-window-bug.md.
|
|
*/
|
|
class BackfillWindowBugTest {
|
|
|
|
@Test
|
|
void naiveStage3ReadReturnsNullForARowStage1WroteDuringTheRollout(@TempDir Path tmp) throws Exception {
|
|
Path db = tmp.resolve("backfill-window");
|
|
TestSupport.migrateTo(db, "2");
|
|
Transcript t = Transcript.start("07-backfill-window-bug",
|
|
"The backfill window: a Stage 1 write after Deploy 1, read by a naive Stage 3");
|
|
|
|
// A Stage 1 instance is still serving traffic during the Deploy 2 rollout and
|
|
// writes a brand new row exactly as it always has - it has never heard of
|
|
// email_address. This is not a hypothetical; it is guaranteed to happen for
|
|
// however long the rollout takes.
|
|
JdbcClient jdbc = TestSupport.jdbcClient(db);
|
|
jdbc.sql("INSERT INTO customers(name, email) VALUES (?, ?)")
|
|
.param("Katherine Johnson").param("[email protected]")
|
|
.update();
|
|
long insertedId = jdbc.sql("SELECT id FROM customers WHERE name = ?")
|
|
.param("Katherine Johnson").query(Long.class).single();
|
|
|
|
t.section("the row a lingering Stage 1 instance just wrote");
|
|
String naiveRead = jdbc.sql("SELECT name, email, email_address FROM customers WHERE id = ?")
|
|
.param(insertedId).query().listOfRows().toString();
|
|
t.line(naiveRead);
|
|
assertThat(naiveRead).containsIgnoringCase("email_address=null");
|
|
|
|
t.section("a NAIVE Stage 3 read (email_address alone) - the bug");
|
|
String naiveEmail = jdbc.sql("SELECT email_address FROM customers WHERE id = ?")
|
|
.param(insertedId).query(String.class).optional().orElse(null);
|
|
t.line("naive Stage 3 email column value: " + naiveEmail);
|
|
assertThat(naiveEmail).isNull();
|
|
|
|
t.section("the SHIPPED Stage 3 read (CustomerService, COALESCE) - the fix");
|
|
CustomerService stage3 = new CustomerService(jdbc, 3);
|
|
Optional<Customer> fixed = stage3.findById(insertedId);
|
|
t.line("CustomerService (stage 3) result: " + fixed);
|
|
assertThat(fixed).isPresent();
|
|
assertThat(fixed.get().email()).isEqualTo("[email protected]");
|
|
|
|
t.write();
|
|
}
|
|
}
|