asmhatre/spring-boot-demo
1
0
Fork 0
Code Activity

Add db-migrations-expand-contract: zero-downtime schema migrations proven with a real 4-deploy rolling run

Browse Source 
Companion code for Zero-Downtime Database Migrations: Expand-Contract in Practice
with Spring Boot: a full expand/migrate-writes/migrate-reads/contract sequence run
as an actual rolling deploy across two live replicas, with a load generator sending
continuous HTTP traffic through all four deploys (99.98% success, every residual
error traced to a root cause rather than left unexplained). Findings include a real
NOT NULL constraint trap in the expand migration, a backfill-window bug in the read
switch, H2's AUTO_SERVER=TRUE single-point-of-failure behavior under a rolling
restart, the drain-before-SIGTERM fix needed to close a health-check gap during
graceful shutdown, and H2 silently discarding a concurrently committed INSERT during
an ALTER TABLE ADD/DROP COLUMN rebuild - confirmed, by primary source, to be an
H2-specific behavior rather than a property of the technique itself.

Co-Authored-By: Claude Sonnet 5 <[email protected]>
Claude-Session: https://claude.ai/code/session_019Fb7vW8vLyLKngBc4R3huA
This commit is contained in:
asmhatre
2026-09-16 19:22:48 +00:00
co-authored by Claude Sonnet 5
parent 22c30d4a5b
commit e478eafda3
60 changed files with 3561 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/DdlSilentDataLossTest.java
+132
View File
@@ -0,0 +1,132 @@
package com.ankurm.expandcontract;
import com.ankurm.expandcontract.customer.CustomerService;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.jdbc.core.simple.JdbcClient;
import java.nio.file.Path;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.assertj.core.api.Assertions.assertThat;
/**
* The finding that {@link com.ankurm.expandcontract.customer.CustomerService#withRetryForConcurrentDdl}
* cannot fix, because nothing throws. The retry helper exists for the case where a concurrent
* statement collides with an in-progress {@code ALTER TABLE} and H2 answers with
* "table not found" - a real error the caller can see and retry. This test demonstrates a second,
* stranger failure mode found the same way the first one was: by running this module's own load
* generator against a live rolling deploy and noticing a handful of reads and updates come back
* 404 for a customer id that a 201 response had already confirmed existed.
* <p>
* H2 implements both {@code ALTER TABLE ... ADD COLUMN} and {@code ALTER TABLE ... DROP COLUMN}
* by rebuilding the table: copying every row into a new table with the new column layout and
* swapping it in. If an ordinary {@code INSERT} on another connection commits - with no error,
* with a generated key handed back to the caller - while that rebuild is in flight, the inserted
* row can be copied into the new table or left behind in the old one depending on exactly when
* the rebuild's internal scan ran relative to the commit. When it is left behind, the row is gone
* the instant the rebuild finishes, and nothing on the inserting connection was ever told.
* <p>
* This is not how every database implements {@code ADD COLUMN} and {@code DROP COLUMN}. PostgreSQL's
* reference manual is explicit that both are metadata-only operations on tables like this one -
* no non-volatile default and no immediate space reclamation requested - so this is a property of
* H2's implementation, not of the expand-contract technique itself. See
* docs/14-the-ddl-lock-window.md for the full explanation, the standalone reproduction this test
* is built from, and what it implies for choosing a target database for a real migration.
*/
class DdlSilentDataLossTest {
// Whether the rebuild's internal scan actually passes a given row before or after that
// row's INSERT commits is OS scheduling, not application logic - so a single attempt at
// this race can genuinely land on either side of it. A single-attempt version of this test
// failed about 1 run in 5 while writing it. Rather than assert on one attempt (flaky either
// way) or loosen the assertion to "zero or more" (which would silently stop proving anything
// the day this stops reproducing), this test repeats the race, on a fresh table each time,
// until it reproduces - the same thing a human would do at a terminal to confirm a suspected
// race is real. Twenty attempts reproduced it within the first 4 in every run made while
// writing this test.
private static final int MAX_ATTEMPTS = 20;
@Test
void concurrentAlterTableAddColumnCanSilentlyDropAnAlreadyCommittedInsert(@TempDir Path tmp) throws Exception {
Transcript t = Transcript.start("14-ddl-silent-data-loss",
"The failure the retry cannot catch: a committed INSERT that ALTER TABLE loses silently");
for (int attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
Path db = tmp.resolve("silent-data-loss-" + attempt);
TestSupport.migrateTo(db, "1");
JdbcClient jdbc = TestSupport.jdbcClient(db);
CustomerService stage1 = new CustomerService(jdbc, 1);
AtomicBoolean stop = new AtomicBoolean(false);
List<Long> confirmedIds = new CopyOnWriteArrayList<>();
List<String> insertErrors = new CopyOnWriteArrayList<>();
CountDownLatch started = new CountDownLatch(1);
Thread inserter = new Thread(() -> {
started.countDown();
int n = 0;
while (!stop.get()) {
n++;
try {
long id = stage1.create("Concurrent " + n, "concurrent" + n + "@example.test");
confirmedIds.add(id);
} catch (Exception ex) {
// The already-documented, already-fixed failure mode: a statement that
// collides with the DDL and is told so. Counted here only to show it is
// rare and separate from the silent loss this test is isolating.
insertErrors.add(ex.getClass().getSimpleName());
}
}
});
inserter.start();
started.await();
// The DDL runs on its own connection, exactly as Deploy 1 (EXPAND) runs V2 live
// while both replicas keep taking traffic - see scripts/run-all.sh.
TestSupport.migrateTo(db, "2");
stop.set(true);
inserter.join();
int missing = 0;
List<Long> missingIds = new java.util.ArrayList<>();
for (Long id : confirmedIds) {
if (jdbc.sql("SELECT id FROM customers WHERE id = ?").param(id).query().listOfRows().isEmpty()) {
missing++;
missingIds.add(id);
}
}
if (missing == 0 && attempt < MAX_ATTEMPTS) {
continue;
}
t.line("attempts needed to reproduce the race: " + attempt + " of " + MAX_ATTEMPTS);
t.line("customer creates that returned a generated id with no error: " + confirmedIds.size());
t.line("customer creates that got the already-documented, already-fixed DDL-collision error: " + insertErrors.size());
t.line("of the ids that came back with no error, missing from the table once V2 finished: " + missing);
if (!missingIds.isEmpty()) {
t.line("example missing ids: " + missingIds.subList(0, Math.min(5, missingIds.size())));
}
t.line("");
t.line("This is why the retry in CustomerService cannot be the whole fix: these inserts");
t.line("never threw anything to retry. The row was committed, then discarded when the");
t.line("ADD COLUMN rebuild swapped in a new table that had already been scanned.");
t.write();
// If this fails on the very last attempt, either H2's rebuild strategy changed
// (worth its own investigation) or this environment schedules threads differently
// enough that this test needs a heavier inserter - not that the finding is wrong.
assertThat(missing)
.withFailMessage("expected at least one silently-lost row within %d attempts; "
+ "either H2's ALTER TABLE implementation changed, or this environment "
+ "needs a heavier concurrent inserter to reproduce the race", MAX_ATTEMPTS)
.isGreaterThan(0);
return;
}
}
}