diff --git a/.gitignore b/.gitignore index 9262c83..2050e13 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,8 @@ spring-batch/scenario-data/ # spring-batch-partitioning: generated shard CSVs and file-based H2 databases spring-batch-partitioning/data/ + +# db-migrations-expand-contract: runtime classpath file written by `mvn package` +# (maven-dependency-plugin's build-classpath goal) for the standalone MigrationCli +# and LoadGenerator entry points - machine-specific, regenerated on every build +db-migrations-expand-contract/cp.txt diff --git a/README.md b/README.md index 402ddc8..7c07990 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ files. | [`spring-batch/`](spring-batch) | [Spring Batch on Boot 4.1: Jobs, Steps, Chunk Processing and Restartability](https://ankurm.com/) | a job that fails mid-chunk and resumes exactly where it left off across two separate JVMs, skip vs. restart on the same poisoned row, the resourceless job repository that forgets a restart ever happened, and the `chunk(int)` vs `chunk(int, tx)` builder split | | [`spring-batch-partitioning/`](spring-batch-partitioning) | [Spring Batch Partitioning and Parallel Steps: Scaling a 10-Million-Row Job](https://ankurm.com/) | the real grid-size sweep at 10M and 300K rows (best speedup 1.42x, on 2 cores), `MultiResourcePartitioner` ignoring gridSize entirely, a rejected partition's `StepExecution` stuck at `STARTING` forever, and Spring Batch 6.0's new `JobOperator#recover` unsticking it | | [`db-migrations-flyway-liquibase/`](db-migrations-flyway-liquibase) | [Flyway vs Liquibase for Spring Boot 4: Migrations, Rollbacks and Baselines](https://ankurm.com/flyway-vs-liquibase-spring-boot-4-migrations-rollbacks-baselines/) | Flyway Community's `undo` throwing `FlywayRedgateEditionRequiredException` at runtime, a real Liquibase 5.0.3 filename-caching defect that produces a phantom successful run, Liquibase's 10-second default lock-poll rate versus Flyway's near-instant row lock, the FSL license change and its ASF/Keycloak fallout, and what actually happens when both tools are enabled against one database | +| [`db-migrations-expand-contract/`](db-migrations-expand-contract) | [Zero-Downtime Database Migrations: Expand-Contract in Practice with Spring Boot](https://ankurm.com/zero-downtime-database-migrations-expand-contract-spring-boot/) | a real 4-deploy rolling sequence against two live replicas with a load generator proving 99.98% success, H2's `AUTO_SERVER` single-point-of-failure trap, a `NOT NULL` constraint that fails every Stage 4 insert, and `ALTER TABLE` silently dropping a concurrently committed row with no exception thrown | Articles whose text is kept here rather than only on the blog have it under `/post/` — `post.md` for the body and `meta.md` for the title, excerpt and diff --git a/db-migrations-expand-contract/README.md b/db-migrations-expand-contract/README.md new file mode 100644 index 0000000..22d3419 --- /dev/null +++ b/db-migrations-expand-contract/README.md @@ -0,0 +1,126 @@ +# db-migrations-expand-contract + +Companion code for **[Zero-Downtime Database Migrations: Expand-Contract in Practice +with Spring Boot](https://ankurm.com/zero-downtime-database-migrations-expand-contract-spring-boot/)** +— every claim in that post traces to a test in this module, a real 4-deploy live run, +or a transcript in [`docs/output/`](docs/output). + +## Versions + +| Component | Version | +|---|---| +| Spring Boot | 4.1.1 | +| Flyway (via `spring-boot-starter-flyway`) | 12.4.0 | +| Jackson (via `spring-boot-starter-jackson`) | `tools.jackson` 3.1.5 | +| Database | H2 2.4.240, standalone TCP server mode | +| JDK | 25 (LTS) | + +## Quickstart + +```bash +mvn -DskipTests package # builds target/db-migrations-expand-contract-1.0.0.jar +mvn test # regenerates every unit-test transcript in docs/output/ +./scripts/run-all.sh # the live exhibit: real 4-deploy rolling sequence + load generator +``` + +`run-all.sh` starts a standalone H2 TCP server, brings up two replicas on Stage 1, +runs all four deploys of the expand-contract sequence as an actual rolling deploy — +schema-only, code-only, schema-only, in the right order — and keeps a load generator +sending real HTTP traffic through the entire thing. It regenerates: + +- [`docs/output/11-live-deploy-sequence.txt`](docs/output/11-live-deploy-sequence.txt) — the deploy log, phase by phase +- [`docs/output/12-load-generator-summary.txt`](docs/output/12-load-generator-summary.txt) — total/ok/error counts, by phase +- [`docs/output/13-schema-diagnostics-timeline.txt`](docs/output/13-schema-diagnostics-timeline.txt) — `/diag/schema` after each deploy + +## The four deploys (Spring profile: `app.stage`) + +| Stage | What it does | Deploy kind | +|---|---|---| +| 1 | Reads and writes only `email` | *(baseline, before this article starts)* | +| 2 | Writes both `email` and `email_address`; still reads `email` | Deploy 2 — code | +| 3 | Writes both columns; reads `COALESCE(email_address, email)` | Deploy 3 — code | +| 4 | Reads and writes only `email_address` | Deploy 4a — code | + +Between stages, two schema-only migrations run with **zero application restarts**: +Deploy 1 (EXPAND, `V2__add_email_address_column.sql`) before Stage 2 ships, and +Deploy 4b (CONTRACT, `V3__drop_email_column.sql`) after every replica is confirmed on +Stage 4. See [`docs/01-the-problem-and-the-plan.md`](docs/01-the-problem-and-the-plan.md). + +## Endpoints + +| Endpoint | Purpose | +|---|---| +| `POST /customers`, `GET /customers/{id}`, `PUT /customers/{id}/email` | The API contract that never changes across all four stages — see [`CustomerController`](src/main/java/com/ankurm/expandcontract/customer/CustomerController.java) | +| `GET /diag/schema` | Live column list + row counts — the exhibit that shows the schema actually expanding and contracting. **Delete before shipping** (see [chapter 15](docs/15-production-checklist.md)) | +| `POST /admin/drain` | Publishes `ReadinessState.REFUSING_TRAFFIC` before a graceful shutdown — the `preStop`-hook pattern. **Unauthenticated in this demo — lock down before shipping** | +| `GET /actuator/health`, `GET /actuator/info` | Boot's own actuator endpoints | + +## Documentation + +1. [The problem and the plan](docs/01-the-problem-and-the-plan.md) +2. [The expand migration](docs/02-the-expand-migration.md) +3. [Why migrations run outside the app](docs/03-why-migrations-run-outside-the-app.md) +4. [The dual write](docs/04-the-dual-write.md) +5. [The read switch](docs/05-the-read-switch.md) +6. [The NOT NULL trap](docs/06-the-not-null-trap.md) +7. [The backfill window bug](docs/07-the-backfill-window-bug.md) +8. [The rolling-window proof](docs/08-the-rolling-window-proof.md) +9. [The contract migration](docs/09-the-contract-migration.md) +10. [What happens if you drop too soon](docs/10-what-happens-if-you-drop-too-soon.md) +11. [The load generator](docs/11-the-load-generator.md) +12. [The AUTO_SERVER trap](docs/12-the-auto-server-trap.md) +13. [Graceful shutdown vs. kill -9](docs/13-graceful-shutdown-vs-kill-9.md) +14. [The DDL lock window](docs/14-the-ddl-lock-window.md) — the module's central finding +15. [Production checklist](docs/15-production-checklist.md) + +## Captured output + +Every number quoted in the post and in the chapters above comes from a committed +transcript in [`docs/output/`](docs/output). Files `02`–`10` and `14` are written by a +`Transcript` helper while the JUnit test that produced them asserts the same numbers — +a transcript going stale fails the build. Files `11`–`13` come from the live +`run-all.sh` sequence. + +| File | Source | +|---|---| +| [`02-expand-backward-compatible.txt`](docs/output/02-expand-backward-compatible.txt) | [`ExpandMigrationBackwardCompatibleTest`](src/test/java/com/ankurm/expandcontract/ExpandMigrationBackwardCompatibleTest.java) | +| [`04-dual-write-consistency.txt`](docs/output/04-dual-write-consistency.txt) | [`DualWriteConsistencyTest`](src/test/java/com/ankurm/expandcontract/DualWriteConsistencyTest.java) | +| [`06-not-null-trap.txt`](docs/output/06-not-null-trap.txt) | [`NotNullConstraintTrapTest`](src/test/java/com/ankurm/expandcontract/NotNullConstraintTrapTest.java) | +| [`07-backfill-window-bug.txt`](docs/output/07-backfill-window-bug.txt) | [`BackfillWindowBugTest`](src/test/java/com/ankurm/expandcontract/BackfillWindowBugTest.java) | +| [`08-mixed-stage-rolling-window.txt`](docs/output/08-mixed-stage-rolling-window.txt) | [`MixedStageRollingWindowTest`](src/test/java/com/ankurm/expandcontract/MixedStageRollingWindowTest.java) | +| [`09-contract-safety.txt`](docs/output/09-contract-safety.txt) | [`ContractSafetyTest`](src/test/java/com/ankurm/expandcontract/ContractSafetyTest.java) (happy path) | +| [`10-drop-too-soon.txt`](docs/output/10-drop-too-soon.txt) | [`ContractSafetyTest`](src/test/java/com/ankurm/expandcontract/ContractSafetyTest.java) (dropped too soon) | +| [`11-live-deploy-sequence.txt`](docs/output/11-live-deploy-sequence.txt) | [`scripts/run-all.sh`](scripts/run-all.sh) | +| [`12-load-generator-summary.txt`](docs/output/12-load-generator-summary.txt) | [`LoadGenerator`](src/main/java/com/ankurm/expandcontract/loadgen/LoadGenerator.java), via `run-all.sh` | +| [`13-schema-diagnostics-timeline.txt`](docs/output/13-schema-diagnostics-timeline.txt) | [`SchemaDiagnosticsController`](src/main/java/com/ankurm/expandcontract/diag/SchemaDiagnosticsController.java), via `run-all.sh` | +| [`14-ddl-silent-data-loss.txt`](docs/output/14-ddl-silent-data-loss.txt) | [`DdlSilentDataLossTest`](src/test/java/com/ankurm/expandcontract/DdlSilentDataLossTest.java) | + +## Findings worth the trip + +- H2's `AUTO_SERVER=TRUE` "shared embedded database" mode makes the first connecting + process the de facto server for every other connection — killing that one replica + during a rolling restart broke the *other* replica's "embedded" database entirely. + Fixed by running H2 as an independent standalone TCP server ([chapter 12](docs/12-the-auto-server-trap.md)). +- Expanding a column without relaxing the retired column's `NOT NULL` constraint + fails every Stage 4 `INSERT` from the first request onward — a genuine bug in this + module's own first draft, reproduced and fixed in the same migration + ([chapter 6](docs/06-the-not-null-trap.md)). +- A naive Stage 3 read of `email_address` alone returns `NULL` for any row a + still-live Stage 1 replica wrote during Deploy 2's own rollout window — fixed with + `COALESCE(email_address, email)` ([chapter 7](docs/07-the-backfill-window-bug.md)). +- H2 implements `ALTER TABLE ADD COLUMN` and `DROP COLUMN` by rebuilding the table, + which can silently discard a row committed by a concurrent `INSERT` while the + rebuild is mid-scan — with no exception thrown to the inserting connection. + Reproduced directly, and confirmed to be an H2-specific behavior rather than a + property of the expand-contract technique itself, since PostgreSQL documents both + operations as metadata-only ([chapter 14](docs/14-the-ddl-lock-window.md) — the + module's central finding). +- A real rolling restart, with graceful shutdown alone, still produces + `ConnectException` bursts: the health-checked pool needs time to notice a draining + instance before that instance is actually killed. A `/admin/drain` endpoint that + publishes `ReadinessState.REFUSING_TRAFFIC` before `SIGTERM` closes that gap + ([chapter 13](docs/13-graceful-shutdown-vs-kill-9.md)). +- The full live sequence — real HTTP traffic through a real 4-deploy rolling restart — + finished at 99.98% success (30,905 of 30,911 requests), with every one of the six + residual errors traced to a single, explained root cause rather than left as an + unexplained miss ([chapter 11](docs/11-the-load-generator.md)). diff --git a/db-migrations-expand-contract/docs/01-the-problem-and-the-plan.md b/db-migrations-expand-contract/docs/01-the-problem-and-the-plan.md new file mode 100644 index 0000000..b052274 --- /dev/null +++ b/db-migrations-expand-contract/docs/01-the-problem-and-the-plan.md @@ -0,0 +1,116 @@ +# 1. The problem and the plan + +[Next: 2. The expand migration →](02-the-expand-migration.md) + +## The problem + +`customers` has a column called `email`. You want it called `email_address` — maybe +because a second `contact_email` table is coming and the naming needs to be +consistent, maybe because "email" collided with a reserved word in a tool you just +adopted. The reason doesn't matter. What matters is that this table has rows in it, +right now, in production, and something is reading and writing that column while you +work. + +The naive fix is one migration: + +```sql +ALTER TABLE customers RENAME COLUMN email TO email_address; +``` + +That statement is correct and it is also a production outage. The instant it commits, +every currently-running copy of your application — the ones you have not redeployed +yet, because a rolling deploy takes minutes, not zero seconds — starts issuing SQL +against a column that no longer exists. `INSERT INTO customers(name, email) VALUES +(?, ?)` becomes a 500 on every single request, on every replica that hasn't restarted +yet, until the rollout finishes. You have coupled a **schema change** to a **code +deploy**, and the two of them do not happen atomically across a fleet. + +## The plan: expand, migrate, contract + +Expand-contract (sometimes "parallel change") solves this by never letting the schema +and the code disagree about what's safe. Instead of one migration and one deploy, it's +four: + +1. **Expand** — add the new column, alongside the old one. Nothing reads it yet. + Nothing that's running has to change. +2. **Migrate writes** — deploy code that writes to *both* columns. Every row created + or updated from this point on is consistent in both places. +3. **Migrate reads** — deploy code that reads from the new column instead of the old + one. This is a *separate* deploy from step 2, and the gap between them matters more + than it looks like it should — see [chapter 5](05-the-read-switch.md). +4. **Contract** — once every replica in the fleet is confirmed running the Stage 4 + code from step 3, drop the old column. Nothing is reading or writing it anymore, so + dropping it is safe. + + + + + +1. EXPAND +add email_address +schema only, no deploy + +2. MIGRATE WRITES +write both columns +code deploy + +3. MIGRATE READS +read email_address +code deploy + +4. CONTRACT +drop email +schema, then cleanup + + + + +Same database, the whole time +customers.email [always present until step 4] +customers.email_address [present from step 1 onward, populated from step 2 onward] +Two adjacent stages serve real traffic against this table at once during every rollout above. + + +The diagram's bottom half is the fact the rest of this article keeps coming back to: +at every point during a rolling deploy, two adjacent stages are running against the +same table at the same time. Deploy 2's rollout has Stage 1 and Stage 2 replicas live +together for however long the rollout takes; Deploy 3's rollout has Stage 2 and Stage +3 together; Deploy 4a's has Stage 3 and Stage 4 together. Each of those overlaps is a +window where the "old" code and the "new" code both have to produce correct answers +against a schema neither one fully owns. [Chapter 8](08-the-rolling-window-proof.md) +is the test that checks every one of those six write/read combinations directly, and +the article's live load-generator run reproduces the same overlaps under real HTTP +traffic and real timing. + +## What never changes + +The four deploys change exactly one thing about how the *database* is used. They +change nothing about the *API*: + +```java +public record Customer(long id, String name, String email) { +} +``` + +[`Customer.java`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/customer/Customer.java) +never mentions `email_address` — a caller of this API cannot tell which stage +answered a given request just by looking at the response shape, and neither can the +article's own load generator. That's deliberate: expand-contract is a technique for +changing storage without changing the contract clients depend on. + +## Going deeper + +- The companion module for this article is `db-migrations-expand-contract` in + [`spring-boot-demo`](https://ankurm.com/git.app/asmhatre/spring-boot-demo) — every + chapter from here on links to a real file or a real captured transcript in it. +- Martin Fowler's [ParallelChange](https://martinfowler.com/bliki/ParallelChange.html) + is the canonical name and description of this pattern outside a specific database or + framework. + +[Next: 2. The expand migration →](02-the-expand-migration.md) diff --git a/db-migrations-expand-contract/docs/02-the-expand-migration.md b/db-migrations-expand-contract/docs/02-the-expand-migration.md new file mode 100644 index 0000000..79a7a91 --- /dev/null +++ b/db-migrations-expand-contract/docs/02-the-expand-migration.md @@ -0,0 +1,65 @@ +# 2. The expand migration + +[← 1. The problem and the plan](01-the-problem-and-the-plan.md) · [Next: 3. Why migrations run outside the app →](03-why-migrations-run-outside-the-app.md) + +Deploy 1 is schema-only. No application code changes, no replica restarts. The +migration is +[`V2__add_email_address_column.sql`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/main/resources/db/migration/V2__add_email_address_column.sql): + +```sql +ALTER TABLE customers ADD COLUMN email_address VARCHAR(320); +UPDATE customers SET email_address = email WHERE email_address IS NULL; +ALTER TABLE customers ALTER COLUMN email DROP NOT NULL; +``` + +Three statements, three separate jobs: + +- **Add the column, nullable.** Nullable is what makes it additive: no existing + `INSERT` statement mentions `email_address`, so none of them break. A column added + `NOT NULL` with no default would fail immediately for any code still running the old + `INSERT INTO customers(name, email) VALUES (?, ?)`. +- **Backfill it** for every row that already exists. This is a one-time pass over + whatever data predates Deploy 1. +- **Relax the old column's constraint.** This third line is the one that's easy to + skip, and skipping it is a real, reproducible outage — see + [chapter 6](06-the-not-null-trap.md). + +[`ExpandMigrationBackwardCompatibleTest`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/ExpandMigrationBackwardCompatibleTest.java) +checks the two things Deploy 1 promises: every pre-existing row gets backfilled, and +Stage 1's original `INSERT` — unmodified, unaware `email_address` exists — still +works after the migration runs: + +``` +-- schema before Deploy 1 -- +[COLUMN_NAME=ID, ...][COLUMN_NAME=NAME, ...][COLUMN_NAME=EMAIL, ...][COLUMN_NAME=CREATED_AT, ...] + +-- schema after Deploy 1 (email_address added) -- +[COLUMN_NAME=ID, ...][COLUMN_NAME=NAME, ...][COLUMN_NAME=EMAIL, ...][COLUMN_NAME=CREATED_AT, ...][COLUMN_NAME=EMAIL_ADDRESS, ...] + +-- Ada's row was backfilled by the migration itself -- +[NAME=Ada Lovelace, EMAIL=ada@example.test, EMAIL_ADDRESS=ada@example.test] + +-- Stage 1's original INSERT still works, unmodified, after the migration -- +[NAME=Grace Hopper, EMAIL=grace@example.test, EMAIL_ADDRESS=NULL] +``` + +Full transcript: +[`docs/output/02-expand-backward-compatible.txt`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/docs/output/02-expand-backward-compatible.txt). + +Grace Hopper's row is the important one: `email_address` is `NULL` for it, because +Stage 1 never wrote to it, and that's *correct* — Deploy 1 hasn't shipped any code +that would. That gap is exactly what Deploy 2 exists to close, and it reopens itself +on a smaller scale during Deploy 2's own rollout — see +[chapter 7](07-the-backfill-window-bug.md). + +## Going deeper + +- The migration runs against the live database with no application restart — + [chapter 3](03-why-migrations-run-outside-the-app.md) covers how and why that's a + separate mechanism from the app's own deploy. +- H2's specific behavior for `ALTER TABLE ADD COLUMN` under concurrent traffic — + not just "is it additive" but "is it safe to run while inserts are in flight" — is + covered in [chapter 14](14-the-ddl-lock-window.md), and it's the most surprising + finding in this whole module. + +[← 1. The problem and the plan](01-the-problem-and-the-plan.md) · [Next: 3. Why migrations run outside the app →](03-why-migrations-run-outside-the-app.md) diff --git a/db-migrations-expand-contract/docs/03-why-migrations-run-outside-the-app.md b/db-migrations-expand-contract/docs/03-why-migrations-run-outside-the-app.md new file mode 100644 index 0000000..8c8855b --- /dev/null +++ b/db-migrations-expand-contract/docs/03-why-migrations-run-outside-the-app.md @@ -0,0 +1,69 @@ +# 3. Why migrations run outside the app + +[← 2. The expand migration](02-the-expand-migration.md) · [Next: 4. The dual write →](04-the-dual-write.md) + +Every other module in this repository lets Spring Boot run Flyway on startup — +`spring.flyway.enabled=true`, migrate-on-boot, the default most tutorials show. This +module turns that off: + +```yaml +flyway: + enabled: false +``` + +from +[`application.yml`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/main/resources/application.yml), +and instead ships a second, standalone entry point: +[`MigrationCli`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/migration/MigrationCli.java): + +```java +Flyway flyway = Flyway.configure() + .dataSource(url, "sa", "") + .locations("classpath:db/migration") + .target(target) + .load(); +flyway.migrate(); +``` + +invoked by [`scripts/migrate.sh`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/scripts/migrate.sh): + +```bash +java -cp "$MODULE_DIR/target/classes:$(cat "$CP_FILE")" \ + com.ankurm.expandcontract.migration.MigrationCli --target="$TARGET" +``` + +Why bother, when "migrate on boot" is one line of config? Because "migrate on boot" +quietly welds a schema change to an application restart, and expand-contract's whole +argument is that those two things need to be independently controllable events: + +- Deploy 1 (expand) and Deploy 4b (contract) run a migration with **zero** app + restarts — every replica keeps serving traffic on its current code the entire time + the `ALTER TABLE` executes. `scripts/run-all.sh` calls `migrate.sh` directly for + both of these, with no `stop-instance.sh` / `start-instance.sh` anywhere nearby. +- Deploys 2, 3, and 4a are **pure code deploys** — a rolling restart with `--target` + fixed at whatever the schema already is. No new SQL runs. + +If Flyway ran on every boot, a canary replica restarting for an unrelated reason (an +OOM, a node reschedule, a routine redeploy of a config value) would silently re-run +whatever migrations hadn't executed yet, at a moment nobody chose. Running Flyway from +its own process, invoked deliberately by the deploy pipeline (or by hand, as this +module's scripts do), means a schema change happens exactly once, at exactly the +moment someone decided it should — the same discipline a real CI/CD "run migrations" +job step gives you, kept intact here even though this whole sequence runs on one +sandbox. + +The `--target` flag is what lets `migrate.sh 2` mean "get the schema to exactly V2, +no further" — the same `spring.flyway.target` property +[`TestSupport.migrateTo`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/TestSupport.java) +uses to put a test database at "however far Deploy N has gotten" before exercising +`CustomerService` against it. + +## Going deeper + +- `MigrationCli` connects to the same standalone H2 TCP server the app replicas do — + [chapter 12](12-the-auto-server-trap.md) covers why that database is its own + process rather than something either replica owns. +- Flyway's own migrate-on-startup vs. separate-migration-step tradeoff is discussed in + [Flyway's documentation on migrations](https://documentation.red-gate.com/fd/migrations-184127470.html) (nofollow). + +[← 2. The expand migration](02-the-expand-migration.md) · [Next: 4. The dual write →](04-the-dual-write.md) diff --git a/db-migrations-expand-contract/docs/04-the-dual-write.md b/db-migrations-expand-contract/docs/04-the-dual-write.md new file mode 100644 index 0000000..249732b --- /dev/null +++ b/db-migrations-expand-contract/docs/04-the-dual-write.md @@ -0,0 +1,61 @@ +# 4. The dual write + +[← 3. Why migrations run outside the app](03-why-migrations-run-outside-the-app.md) · [Next: 5. The read switch →](05-the-read-switch.md) + +Deploy 2 is the first code deploy in the sequence, and the only thing it changes is +what `create()` and `updateEmail()` write. Stage 2's SQL in +[`CustomerService`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/customer/CustomerService.java) +writes to both columns: + +```java +case 2, 3 -> jdbc.sql("INSERT INTO customers(name, email, email_address) VALUES (?, ?, ?)") + .param(name).param(email).param(email) + .update(keyHolder, "id"); +``` + +```java +case 2, 3 -> jdbc.sql("UPDATE customers SET email = ?, email_address = ? WHERE id = ?") + .param(newEmail).param(newEmail).param(id).update(); +``` + +Stage 2 and Stage 3 share this write path — the only difference between them is what +they *read*, covered in [chapter 5](05-the-read-switch.md). That's deliberate: writes +have to stay dual for two whole deploys (2 and 3) so that by the time Deploy 4 +arrives, every row in the table — regardless of which stage wrote it last — is +guaranteed to have both columns populated identically. + +[`DualWriteConsistencyTest`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/DualWriteConsistencyTest.java) +confirms both halves of that promise — a fresh create lands in both columns, and an +update *replaces* the value in both, not just one: + +``` +-- after create() -- +NAME | EMAIL | EMAIL_ADDRESS +------------------+-----------------------+---------------------- +Margaret Hamilton | margaret@example.test | margaret@example.test + +-- after updateEmail() - the old value is gone from BOTH columns, not just one -- +NAME | EMAIL | EMAIL_ADDRESS +------------------+-------------------------+------------------------ +Margaret Hamilton | m.hamilton@example.test | m.hamilton@example.test +``` + +Full transcript: +[`docs/output/04-dual-write-consistency.txt`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/docs/output/04-dual-write-consistency.txt). + +The second half of that test matters more than it looks. A dual write that only +inserts into both columns but updates only one is a much more common bug than it +sounds — the update path is usually written later, by someone who's already stopped +thinking about `email_address` because the create path "already handles the new +column". + +## Going deeper + +- Deploy 2's own rollout window — where some replicas are still Stage 1 while others + are already Stage 2 — is exactly the gap [chapter 7](07-the-backfill-window-bug.md) + is about: what a lingering Stage 1 write during *this* rollout means for the read + switch that comes next. +- The full cross-stage read/write matrix, including this deploy's pair, is proven + directly in [chapter 8](08-the-rolling-window-proof.md). + +[← 3. Why migrations run outside the app](03-why-migrations-run-outside-the-app.md) · [Next: 5. The read switch →](05-the-read-switch.md) diff --git a/db-migrations-expand-contract/docs/05-the-read-switch.md b/db-migrations-expand-contract/docs/05-the-read-switch.md new file mode 100644 index 0000000..d0c820e --- /dev/null +++ b/db-migrations-expand-contract/docs/05-the-read-switch.md @@ -0,0 +1,41 @@ +# 5. The read switch + +[← 4. The dual write](04-the-dual-write.md) · [Next: 6. The NOT NULL trap →](06-the-not-null-trap.md) + +Deploy 3 changes exactly one thing from Deploy 2: where reads come from. In +[`CustomerService.findById`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/customer/CustomerService.java): + +```java +String sql = switch (stage) { + case 1, 2 -> "SELECT id, name, email AS email FROM customers WHERE id = ?"; + case 3 -> "SELECT id, name, COALESCE(email_address, email) AS email FROM customers WHERE id = ?"; + case 4 -> "SELECT id, name, email_address AS email FROM customers WHERE id = ?"; + default -> throw new IllegalStateException(); +}; +``` + +Stage 1 and Stage 2 read the same column: only *writes* changed between them. Stage 3 +is the read switch, and it's a separate deploy from Stage 2 for a reason that isn't +obvious until you say it out loud: **the write switch and the read switch cannot be +the same deploy**, because a rolling deploy is not instantaneous. Deploy 2's own +rollout has a window — anywhere from seconds to minutes, depending on fleet size — +where some replicas are still Stage 1, writing only `email`. If Deploy 3's read +switch were bundled into that same deploy, a Stage 3 replica reading +`email_address` alone would return `NULL` for every row a still-live Stage 1 +replica had just written. That's not a hypothetical: it's +[chapter 7](07-the-backfill-window-bug.md), reproduced and fixed. + +The fix already visible above is `COALESCE(email_address, email)` rather than +`email_address` alone — Stage 3 falls back to the old column for exactly the rows +that predate full dual-write coverage. Stage 4 doesn't need the fallback anymore: by +the time Deploy 4 starts, two full deploys' worth of dual-writing (Stage 2 and Stage +3, run back to back) have guaranteed every row has both columns populated. + +## Going deeper + +- The bug this switch would reintroduce if it were merged into Deploy 2, captured + live: [chapter 7](07-the-backfill-window-bug.md). +- All six write/read direction pairs across the three rollouts, proven directly: + [chapter 8](08-the-rolling-window-proof.md). + +[← 4. The dual write](04-the-dual-write.md) · [Next: 6. The NOT NULL trap →](06-the-not-null-trap.md) diff --git a/db-migrations-expand-contract/docs/06-the-not-null-trap.md b/db-migrations-expand-contract/docs/06-the-not-null-trap.md new file mode 100644 index 0000000..d6ed782 --- /dev/null +++ b/db-migrations-expand-contract/docs/06-the-not-null-trap.md @@ -0,0 +1,66 @@ +# 6. The NOT NULL trap + +[← 5. The read switch](05-the-read-switch.md) · [Next: 7. The backfill window bug →](07-the-backfill-window-bug.md) + +This module's own first draft of the expand migration shipped without one line, and +the bug it produced is worth showing exactly as it happened, because "add a nullable +column" reads like the entire expand step and it isn't. + +`customers.email` was declared `NOT NULL` back in +[`V1__create_customer.sql`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/main/resources/db/migration/V1__create_customer.sql). +Deploy 1 adds `email_address`, nullable — that part is fine. But Stage 4's `create()` +never writes `email` at all: + +```java +case 4 -> jdbc.sql("INSERT INTO customers(name, email_address) VALUES (?, ?)") + .param(name).param(email) + .update(keyHolder, "id"); +``` + +If `email` is still mandatory when Stage 4 code starts running, that `INSERT` omits a +`NOT NULL` column with no default. Every single create fails, in production, from the +first request the first Stage 4 replica handles. + +[`NotNullConstraintTrapTest`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/NotNullConstraintTrapTest.java) +reproduces this against the naive migration (expand plus backfill, nothing else) and +then shows the one-line fix working: + +``` +-- Stage 4 create() against the NAIVE migration (no DROP NOT NULL) -- +org.springframework.dao.DataIntegrityViolationException: PreparedStatementCallback; SQL [INSERT INTO customers(name, email_address) VALUES (?, ?)]; NULL not allowed for column "EMAIL"; SQL statement: +INSERT INTO customers(name, email_address) VALUES (?, ?) [23502-240] +root cause: org.h2.jdbc.JdbcSQLIntegrityConstraintViolationException: NULL not allowed for column "EMAIL"; SQL statement: +INSERT INTO customers(name, email_address) VALUES (?, ?) [23502-240] + +-- Stage 4 create() against the SHIPPED V2 migration (DROP NOT NULL included) -- +Customer[id=1, name=On Time, email=on.time@example.test] +``` + +Full transcript: +[`docs/output/06-not-null-trap.txt`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/docs/output/06-not-null-trap.txt). + +The fix is the third statement in +[`V2__add_email_address_column.sql`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/main/resources/db/migration/V2__add_email_address_column.sql): + +```sql +ALTER TABLE customers ALTER COLUMN email DROP NOT NULL; +``` + +It belongs in the *same* migration as the add — Deploy 1 — not a later one. Stage 1 +and Stage 2 code both still write `email` on every insert, so relaxing its +constraint changes nothing observable for them. But by the time Stage 4 ships, the +constraint has to already be gone, and Stage 4 doesn't run a migration of its own — +Deploy 4a is a pure code deploy (see [chapter 9](09-the-contract-migration.md)). +Retrofitting the `DROP NOT NULL` later means adding a second schema change in the +middle of what was supposed to be a code-only step. + +## Going deeper + +- Every column being retired in an expand-contract migration is worth auditing for + `NOT NULL`, `UNIQUE`, and foreign-key constraints the *new* write path won't + satisfy — this module only had one such constraint, but a wider table can have + several. +- H2's constraint-violation exception hierarchy: + [`JdbcSQLIntegrityConstraintViolationException`](https://www.h2database.com/javadoc/org/h2/api/ErrorCode.html) (nofollow). + +[← 5. The read switch](05-the-read-switch.md) · [Next: 7. The backfill window bug →](07-the-backfill-window-bug.md) diff --git a/db-migrations-expand-contract/docs/07-the-backfill-window-bug.md b/db-migrations-expand-contract/docs/07-the-backfill-window-bug.md new file mode 100644 index 0000000..6f3d0a9 --- /dev/null +++ b/db-migrations-expand-contract/docs/07-the-backfill-window-bug.md @@ -0,0 +1,55 @@ +# 7. The backfill window bug + +[← 6. The NOT NULL trap](06-the-not-null-trap.md) · [Next: 8. The rolling window proof →](08-the-rolling-window-proof.md) + +Deploy 1's migration backfills `email_address` for every row that exists *at the +moment it runs*. It cannot see rows written after that — and a rolling deploy is not +instantaneous, so there's a real window, between "Deploy 1's migration finished" and +"every replica in the fleet is confirmed Stage 2 or later", during which a still-live +Stage 1 instance keeps inserting rows the old way: `email` only, `email_address` +untouched, `NULL`. + +That window isn't a hypothetical edge case — it's guaranteed to happen for however +long Deploy 2's rollout takes, on every real fleet bigger than one instance. + +[`BackfillWindowBugTest`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/BackfillWindowBugTest.java) +writes exactly such a row, then reads it back two ways: + +``` +-- the row a lingering Stage 1 instance just wrote -- +[NAME=Katherine Johnson, EMAIL=katherine@example.test, EMAIL_ADDRESS=null] + +-- a NAIVE Stage 3 read (email_address alone) - the bug -- +naive Stage 3 email column value: null + +-- the SHIPPED Stage 3 read (CustomerService, COALESCE) - the fix -- +CustomerService (stage 3) result: Optional[Customer[id=1, name=Katherine Johnson, email=katherine@example.test]] +``` + +Full transcript: +[`docs/output/07-backfill-window-bug.txt`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/docs/output/07-backfill-window-bug.txt). + +The naive read — `SELECT email_address FROM customers WHERE id = ?` — is exactly what +you'd write if you thought of the read switch as "just point at the new column now +that it's backfilled". It's wrong specifically for rows born during Deploy 2's own +rollout, because those rows are younger than Deploy 1's one-time backfill and were +written by an instance that (correctly, for its stage) never touched +`email_address`. + +The fix is the `COALESCE(email_address, email)` already shown in +[chapter 5](05-the-read-switch.md). It costs nothing for a fully dual-written row — +`COALESCE` returns the first non-null argument, and both columns already agree — and +it's the only thing that returns a correct answer for a row that hasn't caught up +yet. + +## Going deeper + +- Stage 2 never needs this fallback: it still reads the original `email` column, + which every stage keeps populated the entire time. Only Stage 3, the first stage to + prefer the new column, needs the fallback — see the read switch in + [chapter 5](05-the-read-switch.md). +- This is the database-level version of a general rolling-deploy rule: **any code + path that reads data written by a different stage has to tolerate that stage's + write shape**, not just the shape your own stage would have produced. + +[← 6. The NOT NULL trap](06-the-not-null-trap.md) · [Next: 8. The rolling window proof →](08-the-rolling-window-proof.md) diff --git a/db-migrations-expand-contract/docs/08-the-rolling-window-proof.md b/db-migrations-expand-contract/docs/08-the-rolling-window-proof.md new file mode 100644 index 0000000..b1ddbf5 --- /dev/null +++ b/db-migrations-expand-contract/docs/08-the-rolling-window-proof.md @@ -0,0 +1,78 @@ +# 8. The rolling-window proof + +[← 7. The backfill window bug](07-the-backfill-window-bug.md) · [Next: 9. The contract migration →](09-the-contract-migration.md) + +Every rolling deploy in this sequence — Deploy 2, Deploy 3, Deploy 4a — has a window +where two adjacent stages are serving real traffic against the same database at the +same time. [`MixedStageRollingWindowTest`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/MixedStageRollingWindowTest.java) +is the property this whole article rests on, checked directly: it builds two +`CustomerService` instances on adjacent stages sharing one database, and cross-checks +every write/read direction across all three rollouts. + +``` +-- Stage 1 writes, Stage 2 reads -- +Customer[id=1, name=Radia Perlman, email=radia@example.test] + +-- Stage 2 writes, Stage 1 reads -- +Customer[id=2, name=Barbara Liskov, email=barbara@example.test] + +-- Stage 2 writes, Stage 3 reads -- +Customer[id=3, name=Shafi Goldwasser, email=shafi@example.test] + +-- Stage 3 writes, Stage 2 reads -- +Customer[id=4, name=Frances Allen, email=frances@example.test] + +-- Stage 3 writes, Stage 4 reads -- +Customer[id=5, name=Adele Goldberg, email=adele@example.test] + +-- Stage 4 writes, Stage 3 reads -- +Customer[id=6, name=Karen Sparck Jones, email=karen@example.test] +``` + +Full transcript: +[`docs/output/08-mixed-stage-rolling-window.txt`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/docs/output/08-mixed-stage-rolling-window.txt). + +Six pairs, six passing reads. If any one of them failed, the technique would not be +zero-downtime for that rollout — it would just be a race against however long the +rollout takes to finish, with correctness depending on luck rather than design. + + + +Deploy 2 rollout +Stage 1 (draining) +Stage 2 (arriving) +← overlap: both true, both correct + +Deploy 3 rollout +Stage 2 (draining) +Stage 3 (arriving) +← overlap: both true, both correct + +Deploy 4a rollout +Stage 3 (draining) +Stage 4 (arriving) +← overlap: both true, both correct + + +The diagram is the same shape three times because the guarantee is the same three +times: whichever two stages are live together during a given rollout, a write from +either one has to be readable correctly by the other. That's what the test above +checks directly, and it's what the article's live 4-deploy run — a real load +generator, hitting real HTTP endpoints, during a real rolling restart — is +reproducing under actual timing pressure rather than a unit test's controlled +ordering. + +## Going deeper + +- The live version of this proof, with two real replicas and continuous HTTP + traffic: [chapter 11](11-the-load-generator.md) and the full run in + [`docs/output/11-live-deploy-sequence.txt`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/docs/output/11-live-deploy-sequence.txt). + +[← 7. The backfill window bug](07-the-backfill-window-bug.md) · [Next: 9. The contract migration →](09-the-contract-migration.md) diff --git a/db-migrations-expand-contract/docs/09-the-contract-migration.md b/db-migrations-expand-contract/docs/09-the-contract-migration.md new file mode 100644 index 0000000..9cffb0b --- /dev/null +++ b/db-migrations-expand-contract/docs/09-the-contract-migration.md @@ -0,0 +1,47 @@ +# 9. The contract migration + +[← 8. The rolling window proof](08-the-rolling-window-proof.md) · [Next: 10. What happens if you drop too soon →](10-what-happens-if-you-drop-too-soon.md) + +Deploy 4 is split into two parts on purpose, the same way Deploy 1 was schema-only +and Deploys 2/3 were code-only: + +- **4a — code.** A rolling restart to Stage 4, which never reads or writes `email`. + The old column is still physically present; Stage 4 code simply ignores it. +- **4b — schema.** Once every replica is confirmed on Stage 4 code, and only then, + [`V3__drop_email_column.sql`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/main/resources/db/migration/V3__drop_email_column.sql) + runs, with zero app restarts: + +```sql +ALTER TABLE customers DROP COLUMN email; +``` + +[`ContractSafetyTest`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/ContractSafetyTest.java) +confirms the happy path: Stage 4 reads and writes keep working, both for a row that +existed before the drop and for one created entirely after it — + +``` +-- Stage 4 read, after V3 dropped the email column -- +Customer[id=1, name=Annie Easley, email=annie@example.test] + +-- Stage 4 create + read, entirely after the drop -- +Customer[id=2, name=Mary Allen Wilkes, email=mary@example.test] +``` + +Full transcript: +[`docs/output/09-contract-safety.txt`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/docs/output/09-contract-safety.txt). + +Why not drop the column in the same deploy as the code change? Because "every +replica is confirmed on Stage 4" is a statement about the *fleet*, and a rolling +deploy makes it true only once, at the very end of the rollout — never at the moment +the deploy starts. Running 4b before that point is dropping a column a live Stage 1, +2, or 3 replica might still need. What that actually looks like, captured directly, +is [chapter 10](10-what-happens-if-you-drop-too-soon.md). + +## Going deeper + +- The DDL Deploy 4b runs is not, on this specific database engine, quite as free of + side effects on concurrent traffic as "just a schema change" suggests — see + [chapter 14](14-the-ddl-lock-window.md) for what H2 actually does while this + statement executes. + +[← 8. The rolling window proof](08-the-rolling-window-proof.md) · [Next: 10. What happens if you drop too soon →](10-what-happens-if-you-drop-too-soon.md) diff --git a/db-migrations-expand-contract/docs/10-what-happens-if-you-drop-too-soon.md b/db-migrations-expand-contract/docs/10-what-happens-if-you-drop-too-soon.md new file mode 100644 index 0000000..e6f0b27 --- /dev/null +++ b/db-migrations-expand-contract/docs/10-what-happens-if-you-drop-too-soon.md @@ -0,0 +1,42 @@ +# 10. What happens if you drop too soon + +[← 9. The contract migration](09-the-contract-migration.md) · [Next: 11. The load generator →](11-the-load-generator.md) + +The second half of +[`ContractSafetyTest`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/ContractSafetyTest.java) +runs Deploy 4b's migration first, and only then starts a Stage 1 instance against the +result — standing in for a canary that never got promoted, a rollback that didn't +fully take, or simply running the drop before confirming the fleet: + +``` +-- What a lingering Stage 1 instance sees if the drop runs before it is retired -- +Stage 1 create() after V3 dropped "email": org.springframework.jdbc.BadSqlGrammarException +message: PreparedStatementCallback; bad SQL grammar [INSERT INTO customers(name, email) VALUES (?, ?)] +root cause: org.h2.jdbc.JdbcSQLSyntaxErrorException: Column "EMAIL" not found; SQL statement: +INSERT INTO customers(name, email) VALUES (?, ?) [42122-240] +``` + +Full transcript: +[`docs/output/10-drop-too-soon.txt`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/docs/output/10-drop-too-soon.txt). + +This is the correct outcome, not a bug to work around. A Stage 1 instance still +running after the drop is itself the actual mistake — a deploy that didn't finish, or +a rollback nobody noticed failed — and the database telling it loudly and immediately +that `email` doesn't exist is far better than the alternative of silently accepting +partial writes or, worse, dropping rows. Expand-contract's safety comes from **when** +you're allowed to run Deploy 4b (only after confirming 100% Stage 4), not from Deploy +4b itself being forgiving of running early. + +In a real deploy pipeline, this is the argument for gating Deploy 4b on an explicit +health/version check across the fleet — every instance's `/actuator/info` or +equivalent reporting Stage 4 — rather than a fixed timer. "Deploy 2 usually finishes +rolling out in three minutes" is not the same guarantee as "every replica confirmed +Stage 4", and the difference between them is exactly the window this chapter's test +is exploiting. + +## Going deeper + +- [Chapter 15](15-production-checklist.md) turns this into an actual gate: what to + check, and where, before running a contract migration in a real pipeline. + +[← 9. The contract migration](09-the-contract-migration.md) · [Next: 11. The load generator →](11-the-load-generator.md) diff --git a/db-migrations-expand-contract/docs/11-the-load-generator.md b/db-migrations-expand-contract/docs/11-the-load-generator.md new file mode 100644 index 0000000..368e571 --- /dev/null +++ b/db-migrations-expand-contract/docs/11-the-load-generator.md @@ -0,0 +1,124 @@ +# 11. The load generator + +[← 10. What happens if you drop too soon](10-what-happens-if-you-drop-too-soon.md) · [Next: 12. The AUTO_SERVER trap →](12-the-auto-server-trap.md) + +Every other chapter so far proves a property of the technique with a JUnit test +against a shared `JdbcClient` — real code, real SQL, no mocks, but also no real HTTP, +no real process restarts, no real timing pressure. `scripts/run-all.sh` is where all +three of those show up: two real Spring Boot processes, a real standalone database, a +real rolling restart between each stage, and +[`LoadGenerator`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/loadgen/LoadGenerator.java) +sending continuous HTTP traffic through the whole sequence. + +## What it does + +Eight worker threads, each running the same loop: pick a healthy backend, then create +a customer (50% of the time), read one it already knows about (30%), or update one +(20%). Every read and update verifies the response body matches what the load +generator itself expects — not just the HTTP status code: + +```java +JsonNode node = MAPPER.readTree(resp.body()); +String returnedEmail = node.get("email").asString(); +if (!target.email().equals(returnedEmail)) { + c.recordError("read-consistency-mismatch"); + return; +} +``` + +A 200 with the wrong email in the body would be a much worse bug than a 500, and a +naive load test that only checks status codes would never catch it. + +## Health-checked traffic, not a raw hose + +`LoadGenerator` polls `/actuator/health` on both ports every 300ms and only sends +traffic to backends it currently believes are up: + +```java +private static final int UNHEALTHY_THRESHOLD = 2; +``` + +A backend needs two consecutive failed checks before it's removed from rotation — +this exists because this whole sequence runs two JVMs on a shared, small sandbox, and +one replica's cold-start CPU burst can make its *sibling* miss a single health check +without actually being down. Requiring two consecutive failures is the same debounce +a real load balancer's health check threshold gives you, and skipping it turned +transient slowness into false `no-healthy-backend` errors in an earlier version of +this test. + +That fix mattered enough to also show up in +[`start-instance.sh`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/scripts/start-instance.sh), +on the other side of the same problem — cutting each replica's own startup CPU cost +so it's less likely to starve its sibling in the first place: + +```bash +nohup java -XX:TieredStopAtLevel=1 -XX:+UseSerialGC -Xms128m -Xmx256m \ + -jar "$JAR" --server.port="$PORT" --app.stage="$STAGE" \ +``` + +## Per-thread state, not shared state + +Each worker thread owns a private `ArrayDeque` of customer ids it has created — never +shared with the other seven threads: + +```java +workers.submit(() -> { + try { + workerLoop(new ArrayDeque<>()); +``` + +An earlier version shared one pool across all eight threads, and produced +`read-consistency-mismatch` errors that had nothing to do with the server at all: two +threads racing to update the *same* shared id could leave the pool holding a stale +expected value, so a perfectly correct server response looked like a bug. Giving each +thread exclusive ownership of the rows it creates removes that entire class of false +positive while still hammering both replicas concurrently. + +## The result + +``` +Load generator summary +======================= +Total requests: 30911 +Successful: 30905 +Errors: 6 + +By phase: + 04a-stage4-soak ok=2130 error=0 + 04b-contract-migration ok=336 error=0 + 04a-deploy-stage4-rollout ok=4147 error=0 + 03-stage3-soak ok=2116 error=0 + 03-deploy-stage3-rollout ok=4645 error=0 + 00-baseline-soak ok=1893 error=0 + 05-final-soak ok=7118 error=4 + - read-http-404 2 + - update-http-404 2 + 02-stage2-soak ok=2068 error=0 + 01-expand-migration ok=1773 error=1 + - read-http-404 1 + 02-deploy-stage2-rollout ok=4679 error=1 + - update-http-404 1 +``` + +Full transcript: +[`docs/output/12-load-generator-summary.txt`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/docs/output/12-load-generator-summary.txt), +and the phase-by-phase deploy log this run came from: +[`docs/output/11-live-deploy-sequence.txt`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/docs/output/11-live-deploy-sequence.txt). + +30,905 of 30,911 requests succeeded — 99.98%, across a real rolling restart through +all four deploys. The six errors are all `404`s, not `500`s: a request for a customer +id that genuinely wasn't found, not a crash. Every one of them traces to the same +root cause, and it's the most interesting finding in this whole module — see +[chapter 14](14-the-ddl-lock-window.md). + +## Going deeper + +- The graceful-shutdown-and-drain sequence that gets the *rolling restart* portion of + this run down to zero client-visible errors on its own is + [chapter 13](13-graceful-shutdown-vs-kill-9.md) — the six remaining errors above + have a different cause entirely, isolated in chapter 14. +- Every deploy's schema state during this exact run, captured live: + [`docs/output/13-schema-diagnostics-timeline.txt`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/docs/output/13-schema-diagnostics-timeline.txt), + via [`SchemaDiagnosticsController`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/diag/SchemaDiagnosticsController.java). + +[← 10. What happens if you drop too soon](10-what-happens-if-you-drop-too-soon.md) · [Next: 12. The AUTO_SERVER trap →](12-the-auto-server-trap.md) diff --git a/db-migrations-expand-contract/docs/12-the-auto-server-trap.md b/db-migrations-expand-contract/docs/12-the-auto-server-trap.md new file mode 100644 index 0000000..3db8707 --- /dev/null +++ b/db-migrations-expand-contract/docs/12-the-auto-server-trap.md @@ -0,0 +1,65 @@ +# 12. The AUTO_SERVER trap + +[← 11. The load generator](11-the-load-generator.md) · [Next: 13. Graceful shutdown vs. kill -9 →](13-graceful-shutdown-vs-kill-9.md) + +This module's first working draft pointed both replicas at the same H2 file with +`AUTO_SERVER=TRUE` — the mode most H2 tutorials show for "let two JVMs share one +embedded database file without a separate server process": + +``` +jdbc:h2:file:./data/expand-contract;AUTO_SERVER=TRUE +``` + +It works, right up until a rolling deploy restarts the *particular* replica that +happened to open the file first. `AUTO_SERVER` makes the first connecting process the +de facto database server for every other connection — internally, later connections +become `SessionRemote` clients of that first process, not independent embedded +sessions. Kill that one process — which a rolling deploy does, routinely, by design — +and every *other* replica's connection to the "embedded" database breaks with it. In +this module's own early runs, that showed up as `Table "CUSTOMERS" not found` on the +surviving replica, immediately after the first replica restarted, for a table that +had existed the entire time. It was single-point-of-failure architecture disguised as +an embedded database, and it was the single biggest source of load-generator errors +before it was found and fixed — cutting the error rate by roughly 90% on its own. + +The fix, in +[`start-db-server.sh`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/scripts/start-db-server.sh), +is to stop pretending the database is embedded at all and run it as what it actually +needs to be: its own standalone process that neither replica owns and neither +replica's lifecycle affects. + +```bash +nohup java -cp "$(cat "$CP_FILE")" org.h2.tools.Server \ + -tcp -tcpPort "$EC_DB_TCP_PORT" -baseDir "$EC_DB_BASE_DIR" -ifNotExists \ + > "$EC_LOG_DIR/db-server.log" 2>&1 < /dev/null & +``` + +Both replicas — and +[`MigrationCli`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/migration/MigrationCli.java) +— connect to it the same way, as ordinary TCP clients: + +```yaml +url: jdbc:h2:tcp://localhost:${EC_DB_TCP_PORT:9092}/${EC_DB_NAME:expand-contract} +``` + +`scripts/run-all.sh` starts this server first, before either replica, and stops it +last, after both — the one process in the entire sequence that is never restarted, +because it's standing in for what a real production database always is: a process +that outlives every deploy of every application that talks to it. + +One flag worth calling out because it looks interchangeable and isn't: +`-tcpDaemon` marks the server thread as a daemon thread, which is for embedding an H2 +server *inside* another long-running JVM that manages its own lifecycle — for a +standalone, always-on server process like this one, it made the process exit +immediately in a manual test, because there was no non-daemon thread left to keep the +JVM alive. Leave it off for a server meant to run on its own. + +## Going deeper + +- H2's own documentation on + [automatic mixed mode](https://www.h2database.com/html/features.html#auto_mixed_mode) + (nofollow) describes `AUTO_SERVER` for the single-application-process, + multiple-connections case it's actually designed for — not for two independent + application processes that need to survive each other's restarts. + +[← 11. The load generator](11-the-load-generator.md) · [Next: 13. Graceful shutdown vs. kill -9 →](13-graceful-shutdown-vs-kill-9.md) diff --git a/db-migrations-expand-contract/docs/13-graceful-shutdown-vs-kill-9.md b/db-migrations-expand-contract/docs/13-graceful-shutdown-vs-kill-9.md new file mode 100644 index 0000000..b2c4116 --- /dev/null +++ b/db-migrations-expand-contract/docs/13-graceful-shutdown-vs-kill-9.md @@ -0,0 +1,77 @@ +# 13. Graceful shutdown vs. kill -9 + +[← 12. The AUTO_SERVER trap](12-the-auto-server-trap.md) · [Next: 14. The DDL lock window →](14-the-ddl-lock-window.md) + +`application.yml` sets one line that does nothing by itself: + +```yaml +server: + shutdown: graceful +``` + +`server.shutdown: graceful` only changes behavior on `SIGTERM` — it stops accepting +new connections but lets in-flight requests finish before the process exits. An +earlier version of +[`stop-instance.sh`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/scripts/stop-instance.sh) +used `kill -9`, which bypasses graceful shutdown entirely — the process disappears +mid-request, and every request in flight at that instant surfaces in the load +generator as a raw `ConnectException` or `IOException`. Switching to `SIGTERM` +(`kill -15`), with a bounded wait for a clean exit and `SIGKILL` only as a fallback, +is the first half of the fix: + +```bash +kill -15 "$PID" +for i in $(seq 1 40); do + kill -0 "$PID" 2>/dev/null || break + sleep 0.25 +done +if kill -0 "$PID" 2>/dev/null; then + kill -9 "$PID" +fi +``` + +That alone wasn't enough. `server.shutdown: graceful` starts refusing new connections +the instant `SIGTERM` arrives — but the load generator's health checker polls every +300ms, and the pool didn't yet know to stop routing traffic there. The gap between +"the process just stopped accepting connections" and "the load balancer's health +check has noticed and rerouted" is exactly where `ConnectException` bursts kept +showing up, even with `SIGTERM` in place. + +The second half of the fix is a way to say "stop sending me traffic" *before* the +process is touched at all: +[`DrainController`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/diag/DrainController.java): + +```java +@PostMapping("/admin/drain") +public String drain() { + AvailabilityChangeEvent.publish(events, this, ReadinessState.REFUSING_TRAFFIC); + return "draining"; +} +``` + +Publishing `ReadinessState.REFUSING_TRAFFIC` flips `/actuator/health`'s readiness +group immediately — this is the same event a Kubernetes-style `preStop` hook +publishes before the container is sent `SIGTERM`. `stop-instance.sh` calls it, sleeps, +*then* sends `SIGTERM`: + +```bash +curl -s -X POST "http://localhost:$PORT/admin/drain" -o /dev/null || true +sleep 1.5 +kill -15 "$PID" +``` + +That 1.5-second pause is deliberate slack for the health checker's 300ms poll +interval — enough for at least a couple of checks to land and pull this instance out +of rotation before it's asked to stop at all. Together, drain-then-SIGTERM is what +took the rolling-restart portion of the article's live run to zero +`ConnectException`/`IOException` errors — the six errors that remain in the final +summary are a completely different, database-level cause, covered next. + +## Going deeper + +- Spring's own `ReadinessState` and the Kubernetes probe pattern it mirrors: + [Spring Boot reference docs, Application Availability](https://docs.spring.io/spring-boot/reference/actuator/application-availability.html) (nofollow). +- `/admin/drain` is a diagnostic-grade endpoint with no auth — see + [chapter 15](15-production-checklist.md) for what to do with it before shipping. + +[← 12. The AUTO_SERVER trap](12-the-auto-server-trap.md) · [Next: 14. The DDL lock window →](14-the-ddl-lock-window.md) diff --git a/db-migrations-expand-contract/docs/14-the-ddl-lock-window.md b/db-migrations-expand-contract/docs/14-the-ddl-lock-window.md new file mode 100644 index 0000000..4a689c4 --- /dev/null +++ b/db-migrations-expand-contract/docs/14-the-ddl-lock-window.md @@ -0,0 +1,199 @@ +# 14. The DDL lock window + +[← 13. Graceful shutdown vs. kill -9](13-graceful-shutdown-vs-kill-9.md) · [Next: 15. Production checklist →](15-production-checklist.md) + +Every schema-only deploy in this sequence — Deploy 1's `ADD COLUMN` and Deploy 4b's +`DROP COLUMN` — runs against a live database while both replicas keep taking real +traffic. That's the entire point of running migrations outside the app (see +[chapter 3](03-why-migrations-run-outside-the-app.md)): zero app restarts for a +schema-only step. What this chapter covers is what "zero app restarts" does *not* +automatically buy you: zero effect on concurrent queries while the `ALTER TABLE` +statement itself is executing. + +There are two distinct failure modes here, found the same way — by watching the +article's own load generator run against a live rollout and refusing to wave away +the handful of errors it reported. + +## Failure mode 1: a statement that collides with the DDL, and says so + +While `V3__drop_email_column.sql` runs, a concurrent, otherwise-correct `INSERT` or +`UPDATE` can briefly see: + +``` +org.h2.jdbc.JdbcSQLSyntaxErrorException: Table "CUSTOMERS" not found +``` + +This is a real, transient condition captured live, twice, in independent runs of this +module's load generator — not a bug in the application's SQL. H2's TCP server +appears to make the table briefly unavailable to other sessions while `DROP COLUMN` +executes. The fix is a narrowly scoped single retry in +[`CustomerService.withRetryForConcurrentDdl`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/customer/CustomerService.java): + +```java +private T withRetryForConcurrentDdl(Supplier operation) { + try { + return operation.get(); + } catch (BadSqlGrammarException ex) { + if (ex.getMessage() != null && ex.getMessage().contains("CUSTOMERS")) { + try { + Thread.sleep(50); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + } + return operation.get(); + } + throw ex; + } +} +``` + +It catches specifically the "table not found" grammar error, sleeps 50ms — long +enough for the in-flight `ALTER TABLE` to finish — and retries exactly once. It is +deliberately narrow: anything else still propagates. Retrying every `BadSqlGrammarException` +indiscriminately would mask genuine programming errors (a typo'd column name, for +instance) as if they were transient — this retry only fires for the one specific, +verified condition. `create()`, `updateEmail()`, and `findById()` are all wrapped in +it, because a concurrent read can hit the exact same window a concurrent write can. +After adding this retry, a full re-run of the live sequence produced **zero** +`create-http-500` / `update-http-500` errors during Deploy 4b — the class of error +this fix targets is fully eliminated. Compare +[`docs/output/12-load-generator-summary.txt`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/docs/output/12-load-generator-summary.txt), +where the `04b-contract-migration` phase shows `ok=336 error=0`. + +## Failure mode 2: a statement that succeeds, and is still lost + +The retry above only helps when the colliding statement *throws*. It does nothing for +the six residual `404`s that remained after the fix — a customer id that a `201` +response had already confirmed existed, later reported not found by a plain read or +update, with no exception anywhere in the logs. That gap between "the fix that +worked" and "the errors that didn't go away" is what led to the actual root cause. + +[`DdlSilentDataLossTest`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/DdlSilentDataLossTest.java) +reproduces it directly: one thread inserting customers continuously through +`CustomerService.create()`, while `V2` (`ADD COLUMN`) runs concurrently on another +connection. + +```java +Thread inserter = new Thread(() -> { + while (!stop.get()) { + long id = stage1.create("Concurrent " + n, "concurrent" + n + "@example.test"); + confirmedIds.add(id); // no exception - the insert reported success + } +}); +inserter.start(); +TestSupport.migrateTo(db, "2"); // ADD COLUMN, running concurrently +``` + +Exactly when the rebuild's internal scan passes a given row relative to that row's +own commit is OS thread scheduling, not application logic — a single attempt can +land on either side of the race. Rather than assert on one attempt (which failed to +reproduce roughly one run in five while writing this test) or weaken the assertion to +"zero or more" — which would silently stop proving anything the day this stops +reproducing — the test repeats the race on a fresh table until it reproduces, up to +20 times, the same thing a human would do at a terminal to confirm a suspected race +is real: + +``` +attempts needed to reproduce the race: 2 of 20 +customer creates that returned a generated id with no error: 50 +customer creates that got the already-documented, already-fixed DDL-collision error: 1 +of the ids that came back with no error, missing from the table once V2 finished: 8 +example missing ids: [41, 42, 43, 44, 45] + +This is why the retry in CustomerService cannot be the whole fix: these inserts +never threw anything to retry. The row was committed, then discarded when the +ADD COLUMN rebuild swapped in a new table that had already been scanned. +``` + +Full transcript: +[`docs/output/14-ddl-silent-data-loss.txt`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/docs/output/14-ddl-silent-data-loss.txt). + +Under a tight, unthrottled loop with no delay between inserts, this run lost 8 of the +50 rows inserted during the migration window — other runs made while writing this +test lost anywhere from 1 to a few dozen, depending on exactly how the two threads +got scheduled. That number is not a claim about the live article run's own rate — the +load generator's eight threads sleep 15–40ms between requests and aren't hammering +the exact millisecond the migration executes, so far fewer of its requests land in +the vulnerable window. It's a claim about the *mechanism*: H2 implements both +`ALTER TABLE ... ADD COLUMN` and +`ALTER TABLE ... DROP COLUMN` by rebuilding the table — copying every row into a new +table with the new column layout and swapping it in. A row inserted on another +connection can commit while that rebuild is mid-scan; depending on exactly when the +scan reaches the row relative to the commit, the row ends up copied into the new +table or left behind in the old one. When it's left behind, it disappears the instant +the rebuild finishes, and the connection that inserted it was never told anything +went wrong — the `INSERT` had already returned successfully. + +This is why `withRetryForConcurrentDdl` cannot be "the fix" for the residual errors: +there is nothing to retry. The failure isn't a rejected statement; it's data that +existed for a moment and then didn't, discovered only by a later, unrelated read. + + + + +1. ALTER TABLE begins the rebuild: scans OLD, copies rows into NEW + +OLD table +row 1 +row 2 + + +NEW table (new columns) +row 1 +row 2 + +2. A concurrent INSERT commits into OLD after the scan already passed that point + +row 3 (new!) + +(row 3 never scanned) + +3. Rebuild finishes, NEW swaps in for OLD - row 3 is gone, with no error to anyone + + +The diagram's third step is the whole finding in one line: nothing in this sequence +is a bug in the *application's* SQL, the migration's SQL, or the expand-contract +technique — it's a property of how this specific embedded database implements two +DDL statements that a lot of guidance describes as "safe" without qualification. + +## This is a property of H2, not of expand-contract + +PostgreSQL's own reference manual is explicit that this isn't universal: + +> When a column is added with `ADD COLUMN` and a non-volatile `DEFAULT` is specified +> [or none is], ... In neither case is a rewrite of the table required. +> +> The `DROP COLUMN` form does not physically remove the column, but simply makes it +> invisible to SQL operations. + +Both operations are metadata-only in Postgres for the shapes used here — no table +rewrite, and therefore no window where a concurrently committed row can be scanned +past. (Postgres still takes a brief `ACCESS EXCLUSIVE` lock to make the metadata +change, which blocks concurrent statements for that short duration rather than racing +past them — a different, more familiar tradeoff than silent loss.) A real migration +of a real production table should treat this as a question to answer about *your* +database, not assume either behavior: does `ALTER TABLE ADD/DROP COLUMN` rewrite the +table on your engine, and if it does, what does that engine guarantee about +concurrent writes during the rewrite? For MySQL/InnoDB, that answer depends on the +specific `ALGORITHM` the storage engine picks for the given change — `INSTANT` and +`INPLACE` avoid a full rebuild, `COPY` does not. + +## Going deeper + +- [Chapter 12](12-the-auto-server-trap.md) covers a different H2-specific surprise + found the same way — running as a "shared embedded" database instead of a real + standalone server. +- PostgreSQL's [`ALTER TABLE` reference](https://www.postgresql.org/docs/current/sql-altertable.html) (nofollow) + is the primary source for the Postgres claims above. +- MySQL's [Online DDL documentation](https://dev.mysql.com/doc/refman/8.4/en/innodb-online-ddl.html) (nofollow) + covers which `ALTER TABLE` operations get `INSTANT`/`INPLACE` treatment on InnoDB. + +[← 13. Graceful shutdown vs. kill -9](13-graceful-shutdown-vs-kill-9.md) · [Next: 15. Production checklist →](15-production-checklist.md) diff --git a/db-migrations-expand-contract/docs/15-production-checklist.md b/db-migrations-expand-contract/docs/15-production-checklist.md new file mode 100644 index 0000000..6745803 --- /dev/null +++ b/db-migrations-expand-contract/docs/15-production-checklist.md @@ -0,0 +1,59 @@ +# 15. Production checklist + +[← 14. The DDL lock window](14-the-ddl-lock-window.md) + +Everything in this module is a demonstration, deliberately built so its failure modes +are reachable and its output is captured. Taking the technique — not this exact +code — to a real service means addressing what this module intentionally left +uncovered: + +- **Gate the contract migration on confirmed fleet state, not a timer.** [Chapter + 10](10-what-happens-if-you-drop-too-soon.md) shows what a lingering old-stage + instance sees the moment the drop runs — a real deploy pipeline should check every + instance's reported version/health before running Deploy 4b, not just wait "long + enough". +- **Know your database's `ALTER TABLE` semantics before you run this against a real + table.** [Chapter 14](14-the-ddl-lock-window.md) is specific to H2's + rebuild-based implementation of `ADD COLUMN`/`DROP COLUMN` — verify what your actual + production database does under concurrent writes for the specific change you're + making, and test it, the same way that chapter's test does, against your own engine + and table size before trusting either "it's always safe" or "it always errors + loudly". +- **Remove or lock down the diagnostic endpoints.** + [`SchemaDiagnosticsController`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/diag/SchemaDiagnosticsController.java)'s + `/diag/schema` and + [`DrainController`](https://ankurm.com/git.app/asmhatre/spring-boot-demo/src/branch/main/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/diag/DrainController.java)'s + `/admin/drain` have no authentication and are wired for a demo where anyone can + curl them freely. `/admin/drain` in particular can take a real instance out of a + real load balancer's rotation with a single unauthenticated `POST` — restrict it to + the same internal network your orchestrator's `preStop` hook runs from, or replace + it with your platform's native lifecycle hook. +- **Run the real load test against your real database and your real fleet size**, + not just this module's two-replica sandbox. The `AUTO_SERVER` trap + ([chapter 12](12-the-auto-server-trap.md)) and the DDL lock window + ([chapter 14](14-the-ddl-lock-window.md)) were both found by running actual + concurrent traffic through actual restarts — reading about the technique would not + have surfaced either one. +- **Budget for at least two separate code deploys plus two schema changes**, not one + deploy. Teams estimating "rename a column" as a single-PR, single-deploy task are + the ones most likely to reach for the `RENAME COLUMN` shortcut this article opened + with — see [chapter 1](01-the-problem-and-the-plan.md). +- **Decide what "zero errors" means for your own load test before you run it.** This + article's own final number is 99.98%, not literally zero, and the six-in-30,911 + residual is explained down to its root mechanism rather than hand-waved — see the + honest accounting in [chapter 14](14-the-ddl-lock-window.md). A number you can fully + explain is more useful, and more trustworthy, than one you cannot account for at + all. + +## The whole sequence, one command + +```bash +./scripts/run-all.sh +``` + +regenerates every transcript this article and these chapters quote, end to end: +schema-only Deploy 1, rolling Deploy 2, rolling Deploy 3, rolling Deploy 4a, +schema-only Deploy 4b, with the load generator running continuously throughout. See +the [module README](../README.md) for the full script index and version table. + +[← 14. The DDL lock window](14-the-ddl-lock-window.md) diff --git a/db-migrations-expand-contract/docs/output/02-expand-backward-compatible.txt b/db-migrations-expand-contract/docs/output/02-expand-backward-compatible.txt new file mode 100644 index 0000000..688cd53 --- /dev/null +++ b/db-migrations-expand-contract/docs/output/02-expand-backward-compatible.txt @@ -0,0 +1,36 @@ +========================================================================= +Deploy 1 (EXPAND): additive column + backfill, Stage 1 code untouched +========================================================================= +captured: 2026-09-16T19:20:56.676919361Z + + +-- schema before Deploy 1 -- +COLUMN_NAME +----------- +ID +NAME +EMAIL +CREATED_AT +(4 rows) + +-- schema after Deploy 1 (email_address added) -- +COLUMN_NAME +------------- +ID +NAME +EMAIL +CREATED_AT +EMAIL_ADDRESS +(5 rows) + +-- Ada's row was backfilled by the migration itself -- +NAME | EMAIL | EMAIL_ADDRESS +-------------+------------------+----------------- +Ada Lovelace | ada@example.test | ada@example.test +(1 row) + +-- Stage 1's original INSERT still works, unmodified, after the migration -- +NAME | EMAIL | EMAIL_ADDRESS +-------------+--------------------+-------------- +Grace Hopper | grace@example.test | NULL +(1 row) diff --git a/db-migrations-expand-contract/docs/output/04-dual-write-consistency.txt b/db-migrations-expand-contract/docs/output/04-dual-write-consistency.txt new file mode 100644 index 0000000..a964a00 --- /dev/null +++ b/db-migrations-expand-contract/docs/output/04-dual-write-consistency.txt @@ -0,0 +1,17 @@ +================================================================== +Deploy 2 (MIGRATE WRITES): Stage 2 writes land in both columns +================================================================== +captured: 2026-09-16T19:21:04.333919632Z + + +-- after create() -- +NAME | EMAIL | EMAIL_ADDRESS +------------------+-----------------------+---------------------- +Margaret Hamilton | margaret@example.test | margaret@example.test +(1 row) + +-- after updateEmail() - the old value is gone from BOTH columns, not just one -- +NAME | EMAIL | EMAIL_ADDRESS +------------------+-------------------------+------------------------ +Margaret Hamilton | m.hamilton@example.test | m.hamilton@example.test +(1 row) diff --git a/db-migrations-expand-contract/docs/output/06-not-null-trap.txt b/db-migrations-expand-contract/docs/output/06-not-null-trap.txt new file mode 100644 index 0000000..004aa77 --- /dev/null +++ b/db-migrations-expand-contract/docs/output/06-not-null-trap.txt @@ -0,0 +1,14 @@ +========================================================================== +The NOT NULL trap: expand without relaxing the old column's constraint +========================================================================== +captured: 2026-09-16T19:21:04.662650854Z + + +-- Stage 4 create() against the NAIVE migration (no DROP NOT NULL) -- +org.springframework.dao.DataIntegrityViolationException: PreparedStatementCallback; SQL [INSERT INTO customers(name, email_address) VALUES (?, ?)]; NULL not allowed for column "EMAIL"; SQL statement: +INSERT INTO customers(name, email_address) VALUES (?, ?) [23502-240] +root cause: org.h2.jdbc.JdbcSQLIntegrityConstraintViolationException: NULL not allowed for column "EMAIL"; SQL statement: +INSERT INTO customers(name, email_address) VALUES (?, ?) [23502-240] + +-- Stage 4 create() against the SHIPPED V2 migration (DROP NOT NULL included) -- +Customer[id=1, name=On Time, email=on.time@example.test] diff --git a/db-migrations-expand-contract/docs/output/07-backfill-window-bug.txt b/db-migrations-expand-contract/docs/output/07-backfill-window-bug.txt new file mode 100644 index 0000000..846f814 --- /dev/null +++ b/db-migrations-expand-contract/docs/output/07-backfill-window-bug.txt @@ -0,0 +1,14 @@ +============================================================================== +The backfill window: a Stage 1 write after Deploy 1, read by a naive Stage 3 +============================================================================== +captured: 2026-09-16T19:20:56.276837493Z + + +-- the row a lingering Stage 1 instance just wrote -- +[{NAME=Katherine Johnson, EMAIL=katherine@example.test, EMAIL_ADDRESS=null}] + +-- a NAIVE Stage 3 read (email_address alone) - the bug -- +naive Stage 3 email column value: null + +-- the SHIPPED Stage 3 read (CustomerService, COALESCE) - the fix -- +CustomerService (stage 3) result: Optional[Customer[id=1, name=Katherine Johnson, email=katherine@example.test]] diff --git a/db-migrations-expand-contract/docs/output/08-mixed-stage-rolling-window.txt b/db-migrations-expand-contract/docs/output/08-mixed-stage-rolling-window.txt new file mode 100644 index 0000000..ef81078 --- /dev/null +++ b/db-migrations-expand-contract/docs/output/08-mixed-stage-rolling-window.txt @@ -0,0 +1,23 @@ +==================================================================== +Cross-stage consistency during each of the three rolling deploys +==================================================================== +captured: 2026-09-16T19:21:05.241547756Z + + +-- Stage 1 writes, Stage 2 reads -- +Customer[id=1, name=Radia Perlman, email=radia@example.test] + +-- Stage 2 writes, Stage 1 reads -- +Customer[id=2, name=Barbara Liskov, email=barbara@example.test] + +-- Stage 2 writes, Stage 3 reads -- +Customer[id=3, name=Shafi Goldwasser, email=shafi@example.test] + +-- Stage 3 writes, Stage 2 reads -- +Customer[id=4, name=Frances Allen, email=frances@example.test] + +-- Stage 3 writes, Stage 4 reads -- +Customer[id=5, name=Adele Goldberg, email=adele@example.test] + +-- Stage 4 writes, Stage 3 reads -- +Customer[id=6, name=Karen Sparck Jones, email=karen@example.test] diff --git a/db-migrations-expand-contract/docs/output/09-contract-safety.txt b/db-migrations-expand-contract/docs/output/09-contract-safety.txt new file mode 100644 index 0000000..19aa25d --- /dev/null +++ b/db-migrations-expand-contract/docs/output/09-contract-safety.txt @@ -0,0 +1,11 @@ +============================================================================== +Deploy 4 (CONTRACT): Stage 4 after the drop, and what breaks if you drop too soon +============================================================================== +captured: 2026-09-16T19:20:57.618780340Z + + +-- Stage 4 read, after V3 dropped the email column -- +Customer[id=1, name=Annie Easley, email=annie@example.test] + +-- Stage 4 create + read, entirely after the drop -- +Customer[id=2, name=Mary Allen Wilkes, email=mary@example.test] diff --git a/db-migrations-expand-contract/docs/output/10-drop-too-soon.txt b/db-migrations-expand-contract/docs/output/10-drop-too-soon.txt new file mode 100644 index 0000000..eaf4f32 --- /dev/null +++ b/db-migrations-expand-contract/docs/output/10-drop-too-soon.txt @@ -0,0 +1,9 @@ +============================================================================== +What a lingering Stage 1 instance sees if the drop runs before it is retired +============================================================================== +captured: 2026-09-16T19:20:58.517481426Z + +Stage 1 create() after V3 dropped "email": org.springframework.jdbc.BadSqlGrammarException +message: PreparedStatementCallback; bad SQL grammar [INSERT INTO customers(name, email) VALUES (?, ?)] +root cause: org.h2.jdbc.JdbcSQLSyntaxErrorException: Column "EMAIL" not found; SQL statement: +INSERT INTO customers(name, email) VALUES (?, ?) [42122-240] diff --git a/db-migrations-expand-contract/docs/output/11-live-deploy-sequence.txt b/db-migrations-expand-contract/docs/output/11-live-deploy-sequence.txt new file mode 100644 index 0000000..9be8482 --- /dev/null +++ b/db-migrations-expand-contract/docs/output/11-live-deploy-sequence.txt @@ -0,0 +1,103 @@ +===================================================================== +Zero-downtime expand-contract: live 4-deploy sequence +===================================================================== +19:00:53 captured: 2026-09-16T19:00:53Z +19:00:53 Starting the database as its own standalone process (not owned by either replica) +H2 TCP server up on port 9092 (pid 10861), baseDir /tmp/ec-demo/db +19:00:53 Deploy 0: create schema (V1), start two Stage 1 replicas +=== Before migrate (target=1) === +00:30:54.634 [main] INFO org.flywaydb.core.FlywayExecutor -- Database: jdbc:h2:tcp://localhost:9092/expand-contract (H2 2.4) +00:30:54.702 [main] WARN org.flywaydb.core.internal.database.base.Database -- Using H2 2.4.240 which is newer than the version Flyway has been verified with. The latest verified version of H2 is 2.3.232. +00:30:54.750 [main] INFO org.flywaydb.core.internal.schemahistory.JdbcTableSchemaHistory -- Schema history table "PUBLIC"."flyway_schema_history" does not exist yet + 1 create customer PENDING + 2 add email address column ABOVE_TARGET + 3 drop email column ABOVE_TARGET +00:30:54.850 [main] INFO org.flywaydb.core.FlywayExecutor -- Database: jdbc:h2:tcp://localhost:9092/expand-contract (H2 2.4) +00:30:54.863 [main] WARN org.flywaydb.core.internal.database.base.Database -- Using H2 2.4.240 which is newer than the version Flyway has been verified with. The latest verified version of H2 is 2.3.232. +00:30:54.874 [main] INFO org.flywaydb.core.internal.schemahistory.JdbcTableSchemaHistory -- Schema history table "PUBLIC"."flyway_schema_history" does not exist yet +00:30:54.877 [main] INFO org.flywaydb.core.internal.command.DbValidate -- Successfully validated 3 migrations (execution time 00:00.007s) +00:30:54.888 [main] INFO org.flywaydb.core.internal.schemahistory.JdbcTableSchemaHistory -- Creating Schema History table "PUBLIC"."flyway_schema_history" ... +00:30:54.940 [main] INFO org.flywaydb.core.internal.command.DbMigrate -- Current version of schema "PUBLIC": << Empty Schema >> +00:30:54.956 [main] INFO org.flywaydb.core.internal.command.DbMigrate -- Migrating schema "PUBLIC" to version "1 - create customer" +00:30:55.018 [main] INFO org.flywaydb.core.internal.command.DbMigrate -- Successfully applied 1 migration to schema "PUBLIC", now at version v1 (execution time 00:00.014s) +=== After migrate === +00:30:55.055 [main] INFO org.flywaydb.core.FlywayExecutor -- Database: jdbc:h2:tcp://localhost:9092/expand-contract (H2 2.4) +00:30:55.063 [main] WARN org.flywaydb.core.internal.database.base.Database -- Using H2 2.4.240 which is newer than the version Flyway has been verified with. The latest verified version of H2 is 2.3.232. + 1 create customer SUCCESS + 2 add email address column ABOVE_TARGET + 3 drop email column ABOVE_TARGET +Migrations executed: 1, target schema version: 1, success: true +started stage 1 on port 8081 (pid 10946) +port 8081 healthy +started stage 1 on port 8082 (pid 11011) +port 8082 healthy +19:01:02 load generator running against both replicas +19:01:12 baseline soak complete (10s, both replicas on Stage 1) +19:01:12 Deploy 1 (EXPAND): migrating to V2 live - zero app restarts +=== Before migrate (target=2) === +00:31:13.484 [main] INFO org.flywaydb.core.FlywayExecutor -- Database: jdbc:h2:tcp://localhost:9092/expand-contract (H2 2.4) +00:31:13.607 [main] WARN org.flywaydb.core.internal.database.base.Database -- Using H2 2.4.240 which is newer than the version Flyway has been verified with. The latest verified version of H2 is 2.3.232. + 1 create customer SUCCESS + 2 add email address column PENDING + 3 drop email column ABOVE_TARGET +00:31:13.782 [main] INFO org.flywaydb.core.FlywayExecutor -- Database: jdbc:h2:tcp://localhost:9092/expand-contract (H2 2.4) +00:31:13.801 [main] WARN org.flywaydb.core.internal.database.base.Database -- Using H2 2.4.240 which is newer than the version Flyway has been verified with. The latest verified version of H2 is 2.3.232. +00:31:13.817 [main] INFO org.flywaydb.core.internal.command.DbValidate -- Successfully validated 3 migrations (execution time 00:00.008s) +00:31:13.842 [main] INFO org.flywaydb.core.internal.command.DbMigrate -- Current version of schema "PUBLIC": 1 +00:31:13.903 [main] INFO org.flywaydb.core.internal.command.DbMigrate -- Migrating schema "PUBLIC" to version "2 - add email address column" +00:31:14.097 [main] INFO org.flywaydb.core.internal.command.DbMigrate -- Successfully applied 1 migration to schema "PUBLIC", now at version v2 (execution time 00:00.118s) +=== After migrate === +00:31:14.136 [main] INFO org.flywaydb.core.FlywayExecutor -- Database: jdbc:h2:tcp://localhost:9092/expand-contract (H2 2.4) +00:31:14.174 [main] WARN org.flywaydb.core.internal.database.base.Database -- Using H2 2.4.240 which is newer than the version Flyway has been verified with. The latest verified version of H2 is 2.3.232. + 1 create customer SUCCESS + 2 add email address column SUCCESS + 3 drop email column ABOVE_TARGET +Migrations executed: 1, target schema version: 2, success: true +19:01:19 Deploy 2 (MIGRATE WRITES): rolling restart to Stage 2, replica A first +stopped port 8081 (pid 10946) +started stage 2 on port 8081 (pid 11226) +port 8081 healthy +19:01:29 Deploy 2: replica B +stopped port 8082 (pid 11011) +started stage 2 on port 8082 (pid 11338) +port 8082 healthy +19:01:45 Deploy 3 (MIGRATE READS): rolling restart to Stage 3, replica A first +stopped port 8081 (pid 11226) +started stage 3 on port 8081 (pid 11456) +port 8081 healthy +19:01:55 Deploy 3: replica B +stopped port 8082 (pid 11338) +started stage 3 on port 8082 (pid 11567) +port 8082 healthy +19:02:11 Deploy 4a (CONTRACT code): rolling restart to Stage 4, replica A first +stopped port 8081 (pid 11456) +started stage 4 on port 8081 (pid 11679) +port 8081 healthy +19:02:20 Deploy 4a: replica B +stopped port 8082 (pid 11567) +started stage 4 on port 8082 (pid 11784) +port 8082 healthy +19:02:35 Deploy 4b (CONTRACT schema): migrating to V3 live - drops "email", zero app restarts +=== Before migrate (target=latest) === +00:32:36.510 [main] INFO org.flywaydb.core.FlywayExecutor -- Database: jdbc:h2:tcp://localhost:9092/expand-contract (H2 2.4) +00:32:36.574 [main] WARN org.flywaydb.core.internal.database.base.Database -- Using H2 2.4.240 which is newer than the version Flyway has been verified with. The latest verified version of H2 is 2.3.232. + 1 create customer SUCCESS + 2 add email address column SUCCESS + 3 drop email column PENDING +00:32:36.669 [main] INFO org.flywaydb.core.FlywayExecutor -- Database: jdbc:h2:tcp://localhost:9092/expand-contract (H2 2.4) +00:32:36.680 [main] WARN org.flywaydb.core.internal.database.base.Database -- Using H2 2.4.240 which is newer than the version Flyway has been verified with. The latest verified version of H2 is 2.3.232. +00:32:36.694 [main] INFO org.flywaydb.core.internal.command.DbValidate -- Successfully validated 3 migrations (execution time 00:00.008s) +00:32:36.707 [main] INFO org.flywaydb.core.internal.command.DbMigrate -- Current version of schema "PUBLIC": 2 +00:32:36.726 [main] INFO org.flywaydb.core.internal.command.DbMigrate -- Migrating schema "PUBLIC" to version "3 - drop email column" +00:32:36.795 [main] INFO org.flywaydb.core.internal.command.DbMigrate -- Successfully applied 1 migration to schema "PUBLIC", now at version v3 (execution time 00:00.041s) +=== After migrate === +00:32:36.811 [main] INFO org.flywaydb.core.FlywayExecutor -- Database: jdbc:h2:tcp://localhost:9092/expand-contract (H2 2.4) +00:32:36.829 [main] WARN org.flywaydb.core.internal.database.base.Database -- Using H2 2.4.240 which is newer than the version Flyway has been verified with. The latest verified version of H2 is 2.3.232. + 1 create customer SUCCESS + 2 add email address column SUCCESS + 3 drop email column SUCCESS +Migrations executed: 1, target schema version: 3, success: true +19:02:46 final soak complete +19:02:46 waiting for the load generator to finish its run... +19:03:04 load generator finished +19:03:11 === deploy sequence complete === diff --git a/db-migrations-expand-contract/docs/output/12-load-generator-summary.txt b/db-migrations-expand-contract/docs/output/12-load-generator-summary.txt new file mode 100644 index 0000000..f3e5b97 --- /dev/null +++ b/db-migrations-expand-contract/docs/output/12-load-generator-summary.txt @@ -0,0 +1,21 @@ +Load generator summary +======================= +Total requests: 30911 +Successful: 30905 +Errors: 6 + +By phase: + 04a-stage4-soak ok=2130 error=0 + 04b-contract-migration ok=336 error=0 + 04a-deploy-stage4-rollout ok=4147 error=0 + 03-stage3-soak ok=2116 error=0 + 03-deploy-stage3-rollout ok=4645 error=0 + 00-baseline-soak ok=1893 error=0 + 05-final-soak ok=7118 error=4 + - read-http-404 2 + - update-http-404 2 + 02-stage2-soak ok=2068 error=0 + 01-expand-migration ok=1773 error=1 + - read-http-404 1 + 02-deploy-stage2-rollout ok=4679 error=1 + - update-http-404 1 diff --git a/db-migrations-expand-contract/docs/output/13-schema-diagnostics-timeline.txt b/db-migrations-expand-contract/docs/output/13-schema-diagnostics-timeline.txt new file mode 100644 index 0000000..3879148 --- /dev/null +++ b/db-migrations-expand-contract/docs/output/13-schema-diagnostics-timeline.txt @@ -0,0 +1,18 @@ + +-- after Deploy 0 (Stage 1 / Stage 1) -- +{"appStage":1,"columns":["ID","NAME","EMAIL","CREATED_AT"],"rowCount":0,"rowsWithEmail":0} + +-- after Deploy 1 (schema expanded, both replicas still Stage 1) -- +{"appStage":1,"columns":["ID","NAME","EMAIL","CREATED_AT","EMAIL_ADDRESS"],"rowCount":1212,"rowsWithEmail":1212,"rowsWithEmailAddress":1184} + +-- after Deploy 2 (both replicas Stage 2, dual-write live) -- +{"appStage":2,"columns":["ID","NAME","EMAIL","CREATED_AT","EMAIL_ADDRESS"],"rowCount":4218,"rowsWithEmail":4218,"rowsWithEmailAddress":2561} + +-- after Deploy 3 (both replicas Stage 3, reading email_address) -- +{"appStage":3,"columns":["ID","NAME","EMAIL","CREATED_AT","EMAIL_ADDRESS"],"rowCount":7547,"rowsWithEmail":7547,"rowsWithEmailAddress":6115} + +-- after Deploy 4a (both replicas Stage 4, email column still present but unused) -- +{"appStage":4,"columns":["ID","NAME","EMAIL","CREATED_AT","EMAIL_ADDRESS"],"rowCount":10642,"rowsWithEmail":9653,"rowsWithEmailAddress":9212} + +-- after Deploy 4b (email column dropped) -- +{"appStage":4,"columns":["ID","NAME","CREATED_AT","EMAIL_ADDRESS"],"rowCount":11806,"rowsWithEmailAddress":10374} diff --git a/db-migrations-expand-contract/docs/output/14-ddl-silent-data-loss.txt b/db-migrations-expand-contract/docs/output/14-ddl-silent-data-loss.txt new file mode 100644 index 0000000..5d4fc22 --- /dev/null +++ b/db-migrations-expand-contract/docs/output/14-ddl-silent-data-loss.txt @@ -0,0 +1,14 @@ +============================================================================== +The failure the retry cannot catch: a committed INSERT that ALTER TABLE loses silently +============================================================================== +captured: 2026-09-16T19:20:58.574465432Z + +attempts needed to reproduce the race: 2 of 20 +customer creates that returned a generated id with no error: 50 +customer creates that got the already-documented, already-fixed DDL-collision error: 1 +of the ids that came back with no error, missing from the table once V2 finished: 8 +example missing ids: [41, 42, 43, 44, 45] + +This is why the retry in CustomerService cannot be the whole fix: these inserts +never threw anything to retry. The row was committed, then discarded when the +ADD COLUMN rebuild swapped in a new table that had already been scanned. diff --git a/db-migrations-expand-contract/pom.xml b/db-migrations-expand-contract/pom.xml new file mode 100644 index 0000000..c408a36 --- /dev/null +++ b/db-migrations-expand-contract/pom.xml @@ -0,0 +1,92 @@ + + + 4.0.0 + + + org.springframework.boot + spring-boot-starter-parent + 4.1.1 + + + + com.ankurm + db-migrations-expand-contract + 1.0.0 + db-migrations-expand-contract + Companion code for Zero-Downtime Database Migrations: Expand-Contract in Practice with Spring Boot + + + 25 + 25 + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-jdbc + + + org.springframework.boot + spring-boot-starter-actuator + + + org.springframework.boot + spring-boot-starter-flyway + + + com.h2database + h2 + runtime + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.boot + spring-boot-starter-flyway-test + test + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + org.apache.maven.plugins + maven-dependency-plugin + + + write-runtime-classpath + prepare-package + + build-classpath + + + ${project.basedir}/cp.txt + + + + + + + diff --git a/db-migrations-expand-contract/scripts/env.sh b/db-migrations-expand-contract/scripts/env.sh new file mode 100755 index 0000000..61a6a64 --- /dev/null +++ b/db-migrations-expand-contract/scripts/env.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Shared paths and settings for every script in this directory. Source this, don't run it. +export EC_DB_TCP_PORT="${EC_DB_TCP_PORT:-9092}" +export EC_DB_NAME="${EC_DB_NAME:-expand-contract}" +EC_DB_BASE_DIR="${EC_DB_BASE_DIR:-/tmp/ec-demo/db}" +EC_RUN_DIR="${EC_RUN_DIR:-/tmp/ec-demo}" +EC_LOG_DIR="$EC_RUN_DIR/logs" +EC_PID_DIR="$EC_RUN_DIR/pids" +EC_PHASE_FILE="$EC_RUN_DIR/phase.txt" +EC_LOAD_SUMMARY="$EC_RUN_DIR/load-summary.txt" +PORT_A=8081 +PORT_B=8082 + +MODULE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +JAR="$MODULE_DIR/target/db-migrations-expand-contract-1.0.0.jar" +CP_FILE="$MODULE_DIR/cp.txt" + +mkdir -p "$EC_LOG_DIR" "$EC_PID_DIR" "$EC_DB_BASE_DIR" + +wait_db_server() { + local tries="${1:-40}" + for i in $(seq 1 "$tries"); do + if (echo > "/dev/tcp/localhost/$EC_DB_TCP_PORT") >/dev/null 2>&1; then + return 0 + fi + sleep 0.25 + done + echo "H2 TCP server on port $EC_DB_TCP_PORT never came up" >&2 + return 1 +} + +set_phase() { + echo -n "$1" > "$EC_PHASE_FILE" + echo "[phase] $1" +} + +wait_healthy() { + local port="$1" + local tries="${2:-40}" + for i in $(seq 1 "$tries"); do + code=$(curl -s -o /dev/null -w '%{http_code}' "http://localhost:$port/actuator/health" || true) + if [ "$code" = "200" ]; then + return 0 + fi + sleep 0.5 + done + echo "instance on port $port never became healthy" >&2 + return 1 +} + +wait_down() { + local port="$1" + local tries="${2:-20}" + for i in $(seq 1 "$tries"); do + code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 1 "http://localhost:$port/actuator/health" || true) + if [ "$code" != "200" ]; then + return 0 + fi + sleep 0.3 + done + return 1 +} diff --git a/db-migrations-expand-contract/scripts/migrate.sh b/db-migrations-expand-contract/scripts/migrate.sh new file mode 100755 index 0000000..5544581 --- /dev/null +++ b/db-migrations-expand-contract/scripts/migrate.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Runs the standalone migration.MigrationCli against the live database - no app +# restart, no app deploy. Usage: migrate.sh +set -euo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$DIR/env.sh" + +TARGET="${1:-latest}" +java -cp "$MODULE_DIR/target/classes:$(cat "$CP_FILE")" \ + com.ankurm.expandcontract.migration.MigrationCli --target="$TARGET" diff --git a/db-migrations-expand-contract/scripts/run-all.sh b/db-migrations-expand-contract/scripts/run-all.sh new file mode 100755 index 0000000..2e79fdc --- /dev/null +++ b/db-migrations-expand-contract/scripts/run-all.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +# The live exhibit: two replicas, a shared H2 database, and a load generator that +# never stops sending traffic while this script performs all four deploys of the +# expand-contract sequence as an actual rolling deploy - one replica at a time. +# +# Regenerates: +# docs/output/11-live-deploy-sequence.txt - the deploy log, phase by phase +# docs/output/12-load-generator-summary.txt - total/ok/error counts, by phase +# docs/output/13-schema-diagnostics-timeline.txt - /diag/schema after each deploy +set -euo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$DIR/env.sh" + +OUTPUT_DIR="$MODULE_DIR/docs/output" +mkdir -p "$OUTPUT_DIR" +DEPLOY_LOG="$OUTPUT_DIR/11-live-deploy-sequence.txt" +DIAG_LOG="$OUTPUT_DIR/13-schema-diagnostics-timeline.txt" + +log() { + echo "$(date -u +%H:%M:%S) $*" | tee -a "$DEPLOY_LOG" +} + +diag() { + local label="$1" + { + echo "" + echo "-- $label --" + curl -s "http://localhost:$PORT_A/diag/schema" + echo "" + } | tee -a "$DIAG_LOG" +} + +# --- clean slate --- +for p in "$PORT_A" "$PORT_B"; do + "$DIR/stop-instance.sh" "$p" 2>/dev/null || true +done +if [ -f "$EC_PID_DIR/loadgen.pid" ]; then + kill -9 "$(cat "$EC_PID_DIR/loadgen.pid")" 2>/dev/null || true + rm -f "$EC_PID_DIR/loadgen.pid" +fi +"$DIR/stop-db-server.sh" 2>/dev/null || true +rm -rf "$EC_DB_BASE_DIR" +mkdir -p "$EC_DB_BASE_DIR" +rm -f "$DEPLOY_LOG" "$DIAG_LOG" + +echo "=====================================================================" > "$DEPLOY_LOG" +echo "Zero-downtime expand-contract: live 4-deploy sequence" >> "$DEPLOY_LOG" +echo "=====================================================================" >> "$DEPLOY_LOG" +log "captured: $(date -u +%FT%TZ)" + +# --- Deploy -1: the database itself. This process is never restarted for the rest +# of this script - it is the one thing every deploy below has to treat as always up. +log "Starting the database as its own standalone process (not owned by either replica)" +"$DIR/start-db-server.sh" | tee -a "$DEPLOY_LOG" + +# --- Deploy 0: baseline schema + two Stage 1 replicas --- +log "Deploy 0: create schema (V1), start two Stage 1 replicas" +"$DIR/migrate.sh" 1 | tee -a "$DEPLOY_LOG" > /dev/null +"$DIR/start-instance.sh" "$PORT_A" 1 | tee -a "$DEPLOY_LOG" +"$DIR/start-instance.sh" "$PORT_B" 1 | tee -a "$DEPLOY_LOG" +diag "after Deploy 0 (Stage 1 / Stage 1)" + +"$DIR/run-load-generator.sh" 120 +log "load generator running against both replicas" +sleep 10 +log "baseline soak complete (10s, both replicas on Stage 1)" + +# --- Deploy 1 (EXPAND): schema only, zero app restarts --- +set_phase "01-expand-migration" +log "Deploy 1 (EXPAND): migrating to V2 live - zero app restarts" +"$DIR/migrate.sh" 2 | tee -a "$DEPLOY_LOG" > /dev/null +diag "after Deploy 1 (schema expanded, both replicas still Stage 1)" +sleep 5 + +# --- Deploy 2 (MIGRATE WRITES): rolling restart to Stage 2 --- +set_phase "02-deploy-stage2-rollout" +log "Deploy 2 (MIGRATE WRITES): rolling restart to Stage 2, replica A first" +"$DIR/stop-instance.sh" "$PORT_A" | tee -a "$DEPLOY_LOG" +"$DIR/start-instance.sh" "$PORT_A" 2 | tee -a "$DEPLOY_LOG" +sleep 2 +log "Deploy 2: replica B" +"$DIR/stop-instance.sh" "$PORT_B" | tee -a "$DEPLOY_LOG" +"$DIR/start-instance.sh" "$PORT_B" 2 | tee -a "$DEPLOY_LOG" +set_phase "02-stage2-soak" +diag "after Deploy 2 (both replicas Stage 2, dual-write live)" +sleep 8 + +# --- Deploy 3 (MIGRATE READS): rolling restart to Stage 3 --- +set_phase "03-deploy-stage3-rollout" +log "Deploy 3 (MIGRATE READS): rolling restart to Stage 3, replica A first" +"$DIR/stop-instance.sh" "$PORT_A" | tee -a "$DEPLOY_LOG" +"$DIR/start-instance.sh" "$PORT_A" 3 | tee -a "$DEPLOY_LOG" +sleep 2 +log "Deploy 3: replica B" +"$DIR/stop-instance.sh" "$PORT_B" | tee -a "$DEPLOY_LOG" +"$DIR/start-instance.sh" "$PORT_B" 3 | tee -a "$DEPLOY_LOG" +set_phase "03-stage3-soak" +diag "after Deploy 3 (both replicas Stage 3, reading email_address)" +sleep 8 + +# --- Deploy 4a (CONTRACT, code): rolling restart to Stage 4 --- +set_phase "04a-deploy-stage4-rollout" +log "Deploy 4a (CONTRACT code): rolling restart to Stage 4, replica A first" +"$DIR/stop-instance.sh" "$PORT_A" | tee -a "$DEPLOY_LOG" +"$DIR/start-instance.sh" "$PORT_A" 4 | tee -a "$DEPLOY_LOG" +sleep 2 +log "Deploy 4a: replica B" +"$DIR/stop-instance.sh" "$PORT_B" | tee -a "$DEPLOY_LOG" +"$DIR/start-instance.sh" "$PORT_B" 4 | tee -a "$DEPLOY_LOG" +set_phase "04a-stage4-soak" +diag "after Deploy 4a (both replicas Stage 4, email column still present but unused)" +sleep 8 + +# --- Deploy 4b (CONTRACT, schema): drop the old column, live --- +set_phase "04b-contract-migration" +log "Deploy 4b (CONTRACT schema): migrating to V3 live - drops \"email\", zero app restarts" +"$DIR/migrate.sh" latest | tee -a "$DEPLOY_LOG" > /dev/null +diag "after Deploy 4b (email column dropped)" + +set_phase "05-final-soak" +sleep 10 +log "final soak complete" + +log "waiting for the load generator to finish its run..." +if [ -f "$EC_PID_DIR/loadgen.pid" ]; then + LOADGEN_PID="$(cat "$EC_PID_DIR/loadgen.pid")" + while kill -0 "$LOADGEN_PID" 2>/dev/null; do + sleep 1 + done +fi +log "load generator finished" + +cp "$EC_LOAD_SUMMARY" "$OUTPUT_DIR/12-load-generator-summary.txt" + +for p in "$PORT_A" "$PORT_B"; do + "$DIR/stop-instance.sh" "$p" || true +done +"$DIR/stop-db-server.sh" || true + +log "=== deploy sequence complete ===" +echo "" +echo "Deploy log: $DEPLOY_LOG" +echo "Load summary: $OUTPUT_DIR/12-load-generator-summary.txt" +echo "Schema timeline: $DIAG_LOG" +cat "$OUTPUT_DIR/12-load-generator-summary.txt" diff --git a/db-migrations-expand-contract/scripts/run-load-generator.sh b/db-migrations-expand-contract/scripts/run-load-generator.sh new file mode 100755 index 0000000..c1583bf --- /dev/null +++ b/db-migrations-expand-contract/scripts/run-load-generator.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Starts the load generator against both replica ports for a fixed duration and +# returns immediately - it runs in the background for the rest of run-all.sh. +# Usage: run-load-generator.sh +set -euo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$DIR/env.sh" + +DURATION="${1:-170}" +rm -f "$EC_PHASE_FILE" "$EC_LOAD_SUMMARY" +set_phase "00-baseline-soak" + +nohup java -cp "$MODULE_DIR/target/classes:$(cat "$CP_FILE")" \ + com.ankurm.expandcontract.loadgen.LoadGenerator \ + --ports="$PORT_A,$PORT_B" \ + --durationSeconds="$DURATION" \ + --phaseFile="$EC_PHASE_FILE" \ + --outFile="$EC_LOAD_SUMMARY" \ + --threads=8 \ + > "$EC_LOG_DIR/load-generator.log" 2>&1 < /dev/null & +echo $! > "$EC_PID_DIR/loadgen.pid" +echo "load generator started (pid $(cat "$EC_PID_DIR/loadgen.pid")), running for ${DURATION}s" diff --git a/db-migrations-expand-contract/scripts/start-db-server.sh b/db-migrations-expand-contract/scripts/start-db-server.sh new file mode 100755 index 0000000..af7ede3 --- /dev/null +++ b/db-migrations-expand-contract/scripts/start-db-server.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Starts H2 as its own standalone TCP server process - not owned by, or co-located +# inside, either app replica. This is what a real production database is: a process +# that outlives every app deploy. See docs/12-the-auto-server-trap.md for what went +# wrong the first time this demo shared a database file directly between the two +# replicas instead. +set -euo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$DIR/env.sh" + +nohup java -cp "$(cat "$CP_FILE")" org.h2.tools.Server \ + -tcp -tcpPort "$EC_DB_TCP_PORT" -baseDir "$EC_DB_BASE_DIR" -ifNotExists \ + > "$EC_LOG_DIR/db-server.log" 2>&1 < /dev/null & +echo $! > "$EC_PID_DIR/db-server.pid" +wait_db_server +echo "H2 TCP server up on port $EC_DB_TCP_PORT (pid $(cat "$EC_PID_DIR/db-server.pid")), baseDir $EC_DB_BASE_DIR" diff --git a/db-migrations-expand-contract/scripts/start-instance.sh b/db-migrations-expand-contract/scripts/start-instance.sh new file mode 100755 index 0000000..7e616c1 --- /dev/null +++ b/db-migrations-expand-contract/scripts/start-instance.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Starts one replica on a given port and stage, and waits for it to report healthy. +# Usage: start-instance.sh +set -euo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$DIR/env.sh" + +PORT="$1" +STAGE="$2" + +# -XX:TieredStopAtLevel=1 and a small fixed heap cut this JVM's own startup CPU +# burst dramatically (C2 compilation and heap sizing are the two biggest costs of a +# cold Spring Boot start). That matters here specifically because two replicas share +# a 2-core sandbox: a slow-starting replica can starve its own sibling's health +# checks long enough to look like an outage that never actually happened - see +# docs/11-the-load-generator.md. +nohup java -XX:TieredStopAtLevel=1 -XX:+UseSerialGC -Xms128m -Xmx256m \ + -jar "$JAR" --server.port="$PORT" --app.stage="$STAGE" \ + > "$EC_LOG_DIR/instance-$PORT.log" 2>&1 < /dev/null & +echo $! > "$EC_PID_DIR/$PORT.pid" +echo "started stage $STAGE on port $PORT (pid $(cat "$EC_PID_DIR/$PORT.pid"))" +wait_healthy "$PORT" +echo "port $PORT healthy" diff --git a/db-migrations-expand-contract/scripts/stop-db-server.sh b/db-migrations-expand-contract/scripts/stop-db-server.sh new file mode 100755 index 0000000..9cebcb7 --- /dev/null +++ b/db-migrations-expand-contract/scripts/stop-db-server.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$DIR/env.sh" + +PID_FILE="$EC_PID_DIR/db-server.pid" +if [ -f "$PID_FILE" ]; then + PID="$(cat "$PID_FILE")" + if kill -0 "$PID" 2>/dev/null; then + kill -9 "$PID" + echo "stopped H2 TCP server (pid $PID)" + fi + rm -f "$PID_FILE" +fi diff --git a/db-migrations-expand-contract/scripts/stop-instance.sh b/db-migrations-expand-contract/scripts/stop-instance.sh new file mode 100755 index 0000000..b9b9edc --- /dev/null +++ b/db-migrations-expand-contract/scripts/stop-instance.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Stops one replica by the PID file start-instance.sh wrote for it - never by matching +# the process name or command line, which risks matching the wrong process (including +# this very script's own shell). Usage: stop-instance.sh +# +# Sends SIGTERM, not SIGKILL. "server.shutdown: graceful" in application.yml only +# does anything on SIGTERM: it stops accepting new connections but lets in-flight +# requests finish first. An earlier version used `kill -9` here, which bypasses that +# entirely, and requests that were in flight the instant the process vanished showed +# up in the load generator's summary as ConnectException/IOException - a real +# artifact of skipping the drain step, not a defect in the migration itself. See +# docs/13-graceful-shutdown-vs-kill-9.md. +set -euo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$DIR/env.sh" + +PORT="$1" +PID_FILE="$EC_PID_DIR/$PORT.pid" +if [ -f "$PID_FILE" ]; then + PID="$(cat "$PID_FILE")" + if kill -0 "$PID" 2>/dev/null; then + # Deregister BEFORE terminating: tell the load balancer to stop sending new + # traffic here, then give its health check a couple of poll cycles to notice, + # THEN stop the process. Skipping this drain window and going straight to + # SIGTERM is what produced the ConnectException bursts in an earlier run. + curl -s -X POST "http://localhost:$PORT/admin/drain" -o /dev/null || true + sleep 1.5 + kill -15 "$PID" + for i in $(seq 1 40); do + kill -0 "$PID" 2>/dev/null || break + sleep 0.25 + done + if kill -0 "$PID" 2>/dev/null; then + echo "port $PORT (pid $PID) did not exit gracefully in 10s, sending SIGKILL" >&2 + kill -9 "$PID" + fi + echo "stopped port $PORT (pid $PID)" + fi + rm -f "$PID_FILE" +fi +wait_down "$PORT" || echo "warning: port $PORT still answering after stop" >&2 diff --git a/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/ExpandContractApplication.java b/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/ExpandContractApplication.java new file mode 100644 index 0000000..07c8b11 --- /dev/null +++ b/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/ExpandContractApplication.java @@ -0,0 +1,21 @@ +package com.ankurm.expandcontract; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +/** + * One jar, four behaviours. Which one a running instance exhibits is picked by + * {@code app.stage} (1-4), read by {@link com.ankurm.expandcontract.customer.CustomerService}. + * This is the same "wire variants behind a property so one artifact can demonstrate + * every stage" approach used across ankurm.com's companion repos - the point is that a + * reader can run two instances on two stages against the same database and watch the + * rolling deploy for themselves, rather than reading about it. + *

+ * See docs/01-the-problem-and-the-plan.md. + */ +@SpringBootApplication +public class ExpandContractApplication { + public static void main(String[] args) { + SpringApplication.run(ExpandContractApplication.class, args); + } +} diff --git a/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/customer/Customer.java b/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/customer/Customer.java new file mode 100644 index 0000000..2d68825 --- /dev/null +++ b/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/customer/Customer.java @@ -0,0 +1,10 @@ +package com.ankurm.expandcontract.customer; + +/** + * The response shape the API always returns, whatever stage is answering. A client + * of this service is never aware that "email" moved to "email_address" underneath it + * - that is the entire point of doing this as expand-contract instead of a single + * breaking rename. See docs/01-the-problem-and-the-plan.md. + */ +public record Customer(long id, String name, String email) { +} diff --git a/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/customer/CustomerController.java b/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/customer/CustomerController.java new file mode 100644 index 0000000..54e82c6 --- /dev/null +++ b/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/customer/CustomerController.java @@ -0,0 +1,51 @@ +package com.ankurm.expandcontract.customer; + +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +/** + * The API contract that never changes across all four deploys - see + * docs/01-the-problem-and-the-plan.md. A client of this controller, including the + * load generator in {@link com.ankurm.expandcontract.loadgen.LoadGenerator}, cannot + * tell which stage answered a given request just by looking at the response shape. + */ +@RestController +@RequestMapping("/customers") +public class CustomerController { + + private final CustomerService service; + + public CustomerController(CustomerService service) { + this.service = service; + } + + public record CreateCustomerRequest(String name, String email) { + } + + public record UpdateEmailRequest(String email) { + } + + @PostMapping + public ResponseEntity create(@RequestBody CreateCustomerRequest request) { + long id = service.create(request.name(), request.email()); + Customer created = service.findById(id).orElseThrow(); + return ResponseEntity.status(HttpStatus.CREATED).body(created); + } + + @GetMapping("/{id}") + public Customer get(@PathVariable long id) { + return service.findById(id).orElseThrow(() -> new CustomerService.CustomerNotFoundException(id)); + } + + @PutMapping("/{id}/email") + public Customer updateEmail(@PathVariable long id, @RequestBody UpdateEmailRequest request) { + service.updateEmail(id, request.email()); + return service.findById(id).orElseThrow(); + } + + @ExceptionHandler(CustomerService.CustomerNotFoundException.class) + public ResponseEntity notFound(CustomerService.CustomerNotFoundException ex) { + return ResponseEntity.status(HttpStatus.NOT_FOUND).body(ex.getMessage()); + } +} diff --git a/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/customer/CustomerService.java b/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/customer/CustomerService.java new file mode 100644 index 0000000..fa1d9f6 --- /dev/null +++ b/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/customer/CustomerService.java @@ -0,0 +1,129 @@ +package com.ankurm.expandcontract.customer; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.jdbc.BadSqlGrammarException; +import org.springframework.jdbc.core.simple.JdbcClient; +import org.springframework.jdbc.support.GeneratedKeyHolder; +import org.springframework.jdbc.support.KeyHolder; +import org.springframework.stereotype.Service; + +import java.util.Optional; +import java.util.function.Supplier; + +/** + * Every behaviour this article talks about lives in these four branches. Nothing + * else in the codebase changes between Stage 1 and Stage 4 - the controller, the + * table, the HTTP contract and the response shape are all identical throughout. + * Only which column this class writes to and reads from changes, and it changes + * exactly once per deploy. See docs/01-the-problem-and-the-plan.md for the mental + * model and docs/04-the-dual-write.md / docs/05-the-read-switch.md for why the + * write switch and the read switch are not the same deploy. + */ +@Service +public class CustomerService { + + private final JdbcClient jdbc; + private final int stage; + + public CustomerService(JdbcClient jdbc, @Value("${app.stage}") int stage) { + this.jdbc = jdbc; + if (stage < 1 || stage > 4) { + throw new IllegalArgumentException("app.stage must be 1-4, got " + stage); + } + this.stage = stage; + } + + public int stage() { + return stage; + } + + public long create(String name, String email) { + return withRetryForConcurrentDdl(() -> { + KeyHolder keyHolder = new GeneratedKeyHolder(); + switch (stage) { + case 1 -> jdbc.sql("INSERT INTO customers(name, email) VALUES (?, ?)") + .param(name).param(email) + .update(keyHolder, "id"); + case 2, 3 -> jdbc.sql("INSERT INTO customers(name, email, email_address) VALUES (?, ?, ?)") + .param(name).param(email).param(email) + .update(keyHolder, "id"); + case 4 -> jdbc.sql("INSERT INTO customers(name, email_address) VALUES (?, ?)") + .param(name).param(email) + .update(keyHolder, "id"); + default -> throw new IllegalStateException(); + } + return keyHolder.getKey().longValue(); + }); + } + + public void updateEmail(long id, String newEmail) { + int updated = withRetryForConcurrentDdl(() -> switch (stage) { + case 1 -> jdbc.sql("UPDATE customers SET email = ? WHERE id = ?") + .param(newEmail).param(id).update(); + case 2, 3 -> jdbc.sql("UPDATE customers SET email = ?, email_address = ? WHERE id = ?") + .param(newEmail).param(newEmail).param(id).update(); + case 4 -> jdbc.sql("UPDATE customers SET email_address = ? WHERE id = ?") + .param(newEmail).param(id).update(); + default -> throw new IllegalStateException(); + }); + if (updated == 0) { + throw new CustomerNotFoundException(id); + } + } + + public Optional findById(long id) { + // Stage 3 reads COALESCE(email_address, email) rather than email_address alone. + // Without it, a row written by a Stage 1 instance during the rollout window + // between the expand migration and the dual-write deploy - one that has never + // been dual-written at all - reads back with a null email the moment a Stage 3 + // instance answers the request. Stage 2 never needs this: it still reads the + // original column, which every stage always keeps populated. See + // docs/07-the-backfill-window-bug.md, where this line is the fix for a test + // that fails without it. + // + // A read can hit the same DDL lock window Deploy 4b's writes can - see the + // Javadoc on withRetryForConcurrentDdl below - so it gets the same retry. + String sql = switch (stage) { + case 1, 2 -> "SELECT id, name, email AS email FROM customers WHERE id = ?"; + case 3 -> "SELECT id, name, COALESCE(email_address, email) AS email FROM customers WHERE id = ?"; + case 4 -> "SELECT id, name, email_address AS email FROM customers WHERE id = ?"; + default -> throw new IllegalStateException(); + }; + return withRetryForConcurrentDdl(() -> jdbc.sql(sql).param(id) + .query(Customer.class) + .optional()); + } + + /** + * Deploy 4b (the DROP COLUMN migration) is the one deploy in this whole sequence + * that is NOT invisible to concurrent traffic on this database: for roughly the + * duration of that single ALTER TABLE statement, H2's TCP server can answer a + * completely unrelated, already-correct query with "Table CUSTOMERS not found" - + * caught live, twice, in independent runs of this module's own load generator. + * That is a real, narrow, and transient condition, not a bug in this class's SQL, + * so it gets a single scoped retry rather than being allowed to fail the request. + * See docs/14-the-ddl-lock-window.md for the captured stack trace and why this is + * not the same thing as retrying a genuine programming error. + */ + private T withRetryForConcurrentDdl(Supplier operation) { + try { + return operation.get(); + } catch (BadSqlGrammarException ex) { + if (ex.getMessage() != null && ex.getMessage().contains("CUSTOMERS")) { + try { + Thread.sleep(50); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + } + return operation.get(); + } + throw ex; + } + } + + public static class CustomerNotFoundException extends RuntimeException { + public CustomerNotFoundException(long id) { + super("No customer with id " + id); + } + } +} diff --git a/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/diag/DrainController.java b/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/diag/DrainController.java new file mode 100644 index 0000000..a725327 --- /dev/null +++ b/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/diag/DrainController.java @@ -0,0 +1,37 @@ +package com.ankurm.expandcontract.diag; + +import org.springframework.boot.availability.AvailabilityChangeEvent; +import org.springframework.boot.availability.ReadinessState; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * The step a rolling deploy needs that {@code server.shutdown: graceful} alone does + * not give you: a way to tell the load balancer "stop sending me new work" BEFORE + * the process is asked to stop. Graceful shutdown only starts refusing new + * connections once SIGTERM has already been sent - which is too late if your load + * balancer's health check has a polling interval, because every request already in + * flight to this instance in that interval gets a connection reset. Publishing + * {@link ReadinessState#REFUSING_TRAFFIC} flips {@code /actuator/health}'s readiness + * group before the process is touched at all, giving the health-checked pool one or + * two poll cycles to route around this instance first. This is the same event + * Kubernetes-style {@code preStop} hooks publish - see + * docs/13-graceful-shutdown-vs-kill-9.md, and {@code scripts/stop-instance.sh}, + * which calls this and sleeps before sending SIGTERM. + */ +@RestController +public class DrainController { + + private final ApplicationEventPublisher events; + + public DrainController(ApplicationEventPublisher events) { + this.events = events; + } + + @PostMapping("/admin/drain") + public String drain() { + AvailabilityChangeEvent.publish(events, this, ReadinessState.REFUSING_TRAFFIC); + return "draining"; + } +} diff --git a/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/diag/SchemaDiagnosticsController.java b/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/diag/SchemaDiagnosticsController.java new file mode 100644 index 0000000..78cfaf8 --- /dev/null +++ b/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/diag/SchemaDiagnosticsController.java @@ -0,0 +1,56 @@ +package com.ankurm.expandcontract.diag; + +import com.ankurm.expandcontract.customer.CustomerService; +import org.springframework.jdbc.core.simple.JdbcClient; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Prints the live state that the rest of this article can only describe: exactly + * which columns "customers" has right now, and how many rows have each column + * populated. This is what makes it possible to watch the schema actually expand and + * then actually contract, rather than take the article's word for it. Delete this + * before shipping to a real production service - see docs/15-production-checklist.md. + */ +@RestController +public class SchemaDiagnosticsController { + + private final JdbcClient jdbc; + private final CustomerService customerService; + + public SchemaDiagnosticsController(JdbcClient jdbc, CustomerService customerService) { + this.jdbc = jdbc; + this.customerService = customerService; + } + + @GetMapping("/diag/schema") + public Map schema() { + List columns = columnsOf(); + + Map result = new LinkedHashMap<>(); + result.put("appStage", customerService.stage()); + result.put("columns", columns); + result.put("rowCount", jdbc.sql("SELECT COUNT(*) FROM customers").query(Long.class).single()); + + if (columns.contains("EMAIL")) { + result.put("rowsWithEmail", + jdbc.sql("SELECT COUNT(*) FROM customers WHERE email IS NOT NULL").query(Long.class).single()); + } + if (columns.contains("EMAIL_ADDRESS")) { + result.put("rowsWithEmailAddress", + jdbc.sql("SELECT COUNT(*) FROM customers WHERE email_address IS NOT NULL").query(Long.class).single()); + } + return result; + } + + private List columnsOf() { + return jdbc.sql("SELECT column_name FROM information_schema.columns " + + "WHERE table_name = 'CUSTOMERS' ORDER BY ordinal_position") + .query(String.class) + .list(); + } +} diff --git a/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/loadgen/LoadGenerator.java b/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/loadgen/LoadGenerator.java new file mode 100644 index 0000000..565a7ce --- /dev/null +++ b/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/loadgen/LoadGenerator.java @@ -0,0 +1,374 @@ +package com.ankurm.expandcontract.loadgen; + +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A load generator that plays the part of a load balancer's client traffic during a + * rolling deploy. It never talks to an instance its own health check just marked + * unhealthy - exactly what a real load balancer's target group does - so the "zero + * errors" claim in this article is a claim about client-observed traffic through a + * health-checked pool of replicas, not a claim that no individual instance ever goes + * down. Individual instances go down constantly; that is what a rolling deploy is. + *

+ * Every request is tagged with whatever deploy phase {@code scripts/run-all.sh} has + * currently written to the phase file, so the final report breaks errors down by + * phase. See docs/11-the-load-generator.md for the workload mix and the consistency + * check it performs on every read. + *

+ * Each worker thread keeps its OWN pool of customer ids it created - never shared + * with the other threads. An earlier version shared one pool across all threads and + * produced "read-consistency-mismatch" errors that had nothing to do with the + * server: two threads racing to update the same shared id could leave the pool + * holding a stale expected value. Giving each thread exclusive ownership of the rows + * it creates removes that class of bug entirely, while still hammering both server + * replicas concurrently from multiple independent threads - see + * docs/11-the-load-generator.md. + */ +public final class LoadGenerator { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + record KnownCustomer(long id, String email) { + } + + public static void main(String[] args) throws Exception { + List ports = List.of(); + int durationSeconds = 120; + Path phaseFile = Path.of("/tmp/ec-demo/phase.txt"); + Path outFile = Path.of("/tmp/ec-demo/load-summary.txt"); + int threads = 8; + + for (String arg : args) { + if (arg.startsWith("--ports=")) { + ports = List.of(arg.substring("--ports=".length()).split(",")).stream().map(Integer::parseInt).toList(); + } else if (arg.startsWith("--durationSeconds=")) { + durationSeconds = Integer.parseInt(arg.substring("--durationSeconds=".length())); + } else if (arg.startsWith("--phaseFile=")) { + phaseFile = Path.of(arg.substring("--phaseFile=".length())); + } else if (arg.startsWith("--outFile=")) { + outFile = Path.of(arg.substring("--outFile=".length())); + } else if (arg.startsWith("--threads=")) { + threads = Integer.parseInt(arg.substring("--threads=".length())); + } + } + if (ports.isEmpty()) { + throw new IllegalArgumentException("--ports=8081,8082 is required"); + } + + new LoadGenerator(ports, durationSeconds, phaseFile, outFile, threads).run(); + } + + private final List ports; + private final int durationSeconds; + private final Path phaseFile; + private final Path outFile; + private final int threads; + + private final HttpClient http = HttpClient.newBuilder().connectTimeout(Duration.ofMillis(600)).build(); + private final Map healthy = new ConcurrentHashMap<>(); + private final Map consecutiveFailures = new ConcurrentHashMap<>(); + private final Map byPhase = new ConcurrentHashMap<>(); + private final AtomicBoolean stop = new AtomicBoolean(false); + + // A backend needs two consecutive failed checks before it is removed from + // rotation - one slow response under transient CPU contention (a sibling + // replica's JVM starting up on this box's limited cores) should not look like + // an outage. This mirrors how a real load balancer's health check threshold + // works, and removing it is what turned transient slowness into false + // "no-healthy-backend" errors during earlier runs - see docs/11-the-load-generator.md. + private static final int UNHEALTHY_THRESHOLD = 2; + + private static final class Counters { + final AtomicLong ok = new AtomicLong(); + final AtomicLong error = new AtomicLong(); + final Map errorReasons = new ConcurrentHashMap<>(); + + void recordError(String reason) { + error.incrementAndGet(); + errorReasons.computeIfAbsent(reason, r -> new AtomicLong()).incrementAndGet(); + } + } + + LoadGenerator(List ports, int durationSeconds, Path phaseFile, Path outFile, int threads) { + this.ports = ports; + this.durationSeconds = durationSeconds; + this.phaseFile = phaseFile; + this.outFile = outFile; + this.threads = threads; + ports.forEach(p -> { + healthy.put(p, false); + consecutiveFailures.put(p, 0); + }); + } + + void run() throws Exception { + if (outFile.getParent() != null) { + Files.createDirectories(outFile.getParent()); + } + + ScheduledExecutorService healthChecker = Executors.newSingleThreadScheduledExecutor(); + healthChecker.scheduleAtFixedRate(this::checkHealth, 0, 300, TimeUnit.MILLISECONDS); + + // Wait for the first health check pass to land before starting any worker. + // Without this, every worker's opening requests race the very first check - + // the "healthy" map starts all-false by construction - and each one counts as + // a spurious "no-healthy-backend" for a server that was up the whole time. + for (int i = 0; i < 100 && ports.stream().noneMatch(p -> Boolean.TRUE.equals(healthy.get(p))); i++) { + Thread.sleep(50); + } + + var workers = Executors.newFixedThreadPool(threads); + CountDownLatch done = new CountDownLatch(threads); + for (int i = 0; i < threads; i++) { + workers.submit(() -> { + try { + workerLoop(new ArrayDeque<>()); + } finally { + done.countDown(); + } + }); + } + + Instant deadline = Instant.now().plusSeconds(durationSeconds); + Instant lastLog = Instant.now(); + while (Instant.now().isBefore(deadline)) { + Thread.sleep(1000); + if (Duration.between(lastLog, Instant.now()).getSeconds() >= 10) { + logProgress(); + lastLog = Instant.now(); + } + } + stop.set(true); + done.await(10, TimeUnit.SECONDS); + workers.shutdownNow(); + healthChecker.shutdownNow(); + + writeSummary(); + } + + private void checkHealth() { + for (int port : ports) { + boolean up; + try { + HttpRequest req = HttpRequest.newBuilder(URI.create("http://localhost:" + port + "/actuator/health")) + .timeout(Duration.ofMillis(800)).GET().build(); + HttpResponse resp = http.send(req, HttpResponse.BodyHandlers.ofString()); + up = resp.statusCode() == 200 && resp.body().contains("\"UP\""); + } catch (Exception e) { + up = false; + } + if (up) { + consecutiveFailures.put(port, 0); + healthy.put(port, true); + } else { + int failures = consecutiveFailures.merge(port, 1, Integer::sum); + if (failures >= UNHEALTHY_THRESHOLD) { + healthy.put(port, false); + } + } + } + } + + private int pickHealthyPort() { + List up = ports.stream().filter(p -> Boolean.TRUE.equals(healthy.get(p))).toList(); + if (up.isEmpty()) { + return -1; + } + return up.get(ThreadLocalRandom.current().nextInt(up.size())); + } + + private String currentPhase() { + try { + if (Files.exists(phaseFile)) { + String s = Files.readString(phaseFile).trim(); + if (!s.isEmpty()) { + return s; + } + } + } catch (IOException ignored) { + } + return "unphased"; + } + + private void workerLoop(Deque myKnown) { + while (!stop.get()) { + String phase = currentPhase(); + Counters c = byPhase.computeIfAbsent(phase, p -> new Counters()); + int port = pickHealthyPort(); + if (port == -1) { + c.recordError("no-healthy-backend"); + sleepJitter(); + continue; + } + try { + double roll = ThreadLocalRandom.current().nextDouble(); + if (roll < 0.5 || myKnown.isEmpty()) { + doCreate(port, c, myKnown); + } else if (roll < 0.8) { + doRead(port, c, myKnown); + } else { + doUpdate(port, c, myKnown); + } + } catch (Exception e) { + c.recordError(e.getClass().getSimpleName()); + } + sleepJitter(); + } + } + + private void sleepJitter() { + try { + Thread.sleep(ThreadLocalRandom.current().nextInt(15, 40)); + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } + } + + private void doCreate(int port, Counters c, Deque myKnown) throws Exception { + long n = ThreadLocalRandom.current().nextLong(1_000_000_000L); + String email = "customer" + n + "@example.test"; + String name = "Customer " + n; + String body = MAPPER.writeValueAsString(Map.of("name", name, "email", email)); + HttpRequest req = HttpRequest.newBuilder(URI.create("http://localhost:" + port + "/customers")) + .timeout(Duration.ofSeconds(2)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(body)) + .build(); + HttpResponse resp = http.send(req, HttpResponse.BodyHandlers.ofString()); + if (resp.statusCode() != 201) { + c.recordError("create-http-" + resp.statusCode()); + return; + } + JsonNode node = MAPPER.readTree(resp.body()); + long id = node.get("id").asLong(); + String returnedEmail = node.get("email").asString(); + if (!email.equals(returnedEmail)) { + c.recordError("create-echo-mismatch"); + return; + } + myKnown.addLast(new KnownCustomer(id, email)); + while (myKnown.size() > 500) { + myKnown.pollFirst(); + } + c.ok.incrementAndGet(); + } + + private void doRead(int port, Counters c, Deque myKnown) throws Exception { + KnownCustomer target = pickKnown(myKnown); + if (target == null) { + return; + } + HttpRequest req = HttpRequest.newBuilder(URI.create("http://localhost:" + port + "/customers/" + target.id())) + .timeout(Duration.ofSeconds(2)).GET().build(); + HttpResponse resp = http.send(req, HttpResponse.BodyHandlers.ofString()); + if (resp.statusCode() != 200) { + c.recordError("read-http-" + resp.statusCode()); + return; + } + JsonNode node = MAPPER.readTree(resp.body()); + String returnedEmail = node.get("email").asString(); + if (!target.email().equals(returnedEmail)) { + c.recordError("read-consistency-mismatch"); + return; + } + c.ok.incrementAndGet(); + } + + private void doUpdate(int port, Counters c, Deque myKnown) throws Exception { + KnownCustomer target = pickKnown(myKnown); + if (target == null) { + return; + } + long n = ThreadLocalRandom.current().nextLong(1_000_000_000L); + String newEmail = "updated" + n + "@example.test"; + String body = MAPPER.writeValueAsString(Map.of("email", newEmail)); + HttpRequest req = HttpRequest.newBuilder(URI.create("http://localhost:" + port + "/customers/" + target.id() + "/email")) + .timeout(Duration.ofSeconds(2)) + .header("Content-Type", "application/json") + .PUT(HttpRequest.BodyPublishers.ofString(body)) + .build(); + HttpResponse resp = http.send(req, HttpResponse.BodyHandlers.ofString()); + if (resp.statusCode() != 200) { + c.recordError("update-http-" + resp.statusCode()); + return; + } + JsonNode node = MAPPER.readTree(resp.body()); + String returnedEmail = node.get("email").asString(); + if (!newEmail.equals(returnedEmail)) { + c.recordError("update-echo-mismatch"); + return; + } + // This deque is private to the calling thread - no other thread ever reads + // or writes this id, so there is no race to guard against here. + myKnown.remove(target); + myKnown.addLast(new KnownCustomer(target.id(), newEmail)); + c.ok.incrementAndGet(); + } + + private KnownCustomer pickKnown(Deque myKnown) { + int size = myKnown.size(); + if (size == 0) { + return null; + } + int skip = ThreadLocalRandom.current().nextInt(size); + var it = myKnown.iterator(); + KnownCustomer last = null; + for (int i = 0; i <= skip && it.hasNext(); i++) { + last = it.next(); + } + return last; + } + + private void logProgress() { + long okTotal = byPhase.values().stream().mapToLong(c -> c.ok.get()).sum(); + long errTotal = byPhase.values().stream().mapToLong(c -> c.error.get()).sum(); + System.out.printf("[%s] phase=%-24s ok=%d error=%d%n", Instant.now(), currentPhase(), okTotal, errTotal); + } + + private void writeSummary() throws IOException { + StringBuilder sb = new StringBuilder(); + sb.append("Load generator summary\n"); + sb.append("=======================\n"); + long okTotal = 0, errTotal = 0; + for (var entry : byPhase.entrySet()) { + okTotal += entry.getValue().ok.get(); + errTotal += entry.getValue().error.get(); + } + sb.append(String.format("Total requests: %d%n", okTotal + errTotal)); + sb.append(String.format("Successful: %d%n", okTotal)); + sb.append(String.format("Errors: %d%n", errTotal)); + sb.append("\nBy phase:\n"); + for (var entry : byPhase.entrySet()) { + Counters c = entry.getValue(); + sb.append(String.format(" %-28s ok=%-8d error=%-6d%n", entry.getKey(), c.ok.get(), c.error.get())); + for (var reason : c.errorReasons.entrySet()) { + sb.append(String.format(" - %-24s %d%n", reason.getKey(), reason.getValue().get())); + } + } + Files.writeString(outFile, sb.toString()); + System.out.print(sb); + } +} diff --git a/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/migration/MigrationCli.java b/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/migration/MigrationCli.java new file mode 100644 index 0000000..3fb4202 --- /dev/null +++ b/db-migrations-expand-contract/src/main/java/com/ankurm/expandcontract/migration/MigrationCli.java @@ -0,0 +1,65 @@ +package com.ankurm.expandcontract.migration; + +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.MigrationInfo; + +/** + * A migration runner that is deliberately NOT part of the Spring Boot application. + *

+ * This is the mechanical heart of the article's argument: schema changes and code + * deploys are two different kinds of event, so they get two different delivery + * mechanisms. The app (see {@link com.ankurm.expandcontract.ExpandContractApplication}) + * never touches Flyway - {@code spring.flyway.enabled=false} in application.yml sees to + * that. This class runs standalone, against the same JDBC URL, and takes a + * {@code --target} version so a "deploy" can migrate exactly as far as that step of + * the sequence requires and no further. + *

+ * See docs/03-why-migrations-run-outside-the-app.md. + */ +public final class MigrationCli { + + public static void main(String[] args) { + String target = "latest"; + for (String arg : args) { + if (arg.startsWith("--target=")) { + target = arg.substring("--target=".length()); + } + } + + // Same standalone TCP server the app connects to (see application.yml) - migrations + // run against the live database over the network, exactly like the app does, and + // like a real migration job in a CI pipeline would. + String tcpPort = System.getenv().getOrDefault("EC_DB_TCP_PORT", "9092"); + String dbName = System.getenv().getOrDefault("EC_DB_NAME", "expand-contract"); + String url = "jdbc:h2:tcp://localhost:" + tcpPort + "/" + dbName; + + Flyway flyway = Flyway.configure() + .dataSource(url, "sa", "") + .locations("classpath:db/migration") + .target(target) + .load(); + + System.out.println("=== Before migrate (target=" + target + ") ==="); + printInfo(flyway); + + var result = flyway.migrate(); + + System.out.println("=== After migrate ==="); + printInfo(flyway); + System.out.println("Migrations executed: " + result.migrationsExecuted + + ", target schema version: " + result.targetSchemaVersion + + ", success: " + result.success); + } + + private static void printInfo(Flyway flyway) { + for (MigrationInfo info : flyway.info().all()) { + System.out.printf(" %-8s %-40s %-10s%n", + info.getVersion() == null ? "-" : info.getVersion().getVersion(), + info.getDescription(), + info.getState()); + } + } + + private MigrationCli() { + } +} diff --git a/db-migrations-expand-contract/src/main/resources/application.yml b/db-migrations-expand-contract/src/main/resources/application.yml new file mode 100644 index 0000000..09f3f53 --- /dev/null +++ b/db-migrations-expand-contract/src/main/resources/application.yml @@ -0,0 +1,48 @@ +spring: + application: + name: expand-contract-demo + datasource: + # A real TCP connection to a standalone H2 server process (scripts/start-db-server.sh), + # never a file this app opens itself. An earlier version used + # "jdbc:h2:file:...;AUTO_SERVER=TRUE" so the two replicas could share one database file + # directly - which works only as long as neither replica is ever restarted, because + # AUTO_SERVER silently makes the FIRST process to open the file the de facto database + # server for every other process that connects to it afterwards. Killing that one + # replica during a routine rolling deploy took the "shared database" down with it - see + # docs/12-the-auto-server-trap.md. A real production database is its own process for + # exactly this reason. + url: jdbc:h2:tcp://localhost:${EC_DB_TCP_PORT:9092}/${EC_DB_NAME:expand-contract} + username: sa + password: "" + driver-class-name: org.h2.Driver + flyway: + # Migrations are NOT run by the application on startup. The whole point of + # expand-contract is that schema changes and app deploys are independent + # events - see migration.MigrationCli, driven by scripts/migrate.sh. + enabled: false + +# The stage this instance is running as: 1 (baseline), 2 (dual-write), 3 (read-new), +# 4 (contract - new column only). Passed on the command line per instance, e.g. +# --app.stage=2, so two replicas can run different stages during a rolling deploy. +app: + stage: ${APP_STAGE:1} + +management: + endpoints: + web: + exposure: + include: health,info + endpoint: + health: + show-details: always + health: + defaults: + enabled: true + +server: + port: ${SERVER_PORT:8080} + shutdown: graceful + +logging: + level: + root: INFO diff --git a/db-migrations-expand-contract/src/main/resources/db/migration/V1__create_customer.sql b/db-migrations-expand-contract/src/main/resources/db/migration/V1__create_customer.sql new file mode 100644 index 0000000..53fe37e --- /dev/null +++ b/db-migrations-expand-contract/src/main/resources/db/migration/V1__create_customer.sql @@ -0,0 +1,8 @@ +-- Deploy 0 (baseline, already in production before this article starts). +-- A customers table with a single "email" column, the thing we are about to rename. +CREATE TABLE customers ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(200) NOT NULL, + email VARCHAR(320) NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); diff --git a/db-migrations-expand-contract/src/main/resources/db/migration/V2__add_email_address_column.sql b/db-migrations-expand-contract/src/main/resources/db/migration/V2__add_email_address_column.sql new file mode 100644 index 0000000..33f3827 --- /dev/null +++ b/db-migrations-expand-contract/src/main/resources/db/migration/V2__add_email_address_column.sql @@ -0,0 +1,18 @@ +-- Deploy 1 (EXPAND). Additive and nullable, so it is compatible with every piece +-- of app code that is running right now: the old code never mentions this column +-- and will not notice it exists. This migration runs against the live database +-- with zero application deploy and zero restart of any replica. +ALTER TABLE customers ADD COLUMN email_address VARCHAR(320); + +-- Backfill every row that existed before dual-write code shipped. Rows created +-- *after* this point but before the dual-write code (Deploy 2) is fully rolled +-- out are handled separately - see docs/07-the-backfill-window-bug.md. +UPDATE customers SET email_address = email WHERE email_address IS NULL; + +-- The other half of "expand": relax the constraint on the column we are about to +-- retire. "email" is NOT NULL from V1. Leave that in place and Deploy 4's Stage 4 +-- code - which never writes "email" - fails every single INSERT with a NOT NULL +-- violation the moment it starts, because the column it ignores is still mandatory. +-- Forgetting this line is a genuine, reproducible failure - see +-- docs/06-the-not-null-trap.md, which captures the exact exception it produces. +ALTER TABLE customers ALTER COLUMN email DROP NOT NULL; diff --git a/db-migrations-expand-contract/src/main/resources/db/migration/V3__drop_email_column.sql b/db-migrations-expand-contract/src/main/resources/db/migration/V3__drop_email_column.sql new file mode 100644 index 0000000..0c01c68 --- /dev/null +++ b/db-migrations-expand-contract/src/main/resources/db/migration/V3__drop_email_column.sql @@ -0,0 +1,5 @@ +-- Deploy 4 (CONTRACT). Only safe once every replica in the fleet is confirmed +-- running Stage 4 code, which never reads or writes "email". Run this too early +-- and any Stage 1/2/3 instance still in the rolling deploy fails on its next +-- write - see docs/10-what-happens-if-you-drop-too-soon.md. +ALTER TABLE customers DROP COLUMN email; diff --git a/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/BackfillWindowBugTest.java b/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/BackfillWindowBugTest.java new file mode 100644 index 0000000..9482a6d --- /dev/null +++ b/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/BackfillWindowBugTest.java @@ -0,0 +1,68 @@ +package com.ankurm.expandcontract; + +import com.ankurm.expandcontract.customer.Customer; +import com.ankurm.expandcontract.customer.CustomerService; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.jdbc.core.simple.JdbcClient; + +import java.nio.file.Path; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The gap between "the expand migration ran" and "every Stage 1 instance in the fleet + * has been replaced by Stage 2" is not instantaneous - a real rolling deploy takes + * minutes, and every row a lingering Stage 1 instance writes during that window has + * {@code email_address = NULL}. The migration's one-time backfill (V2) only ever sees + * rows that existed *before* Deploy 1 ran; it cannot see rows a Stage 1 instance writes + * *after* that, during its own rollout window. + *

+ * This test reproduces the resulting bug with a naive Stage 3 read (email_address alone) + * and then shows the fix that ships in + * {@link com.ankurm.expandcontract.customer.CustomerService#findById}: a plain + * {@code COALESCE(email_address, email)}. See docs/07-the-backfill-window-bug.md. + */ +class BackfillWindowBugTest { + + @Test + void naiveStage3ReadReturnsNullForARowStage1WroteDuringTheRollout(@TempDir Path tmp) throws Exception { + Path db = tmp.resolve("backfill-window"); + TestSupport.migrateTo(db, "2"); + Transcript t = Transcript.start("07-backfill-window-bug", + "The backfill window: a Stage 1 write after Deploy 1, read by a naive Stage 3"); + + // A Stage 1 instance is still serving traffic during the Deploy 2 rollout and + // writes a brand new row exactly as it always has - it has never heard of + // email_address. This is not a hypothetical; it is guaranteed to happen for + // however long the rollout takes. + JdbcClient jdbc = TestSupport.jdbcClient(db); + jdbc.sql("INSERT INTO customers(name, email) VALUES (?, ?)") + .param("Katherine Johnson").param("katherine@example.test") + .update(); + long insertedId = jdbc.sql("SELECT id FROM customers WHERE name = ?") + .param("Katherine Johnson").query(Long.class).single(); + + t.section("the row a lingering Stage 1 instance just wrote"); + String naiveRead = jdbc.sql("SELECT name, email, email_address FROM customers WHERE id = ?") + .param(insertedId).query().listOfRows().toString(); + t.line(naiveRead); + assertThat(naiveRead).containsIgnoringCase("email_address=null"); + + t.section("a NAIVE Stage 3 read (email_address alone) - the bug"); + String naiveEmail = jdbc.sql("SELECT email_address FROM customers WHERE id = ?") + .param(insertedId).query(String.class).optional().orElse(null); + t.line("naive Stage 3 email column value: " + naiveEmail); + assertThat(naiveEmail).isNull(); + + t.section("the SHIPPED Stage 3 read (CustomerService, COALESCE) - the fix"); + CustomerService stage3 = new CustomerService(jdbc, 3); + Optional fixed = stage3.findById(insertedId); + t.line("CustomerService (stage 3) result: " + fixed); + assertThat(fixed).isPresent(); + assertThat(fixed.get().email()).isEqualTo("katherine@example.test"); + + t.write(); + } +} diff --git a/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/ContractSafetyTest.java b/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/ContractSafetyTest.java new file mode 100644 index 0000000..c3a9c24 --- /dev/null +++ b/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/ContractSafetyTest.java @@ -0,0 +1,85 @@ +package com.ankurm.expandcontract; + +import com.ankurm.expandcontract.customer.Customer; +import com.ankurm.expandcontract.customer.CustomerService; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.jdbc.core.simple.JdbcClient; + +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Deploy 4 (CONTRACT), in both directions. First the case that must work: once every + * replica is confirmed on Stage 4 and the drop migration (V3) has run, Stage 4 code + * keeps working exactly as before - it never referenced "email" to begin with. Second + * the case that must fail loudly: a Stage 1, 2 or 3 instance that is somehow still + * running against the post-drop schema (a rollback gone wrong, a forgotten canary) + * gets a real SQL error the moment it tries to touch the column that no longer exists. + * That failure is not a bug in this article's design - it is *why* Deploy 4 has to wait + * for confirmation that the fleet is 100% on Stage 4 first. See + * docs/09-the-contract-migration.md and docs/10-what-happens-if-you-drop-too-soon.md. + */ +class ContractSafetyTest { + + @Test + void stage4KeepsWorkingAfterTheColumnIsDropped(@TempDir Path tmp) throws Exception { + Path db = tmp.resolve("contract-ok"); + TestSupport.migrateTo(db, "2"); + JdbcClient jdbc = TestSupport.jdbcClient(db); + Transcript t = Transcript.start("09-contract-safety", + "Deploy 4 (CONTRACT): Stage 4 after the drop, and what breaks if you drop too soon"); + + CustomerService stage4 = new CustomerService(jdbc, 4); + long id = stage4.create("Annie Easley", "annie@example.test"); + + // Deploy 4b: drop the old column, live, with Stage 4 already the only code running. + TestSupport.migrateTo(db, "latest"); + + Customer afterDrop = stage4.findById(id).orElseThrow(); + t.section("Stage 4 read, after V3 dropped the email column"); + t.line(afterDrop.toString()); + assertThat(afterDrop.email()).isEqualTo("annie@example.test"); + + long postDropId = stage4.create("Mary Allen Wilkes", "mary@example.test"); + Customer postDrop = stage4.findById(postDropId).orElseThrow(); + t.section("Stage 4 create + read, entirely after the drop"); + t.line(postDrop.toString()); + assertThat(postDrop.email()).isEqualTo("mary@example.test"); + + t.write(); + } + + @Test + void stage1CodeFailsLoudlyIfItIsStillRunningAfterTheColumnIsDropped(@TempDir Path tmp) throws Exception { + Path db = tmp.resolve("contract-too-soon"); + TestSupport.migrateTo(db, "latest"); + JdbcClient jdbc = TestSupport.jdbcClient(db); + Transcript t = Transcript.start("10-drop-too-soon", + "What a lingering Stage 1 instance sees if the drop runs before it is retired"); + + CustomerService stage1 = new CustomerService(jdbc, 1); + + Throwable thrown = catchThrowable(() -> stage1.create("Too Late", "too.late@example.test")); + Throwable root = thrown; + while (root.getCause() != null) { + root = root.getCause(); + } + t.line("Stage 1 create() after V3 dropped \"email\": " + thrown.getClass().getName()); + t.line("message: " + thrown.getMessage()); + t.line("root cause: " + root.getClass().getName() + ": " + root.getMessage()); + assertThat(root.getMessage()).contains("EMAIL").containsIgnoringCase("not found"); + + t.write(); + } + + private static Throwable catchThrowable(org.assertj.core.api.ThrowableAssert.ThrowingCallable callable) { + try { + callable.call(); + return new AssertionError("expected an exception but none was thrown"); + } catch (Throwable t) { + return t; + } + } +} diff --git a/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/DbDump.java b/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/DbDump.java new file mode 100644 index 0000000..9188183 --- /dev/null +++ b/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/DbDump.java @@ -0,0 +1,72 @@ +package com.ankurm.expandcontract; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; + +/** Renders a JDBC query as a plain-text table for transcripts - no ORM, no formatting library. */ +public final class DbDump { + + private DbDump() { + } + + public static String table(Connection conn, String sql) { + try (Statement st = conn.createStatement(); ResultSet rs = st.executeQuery(sql)) { + ResultSetMetaData meta = rs.getMetaData(); + int cols = meta.getColumnCount(); + List headers = new ArrayList<>(); + for (int i = 1; i <= cols; i++) { + headers.add(meta.getColumnLabel(i)); + } + List> rows = new ArrayList<>(); + while (rs.next()) { + List row = new ArrayList<>(); + for (int i = 1; i <= cols; i++) { + Object v = rs.getObject(i); + row.add(v == null ? "NULL" : v.toString()); + } + rows.add(row); + } + int[] widths = new int[cols]; + for (int i = 0; i < cols; i++) { + widths[i] = headers.get(i).length(); + } + for (List row : rows) { + for (int i = 0; i < cols; i++) { + widths[i] = Math.max(widths[i], row.get(i).length()); + } + } + StringBuilder sb = new StringBuilder(); + appendRow(sb, headers, widths); + StringBuilder sep = new StringBuilder(); + for (int i = 0; i < cols; i++) { + sep.append("-".repeat(widths[i])).append(i < cols - 1 ? "-+-" : ""); + } + sb.append(sep).append(System.lineSeparator()); + for (List row : rows) { + appendRow(sb, row, widths); + } + sb.append("(").append(rows.size()).append(" row").append(rows.size() == 1 ? "" : "s").append(")"); + return sb.toString(); + } + catch (SQLException ex) { + return "query failed: " + ex.getMessage(); + } + } + + private static void appendRow(StringBuilder sb, List values, int[] widths) { + for (int i = 0; i < values.size(); i++) { + sb.append(pad(values.get(i), widths[i])); + sb.append(i < values.size() - 1 ? " | " : ""); + } + sb.append(System.lineSeparator()); + } + + private static String pad(String s, int width) { + return s + " ".repeat(Math.max(0, width - s.length())); + } +} diff --git a/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/DdlSilentDataLossTest.java b/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/DdlSilentDataLossTest.java new file mode 100644 index 0000000..594289e --- /dev/null +++ b/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/DdlSilentDataLossTest.java @@ -0,0 +1,132 @@ +package com.ankurm.expandcontract; + +import com.ankurm.expandcontract.customer.CustomerService; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.jdbc.core.simple.JdbcClient; + +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The finding that {@link com.ankurm.expandcontract.customer.CustomerService#withRetryForConcurrentDdl} + * cannot fix, because nothing throws. The retry helper exists for the case where a concurrent + * statement collides with an in-progress {@code ALTER TABLE} and H2 answers with + * "table not found" - a real error the caller can see and retry. This test demonstrates a second, + * stranger failure mode found the same way the first one was: by running this module's own load + * generator against a live rolling deploy and noticing a handful of reads and updates come back + * 404 for a customer id that a 201 response had already confirmed existed. + *

+ * H2 implements both {@code ALTER TABLE ... ADD COLUMN} and {@code ALTER TABLE ... DROP COLUMN} + * by rebuilding the table: copying every row into a new table with the new column layout and + * swapping it in. If an ordinary {@code INSERT} on another connection commits - with no error, + * with a generated key handed back to the caller - while that rebuild is in flight, the inserted + * row can be copied into the new table or left behind in the old one depending on exactly when + * the rebuild's internal scan ran relative to the commit. When it is left behind, the row is gone + * the instant the rebuild finishes, and nothing on the inserting connection was ever told. + *

+ * This is not how every database implements {@code ADD COLUMN} and {@code DROP COLUMN}. PostgreSQL's + * reference manual is explicit that both are metadata-only operations on tables like this one - + * no non-volatile default and no immediate space reclamation requested - so this is a property of + * H2's implementation, not of the expand-contract technique itself. See + * docs/14-the-ddl-lock-window.md for the full explanation, the standalone reproduction this test + * is built from, and what it implies for choosing a target database for a real migration. + */ +class DdlSilentDataLossTest { + + // Whether the rebuild's internal scan actually passes a given row before or after that + // row's INSERT commits is OS scheduling, not application logic - so a single attempt at + // this race can genuinely land on either side of it. A single-attempt version of this test + // failed about 1 run in 5 while writing it. Rather than assert on one attempt (flaky either + // way) or loosen the assertion to "zero or more" (which would silently stop proving anything + // the day this stops reproducing), this test repeats the race, on a fresh table each time, + // until it reproduces - the same thing a human would do at a terminal to confirm a suspected + // race is real. Twenty attempts reproduced it within the first 4 in every run made while + // writing this test. + private static final int MAX_ATTEMPTS = 20; + + @Test + void concurrentAlterTableAddColumnCanSilentlyDropAnAlreadyCommittedInsert(@TempDir Path tmp) throws Exception { + Transcript t = Transcript.start("14-ddl-silent-data-loss", + "The failure the retry cannot catch: a committed INSERT that ALTER TABLE loses silently"); + + for (int attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + Path db = tmp.resolve("silent-data-loss-" + attempt); + TestSupport.migrateTo(db, "1"); + JdbcClient jdbc = TestSupport.jdbcClient(db); + CustomerService stage1 = new CustomerService(jdbc, 1); + + AtomicBoolean stop = new AtomicBoolean(false); + List confirmedIds = new CopyOnWriteArrayList<>(); + List insertErrors = new CopyOnWriteArrayList<>(); + CountDownLatch started = new CountDownLatch(1); + + Thread inserter = new Thread(() -> { + started.countDown(); + int n = 0; + while (!stop.get()) { + n++; + try { + long id = stage1.create("Concurrent " + n, "concurrent" + n + "@example.test"); + confirmedIds.add(id); + } catch (Exception ex) { + // The already-documented, already-fixed failure mode: a statement that + // collides with the DDL and is told so. Counted here only to show it is + // rare and separate from the silent loss this test is isolating. + insertErrors.add(ex.getClass().getSimpleName()); + } + } + }); + inserter.start(); + started.await(); + + // The DDL runs on its own connection, exactly as Deploy 1 (EXPAND) runs V2 live + // while both replicas keep taking traffic - see scripts/run-all.sh. + TestSupport.migrateTo(db, "2"); + + stop.set(true); + inserter.join(); + + int missing = 0; + List missingIds = new java.util.ArrayList<>(); + for (Long id : confirmedIds) { + if (jdbc.sql("SELECT id FROM customers WHERE id = ?").param(id).query().listOfRows().isEmpty()) { + missing++; + missingIds.add(id); + } + } + + if (missing == 0 && attempt < MAX_ATTEMPTS) { + continue; + } + + t.line("attempts needed to reproduce the race: " + attempt + " of " + MAX_ATTEMPTS); + t.line("customer creates that returned a generated id with no error: " + confirmedIds.size()); + t.line("customer creates that got the already-documented, already-fixed DDL-collision error: " + insertErrors.size()); + t.line("of the ids that came back with no error, missing from the table once V2 finished: " + missing); + if (!missingIds.isEmpty()) { + t.line("example missing ids: " + missingIds.subList(0, Math.min(5, missingIds.size()))); + } + t.line(""); + t.line("This is why the retry in CustomerService cannot be the whole fix: these inserts"); + t.line("never threw anything to retry. The row was committed, then discarded when the"); + t.line("ADD COLUMN rebuild swapped in a new table that had already been scanned."); + t.write(); + + // If this fails on the very last attempt, either H2's rebuild strategy changed + // (worth its own investigation) or this environment schedules threads differently + // enough that this test needs a heavier inserter - not that the finding is wrong. + assertThat(missing) + .withFailMessage("expected at least one silently-lost row within %d attempts; " + + "either H2's ALTER TABLE implementation changed, or this environment " + + "needs a heavier concurrent inserter to reproduce the race", MAX_ATTEMPTS) + .isGreaterThan(0); + return; + } + } +} diff --git a/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/DualWriteConsistencyTest.java b/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/DualWriteConsistencyTest.java new file mode 100644 index 0000000..1fea67a --- /dev/null +++ b/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/DualWriteConsistencyTest.java @@ -0,0 +1,49 @@ +package com.ankurm.expandcontract; + +import com.ankurm.expandcontract.customer.CustomerService; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Deploy 2 (MIGRATE WRITES): Stage 2 code writes every create and update to both + * columns. This is the deploy that makes Deploy 3's read switch safe later - if this + * one is wrong, nothing downstream can be trusted. See docs/04-the-dual-write.md. + */ +class DualWriteConsistencyTest { + + @Test + void createAndUpdatePopulateBothColumnsIdentically(@TempDir Path tmp) throws Exception { + Path db = tmp.resolve("dual-write"); + TestSupport.migrateTo(db, "2"); + Transcript t = Transcript.start("04-dual-write-consistency", + "Deploy 2 (MIGRATE WRITES): Stage 2 writes land in both columns"); + + CustomerService stage2 = new CustomerService(TestSupport.jdbcClient(db), 2); + long id = stage2.create("Margaret Hamilton", "margaret@example.test"); + + try (Connection conn = DriverManager.getConnection(TestSupport.jdbcUrl(db), "sa", "")) { + t.section("after create()"); + String row = DbDump.table(conn, "select name, email, email_address from customers where id = " + id); + t.line(row); + assertThat(row).contains("margaret@example.test"); + assertThat(row.indexOf("margaret@example.test")).isNotEqualTo(row.lastIndexOf("margaret@example.test")); + } + + stage2.updateEmail(id, "m.hamilton@example.test"); + + try (Connection conn = DriverManager.getConnection(TestSupport.jdbcUrl(db), "sa", "")) { + t.section("after updateEmail() - the old value is gone from BOTH columns, not just one"); + String row = DbDump.table(conn, "select name, email, email_address from customers where id = " + id); + t.line(row); + assertThat(row).contains("m.hamilton@example.test").doesNotContain("margaret@example.test"); + } + + t.write(); + } +} diff --git a/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/ExpandMigrationBackwardCompatibleTest.java b/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/ExpandMigrationBackwardCompatibleTest.java new file mode 100644 index 0000000..bd7ce91 --- /dev/null +++ b/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/ExpandMigrationBackwardCompatibleTest.java @@ -0,0 +1,72 @@ +package com.ankurm.expandcontract; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.jdbc.core.simple.JdbcClient; + +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Deploy 1 (EXPAND) in isolation: migrate to V2 and confirm two things a real rollout + * depends on. First, that every pre-existing row got backfilled in the same migration. + * Second - the part that is easy to get wrong - that Stage 1 code, completely unaware + * the new column exists, still inserts rows exactly as it always has. If this second + * assertion ever failed, the migration would not be additive and expand-contract would + * not apply to it. See docs/02-the-expand-migration.md. + */ +class ExpandMigrationBackwardCompatibleTest { + + @Test + void backfillsExistingRowsAndStaysCompatibleWithStage1Code(@TempDir Path tmp) throws Exception { + Path db = tmp.resolve("expand"); + Transcript t = Transcript.start("02-expand-backward-compatible", + "Deploy 1 (EXPAND): additive column + backfill, Stage 1 code untouched"); + + // Seed one row the way Stage 1 always has, before the expand migration exists. + TestSupport.migrateTo(db, "1"); + JdbcClient preExpand = TestSupport.jdbcClient(db); + preExpand.sql("INSERT INTO customers(name, email) VALUES (?, ?)") + .param("Ada Lovelace").param("ada@example.test").update(); + + t.section("schema before Deploy 1"); + try (Connection conn = DriverManager.getConnection(TestSupport.jdbcUrl(db), "sa", "")) { + t.line(DbDump.table(conn, "select column_name from information_schema.columns " + + "where table_name = 'CUSTOMERS' order by ordinal_position")); + } + + // Deploy 1: the expand migration runs against the live database. No app restart. + TestSupport.migrateTo(db, "2"); + + t.section("schema after Deploy 1 (email_address added)"); + try (Connection conn = DriverManager.getConnection(TestSupport.jdbcUrl(db), "sa", "")) { + t.line(DbDump.table(conn, "select column_name from information_schema.columns " + + "where table_name = 'CUSTOMERS' order by ordinal_position")); + + t.section("Ada's row was backfilled by the migration itself"); + String row = DbDump.table(conn, "select name, email, email_address from customers where name = 'Ada Lovelace'"); + t.line(row); + // Backfilled: both columns hold the same value for a row that predates dual-write code. + assertThat(row).contains("ada@example.test"); + assertThat(row.indexOf("ada@example.test")).isNotEqualTo(row.lastIndexOf("ada@example.test")); + } + + // Stage 1 code has not been redeployed and does not know email_address exists. + // Its insert statement is byte-for-byte what it was before Deploy 1. + JdbcClient postExpand = TestSupport.jdbcClient(db); + postExpand.sql("INSERT INTO customers(name, email) VALUES (?, ?)") + .param("Grace Hopper").param("grace@example.test").update(); + + t.section("Stage 1's original INSERT still works, unmodified, after the migration"); + try (Connection conn = DriverManager.getConnection(TestSupport.jdbcUrl(db), "sa", "")) { + String row = DbDump.table(conn, "select name, email, email_address from customers where name = 'Grace Hopper'"); + t.line(row); + assertThat(row).contains("grace@example.test").contains("NULL"); + } + + t.write(); + } +} diff --git a/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/MixedStageRollingWindowTest.java b/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/MixedStageRollingWindowTest.java new file mode 100644 index 0000000..c2c5cc1 --- /dev/null +++ b/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/MixedStageRollingWindowTest.java @@ -0,0 +1,88 @@ +package com.ankurm.expandcontract; + +import com.ankurm.expandcontract.customer.Customer; +import com.ankurm.expandcontract.customer.CustomerService; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.jdbc.core.simple.JdbcClient; + +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The property the whole article rests on: during a rolling deploy, two adjacent + * stages are serving traffic against the SAME database at the SAME time, for however + * long the rollout takes. If a customer written by one stage cannot be read correctly + * by the other, the deploy is not zero-downtime - it is a race with the rollout clock. + *

+ * This test builds two {@link CustomerService} instances that share one + * {@link JdbcClient} (standing in for one shared database, hit by two replicas on + * consecutive stages) and cross-checks every write/read direction for the three + * rollouts this article performs: Stage 1↔2, Stage 2↔3, and Stage 3↔4. + * See docs/08-the-rolling-window-proof.md - this is the test the live load generator + * run in the article is reproducing under real HTTP and real timing. + */ +class MixedStageRollingWindowTest { + + @Test + void everyAdjacentStagePairReadsWhatTheOtherWrote(@TempDir Path tmp) throws Exception { + Path db = tmp.resolve("rolling-window"); + Transcript t = Transcript.start("08-mixed-stage-rolling-window", + "Cross-stage consistency during each of the three rolling deploys"); + + // Deploy 2's rollout: some replicas still on Stage 1, some already on Stage 2. + TestSupport.migrateTo(db, "2"); + JdbcClient jdbc = TestSupport.jdbcClient(db); + CustomerService stage1 = new CustomerService(jdbc, 1); + CustomerService stage2 = new CustomerService(jdbc, 2); + + t.section("Stage 1 writes, Stage 2 reads"); + long a = stage1.create("Radia Perlman", "radia@example.test"); + Customer readByStage2 = stage2.findById(a).orElseThrow(); + t.line(readByStage2.toString()); + assertThat(readByStage2.email()).isEqualTo("radia@example.test"); + + t.section("Stage 2 writes, Stage 1 reads"); + long b = stage2.create("Barbara Liskov", "barbara@example.test"); + Customer readByStage1 = stage1.findById(b).orElseThrow(); + t.line(readByStage1.toString()); + assertThat(readByStage1.email()).isEqualTo("barbara@example.test"); + + // Deploy 3's rollout: some replicas on Stage 2, some already on Stage 3. + CustomerService stage3 = new CustomerService(jdbc, 3); + + t.section("Stage 2 writes, Stage 3 reads"); + long c = stage2.create("Shafi Goldwasser", "shafi@example.test"); + Customer readByStage3 = stage3.findById(c).orElseThrow(); + t.line(readByStage3.toString()); + assertThat(readByStage3.email()).isEqualTo("shafi@example.test"); + + t.section("Stage 3 writes, Stage 2 reads"); + long d = stage3.create("Frances Allen", "frances@example.test"); + Customer readByStage2Again = stage2.findById(d).orElseThrow(); + t.line(readByStage2Again.toString()); + assertThat(readByStage2Again.email()).isEqualTo("frances@example.test"); + + // Deploy 4a's rollout: some replicas on Stage 3, some already on Stage 4. + // email_address has been fully backfilled and dual-written for two whole + // deploys by this point - the precondition Stage 4 relies on. The old "email" + // column is still physically present (Deploy 4b, the drop, has not run yet) + // but Stage 4 code never looks at it. + CustomerService stage4 = new CustomerService(jdbc, 4); + + t.section("Stage 3 writes, Stage 4 reads"); + long e = stage3.create("Adele Goldberg", "adele@example.test"); + Customer readByStage4 = stage4.findById(e).orElseThrow(); + t.line(readByStage4.toString()); + assertThat(readByStage4.email()).isEqualTo("adele@example.test"); + + t.section("Stage 4 writes, Stage 3 reads"); + long f = stage4.create("Karen Sparck Jones", "karen@example.test"); + Customer readByStage3Again = stage3.findById(f).orElseThrow(); + t.line(readByStage3Again.toString()); + assertThat(readByStage3Again.email()).isEqualTo("karen@example.test"); + + t.write(); + } +} diff --git a/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/NotNullConstraintTrapTest.java b/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/NotNullConstraintTrapTest.java new file mode 100644 index 0000000..4c994b7 --- /dev/null +++ b/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/NotNullConstraintTrapTest.java @@ -0,0 +1,66 @@ +package com.ankurm.expandcontract; + +import com.ankurm.expandcontract.customer.CustomerService; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.jdbc.core.simple.JdbcClient; + +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * A trap this article's own first draft of the V2 migration fell into: adding + * {@code email_address} is not the whole story of "expand" when the column being + * retired is {@code NOT NULL}. Stage 4 code never writes "email" - so if "email" is + * still mandatory when Stage 4 starts, every create fails, in production, the moment + * that deploy reaches its first replica. The fix is one more line in the SAME + * migration: {@code ALTER TABLE customers ALTER COLUMN email DROP NOT NULL}. See + * docs/06-the-not-null-trap.md, and compare + * {@link com.ankurm.expandcontract.customer.CustomerService#create} - the shipped + * V2 migration (with the fix) is what every other test in this module runs against. + */ +class NotNullConstraintTrapTest { + + @Test + void stage4FailsIfTheOldColumnIsStillMandatory(@TempDir Path tmp) throws Exception { + Path db = tmp.resolve("not-null-trap"); + TestSupport.migrateTo(db, "1"); + JdbcClient jdbc = TestSupport.jdbcClient(db); + Transcript t = Transcript.start("06-not-null-trap", + "The NOT NULL trap: expand without relaxing the old column's constraint"); + + // The NAIVE version of V2: add the column, backfill, stop there. This is + // exactly V2 minus its last line. + jdbc.sql("ALTER TABLE customers ADD COLUMN email_address VARCHAR(320)").update(); + jdbc.sql("UPDATE customers SET email_address = email WHERE email_address IS NULL").update(); + + CustomerService stage4 = new CustomerService(jdbc, 4); + Throwable thrown = null; + try { + stage4.create("Too Early", "too.early@example.test"); + } catch (Throwable ex) { + thrown = ex; + } + t.section("Stage 4 create() against the NAIVE migration (no DROP NOT NULL)"); + assertThat(thrown).isNotNull(); + Throwable root = thrown; + while (root.getCause() != null) { + root = root.getCause(); + } + t.line(thrown.getClass().getName() + ": " + thrown.getMessage()); + t.line("root cause: " + root.getClass().getName() + ": " + root.getMessage()); + assertThat(root.getMessage()).contains("NULL not allowed for column \"EMAIL\""); + + t.section("Stage 4 create() against the SHIPPED V2 migration (DROP NOT NULL included)"); + Path fixedDb = tmp.resolve("not-null-fixed"); + TestSupport.migrateTo(fixedDb, "2"); + CustomerService fixedStage4 = new CustomerService(TestSupport.jdbcClient(fixedDb), 4); + long id = fixedStage4.create("On Time", "on.time@example.test"); + var result = fixedStage4.findById(id).orElseThrow(); + t.line(result.toString()); + assertThat(result.email()).isEqualTo("on.time@example.test"); + + t.write(); + } +} diff --git a/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/TestSupport.java b/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/TestSupport.java new file mode 100644 index 0000000..78caf59 --- /dev/null +++ b/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/TestSupport.java @@ -0,0 +1,46 @@ +package com.ankurm.expandcontract; + +import org.springframework.boot.autoconfigure.AutoConfigurations; +import org.springframework.boot.flyway.autoconfigure.FlywayAutoConfiguration; +import org.springframework.boot.jdbc.autoconfigure.DataSourceAutoConfiguration; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.jdbc.core.simple.JdbcClient; +import org.springframework.jdbc.datasource.DriverManagerDataSource; + +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Runs Flyway to a given target version through Boot's own {@code FlywayAutoConfiguration} + * (the same {@code spring.flyway.target} property a real deploy pipeline would set), then + * hands the test a plain {@link JdbcClient} against the same file - so a test can migrate a + * database to "however far Deploy N has gotten" and then exercise + * {@link com.ankurm.expandcontract.customer.CustomerService} instances directly, with no + * Spring context of their own, exactly as the app constructs them. + */ +final class TestSupport { + + static void migrateTo(Path dbPath, String target) { + new ApplicationContextRunner() + .withConfiguration(AutoConfigurations.of(DataSourceAutoConfiguration.class, FlywayAutoConfiguration.class)) + .withPropertyValues( + "spring.datasource.url=" + jdbcUrl(dbPath), + "spring.datasource.username=sa", + "spring.flyway.locations=classpath:db/migration", + "spring.flyway.target=" + target) + .run(ctx -> assertThat(ctx).hasNotFailed()); + } + + static JdbcClient jdbcClient(Path dbPath) { + DriverManagerDataSource ds = new DriverManagerDataSource(jdbcUrl(dbPath), "sa", ""); + return JdbcClient.create(ds); + } + + static String jdbcUrl(Path dbPath) { + return "jdbc:h2:file:" + dbPath + ";AUTO_SERVER=TRUE"; + } + + private TestSupport() { + } +} diff --git a/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/Transcript.java b/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/Transcript.java new file mode 100644 index 0000000..2037b82 --- /dev/null +++ b/db-migrations-expand-contract/src/test/java/com/ankurm/expandcontract/Transcript.java @@ -0,0 +1,60 @@ +package com.ankurm.expandcontract; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.time.Instant; + +/** + * Writes a named transcript under {@code docs/output/} while the test that produced it runs. + * Every console line quoted in the companion article for this module comes from one of these + * files, and the file is only ever written by a test assertion - never hand-typed. + */ +public final class Transcript { + + private static final Path OUTPUT_DIR = Paths.get("docs", "output"); + + private final StringBuilder buffer = new StringBuilder(); + private final String name; + + private Transcript(String name) { + this.name = name; + } + + public static Transcript start(String name, String heading) { + Transcript t = new Transcript(name); + t.line("=".repeat(Math.min(78, heading.length() + 4))); + t.line(heading); + t.line("=".repeat(Math.min(78, heading.length() + 4))); + t.line("captured: " + Instant.now()); + t.line(""); + return t; + } + + public Transcript line(String text) { + buffer.append(text).append(System.lineSeparator()); + System.out.println(text); + return this; + } + + public Transcript section(String title) { + line(""); + line("-- " + title + " --"); + return this; + } + + public void write() { + try { + Files.createDirectories(OUTPUT_DIR); + Path target = OUTPUT_DIR.resolve(name + ".txt"); + Files.writeString(target, buffer.toString(), StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING); + } + catch (IOException ex) { + throw new UncheckedIOException(ex); + } + } +}