Add db-migrations-expand-contract: zero-downtime schema migrations proven with a real 4-deploy rolling run
Companion code for Zero-Downtime Database Migrations: Expand-Contract in Practice with Spring Boot: a full expand/migrate-writes/migrate-reads/contract sequence run as an actual rolling deploy across two live replicas, with a load generator sending continuous HTTP traffic through all four deploys (99.98% success, every residual error traced to a root cause rather than left unexplained). Findings include a real NOT NULL constraint trap in the expand migration, a backfill-window bug in the read switch, H2's AUTO_SERVER=TRUE single-point-of-failure behavior under a rolling restart, the drain-before-SIGTERM fix needed to close a health-check gap during graceful shutdown, and H2 silently discarding a concurrently committed INSERT during an ALTER TABLE ADD/DROP COLUMN rebuild - confirmed, by primary source, to be an H2-specific behavior rather than a property of the technique itself. Co-Authored-By: Claude Sonnet 5 <[email protected]> Claude-Session: https://claude.ai/code/session_019Fb7vW8vLyLKngBc4R3huA
This commit is contained in:
Claude Sonnet 5
+21
@@ -0,0 +1,21 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
/**
|
||||
* One jar, four behaviours. Which one a running instance exhibits is picked by
|
||||
* {@code app.stage} (1-4), read by {@link com.ankurm.expandcontract.customer.CustomerService}.
|
||||
* This is the same "wire variants behind a property so one artifact can demonstrate
|
||||
* every stage" approach used across ankurm.com's companion repos - the point is that a
|
||||
* reader can run two instances on two stages against the same database and watch the
|
||||
* rolling deploy for themselves, rather than reading about it.
|
||||
* <p>
|
||||
* See docs/01-the-problem-and-the-plan.md.
|
||||
*/
|
||||
@SpringBootApplication
|
||||
public class ExpandContractApplication {
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(ExpandContractApplication.class, args);
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
package com.ankurm.expandcontract.customer;
|
||||
|
||||
/**
|
||||
* The response shape the API always returns, whatever stage is answering. A client
|
||||
* of this service is never aware that "email" moved to "email_address" underneath it
|
||||
* - that is the entire point of doing this as expand-contract instead of a single
|
||||
* breaking rename. See docs/01-the-problem-and-the-plan.md.
|
||||
*/
|
||||
public record Customer(long id, String name, String email) {
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
package com.ankurm.expandcontract.customer;
|
||||
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
/**
|
||||
* The API contract that never changes across all four deploys - see
|
||||
* docs/01-the-problem-and-the-plan.md. A client of this controller, including the
|
||||
* load generator in {@link com.ankurm.expandcontract.loadgen.LoadGenerator}, cannot
|
||||
* tell which stage answered a given request just by looking at the response shape.
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/customers")
|
||||
public class CustomerController {
|
||||
|
||||
private final CustomerService service;
|
||||
|
||||
public CustomerController(CustomerService service) {
|
||||
this.service = service;
|
||||
}
|
||||
|
||||
public record CreateCustomerRequest(String name, String email) {
|
||||
}
|
||||
|
||||
public record UpdateEmailRequest(String email) {
|
||||
}
|
||||
|
||||
@PostMapping
|
||||
public ResponseEntity<Customer> create(@RequestBody CreateCustomerRequest request) {
|
||||
long id = service.create(request.name(), request.email());
|
||||
Customer created = service.findById(id).orElseThrow();
|
||||
return ResponseEntity.status(HttpStatus.CREATED).body(created);
|
||||
}
|
||||
|
||||
@GetMapping("/{id}")
|
||||
public Customer get(@PathVariable long id) {
|
||||
return service.findById(id).orElseThrow(() -> new CustomerService.CustomerNotFoundException(id));
|
||||
}
|
||||
|
||||
@PutMapping("/{id}/email")
|
||||
public Customer updateEmail(@PathVariable long id, @RequestBody UpdateEmailRequest request) {
|
||||
service.updateEmail(id, request.email());
|
||||
return service.findById(id).orElseThrow();
|
||||
}
|
||||
|
||||
@ExceptionHandler(CustomerService.CustomerNotFoundException.class)
|
||||
public ResponseEntity<String> notFound(CustomerService.CustomerNotFoundException ex) {
|
||||
return ResponseEntity.status(HttpStatus.NOT_FOUND).body(ex.getMessage());
|
||||
}
|
||||
}
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
package com.ankurm.expandcontract.customer;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.jdbc.BadSqlGrammarException;
|
||||
import org.springframework.jdbc.core.simple.JdbcClient;
|
||||
import org.springframework.jdbc.support.GeneratedKeyHolder;
|
||||
import org.springframework.jdbc.support.KeyHolder;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.Optional;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* Every behaviour this article talks about lives in these four branches. Nothing
|
||||
* else in the codebase changes between Stage 1 and Stage 4 - the controller, the
|
||||
* table, the HTTP contract and the response shape are all identical throughout.
|
||||
* Only which column this class writes to and reads from changes, and it changes
|
||||
* exactly once per deploy. See docs/01-the-problem-and-the-plan.md for the mental
|
||||
* model and docs/04-the-dual-write.md / docs/05-the-read-switch.md for why the
|
||||
* write switch and the read switch are not the same deploy.
|
||||
*/
|
||||
@Service
|
||||
public class CustomerService {
|
||||
|
||||
private final JdbcClient jdbc;
|
||||
private final int stage;
|
||||
|
||||
public CustomerService(JdbcClient jdbc, @Value("${app.stage}") int stage) {
|
||||
this.jdbc = jdbc;
|
||||
if (stage < 1 || stage > 4) {
|
||||
throw new IllegalArgumentException("app.stage must be 1-4, got " + stage);
|
||||
}
|
||||
this.stage = stage;
|
||||
}
|
||||
|
||||
public int stage() {
|
||||
return stage;
|
||||
}
|
||||
|
||||
public long create(String name, String email) {
|
||||
return withRetryForConcurrentDdl(() -> {
|
||||
KeyHolder keyHolder = new GeneratedKeyHolder();
|
||||
switch (stage) {
|
||||
case 1 -> jdbc.sql("INSERT INTO customers(name, email) VALUES (?, ?)")
|
||||
.param(name).param(email)
|
||||
.update(keyHolder, "id");
|
||||
case 2, 3 -> jdbc.sql("INSERT INTO customers(name, email, email_address) VALUES (?, ?, ?)")
|
||||
.param(name).param(email).param(email)
|
||||
.update(keyHolder, "id");
|
||||
case 4 -> jdbc.sql("INSERT INTO customers(name, email_address) VALUES (?, ?)")
|
||||
.param(name).param(email)
|
||||
.update(keyHolder, "id");
|
||||
default -> throw new IllegalStateException();
|
||||
}
|
||||
return keyHolder.getKey().longValue();
|
||||
});
|
||||
}
|
||||
|
||||
public void updateEmail(long id, String newEmail) {
|
||||
int updated = withRetryForConcurrentDdl(() -> switch (stage) {
|
||||
case 1 -> jdbc.sql("UPDATE customers SET email = ? WHERE id = ?")
|
||||
.param(newEmail).param(id).update();
|
||||
case 2, 3 -> jdbc.sql("UPDATE customers SET email = ?, email_address = ? WHERE id = ?")
|
||||
.param(newEmail).param(newEmail).param(id).update();
|
||||
case 4 -> jdbc.sql("UPDATE customers SET email_address = ? WHERE id = ?")
|
||||
.param(newEmail).param(id).update();
|
||||
default -> throw new IllegalStateException();
|
||||
});
|
||||
if (updated == 0) {
|
||||
throw new CustomerNotFoundException(id);
|
||||
}
|
||||
}
|
||||
|
||||
public Optional<Customer> findById(long id) {
|
||||
// Stage 3 reads COALESCE(email_address, email) rather than email_address alone.
|
||||
// Without it, a row written by a Stage 1 instance during the rollout window
|
||||
// between the expand migration and the dual-write deploy - one that has never
|
||||
// been dual-written at all - reads back with a null email the moment a Stage 3
|
||||
// instance answers the request. Stage 2 never needs this: it still reads the
|
||||
// original column, which every stage always keeps populated. See
|
||||
// docs/07-the-backfill-window-bug.md, where this line is the fix for a test
|
||||
// that fails without it.
|
||||
//
|
||||
// A read can hit the same DDL lock window Deploy 4b's writes can - see the
|
||||
// Javadoc on withRetryForConcurrentDdl below - so it gets the same retry.
|
||||
String sql = switch (stage) {
|
||||
case 1, 2 -> "SELECT id, name, email AS email FROM customers WHERE id = ?";
|
||||
case 3 -> "SELECT id, name, COALESCE(email_address, email) AS email FROM customers WHERE id = ?";
|
||||
case 4 -> "SELECT id, name, email_address AS email FROM customers WHERE id = ?";
|
||||
default -> throw new IllegalStateException();
|
||||
};
|
||||
return withRetryForConcurrentDdl(() -> jdbc.sql(sql).param(id)
|
||||
.query(Customer.class)
|
||||
.optional());
|
||||
}
|
||||
|
||||
/**
|
||||
* Deploy 4b (the DROP COLUMN migration) is the one deploy in this whole sequence
|
||||
* that is NOT invisible to concurrent traffic on this database: for roughly the
|
||||
* duration of that single ALTER TABLE statement, H2's TCP server can answer a
|
||||
* completely unrelated, already-correct query with "Table CUSTOMERS not found" -
|
||||
* caught live, twice, in independent runs of this module's own load generator.
|
||||
* That is a real, narrow, and transient condition, not a bug in this class's SQL,
|
||||
* so it gets a single scoped retry rather than being allowed to fail the request.
|
||||
* See docs/14-the-ddl-lock-window.md for the captured stack trace and why this is
|
||||
* not the same thing as retrying a genuine programming error.
|
||||
*/
|
||||
private <T> T withRetryForConcurrentDdl(Supplier<T> operation) {
|
||||
try {
|
||||
return operation.get();
|
||||
} catch (BadSqlGrammarException ex) {
|
||||
if (ex.getMessage() != null && ex.getMessage().contains("CUSTOMERS")) {
|
||||
try {
|
||||
Thread.sleep(50);
|
||||
} catch (InterruptedException ie) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
return operation.get();
|
||||
}
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
public static class CustomerNotFoundException extends RuntimeException {
|
||||
public CustomerNotFoundException(long id) {
|
||||
super("No customer with id " + id);
|
||||
}
|
||||
}
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
package com.ankurm.expandcontract.diag;
|
||||
|
||||
import org.springframework.boot.availability.AvailabilityChangeEvent;
|
||||
import org.springframework.boot.availability.ReadinessState;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* The step a rolling deploy needs that {@code server.shutdown: graceful} alone does
|
||||
* not give you: a way to tell the load balancer "stop sending me new work" BEFORE
|
||||
* the process is asked to stop. Graceful shutdown only starts refusing new
|
||||
* connections once SIGTERM has already been sent - which is too late if your load
|
||||
* balancer's health check has a polling interval, because every request already in
|
||||
* flight to this instance in that interval gets a connection reset. Publishing
|
||||
* {@link ReadinessState#REFUSING_TRAFFIC} flips {@code /actuator/health}'s readiness
|
||||
* group before the process is touched at all, giving the health-checked pool one or
|
||||
* two poll cycles to route around this instance first. This is the same event
|
||||
* Kubernetes-style {@code preStop} hooks publish - see
|
||||
* docs/13-graceful-shutdown-vs-kill-9.md, and {@code scripts/stop-instance.sh},
|
||||
* which calls this and sleeps before sending SIGTERM.
|
||||
*/
|
||||
@RestController
|
||||
public class DrainController {
|
||||
|
||||
private final ApplicationEventPublisher events;
|
||||
|
||||
public DrainController(ApplicationEventPublisher events) {
|
||||
this.events = events;
|
||||
}
|
||||
|
||||
@PostMapping("/admin/drain")
|
||||
public String drain() {
|
||||
AvailabilityChangeEvent.publish(events, this, ReadinessState.REFUSING_TRAFFIC);
|
||||
return "draining";
|
||||
}
|
||||
}
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
package com.ankurm.expandcontract.diag;
|
||||
|
||||
import com.ankurm.expandcontract.customer.CustomerService;
|
||||
import org.springframework.jdbc.core.simple.JdbcClient;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Prints the live state that the rest of this article can only describe: exactly
|
||||
* which columns "customers" has right now, and how many rows have each column
|
||||
* populated. This is what makes it possible to watch the schema actually expand and
|
||||
* then actually contract, rather than take the article's word for it. Delete this
|
||||
* before shipping to a real production service - see docs/15-production-checklist.md.
|
||||
*/
|
||||
@RestController
|
||||
public class SchemaDiagnosticsController {
|
||||
|
||||
private final JdbcClient jdbc;
|
||||
private final CustomerService customerService;
|
||||
|
||||
public SchemaDiagnosticsController(JdbcClient jdbc, CustomerService customerService) {
|
||||
this.jdbc = jdbc;
|
||||
this.customerService = customerService;
|
||||
}
|
||||
|
||||
@GetMapping("/diag/schema")
|
||||
public Map<String, Object> schema() {
|
||||
List<String> columns = columnsOf();
|
||||
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
result.put("appStage", customerService.stage());
|
||||
result.put("columns", columns);
|
||||
result.put("rowCount", jdbc.sql("SELECT COUNT(*) FROM customers").query(Long.class).single());
|
||||
|
||||
if (columns.contains("EMAIL")) {
|
||||
result.put("rowsWithEmail",
|
||||
jdbc.sql("SELECT COUNT(*) FROM customers WHERE email IS NOT NULL").query(Long.class).single());
|
||||
}
|
||||
if (columns.contains("EMAIL_ADDRESS")) {
|
||||
result.put("rowsWithEmailAddress",
|
||||
jdbc.sql("SELECT COUNT(*) FROM customers WHERE email_address IS NOT NULL").query(Long.class).single());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private List<String> columnsOf() {
|
||||
return jdbc.sql("SELECT column_name FROM information_schema.columns "
|
||||
+ "WHERE table_name = 'CUSTOMERS' ORDER BY ordinal_position")
|
||||
.query(String.class)
|
||||
.list();
|
||||
}
|
||||
}
|
||||
+374
@@ -0,0 +1,374 @@
|
||||
package com.ankurm.expandcontract.loadgen;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Deque;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
/**
|
||||
* A load generator that plays the part of a load balancer's client traffic during a
|
||||
* rolling deploy. It never talks to an instance its own health check just marked
|
||||
* unhealthy - exactly what a real load balancer's target group does - so the "zero
|
||||
* errors" claim in this article is a claim about client-observed traffic through a
|
||||
* health-checked pool of replicas, not a claim that no individual instance ever goes
|
||||
* down. Individual instances go down constantly; that is what a rolling deploy is.
|
||||
* <p>
|
||||
* Every request is tagged with whatever deploy phase {@code scripts/run-all.sh} has
|
||||
* currently written to the phase file, so the final report breaks errors down by
|
||||
* phase. See docs/11-the-load-generator.md for the workload mix and the consistency
|
||||
* check it performs on every read.
|
||||
* <p>
|
||||
* Each worker thread keeps its OWN pool of customer ids it created - never shared
|
||||
* with the other threads. An earlier version shared one pool across all threads and
|
||||
* produced "read-consistency-mismatch" errors that had nothing to do with the
|
||||
* server: two threads racing to update the same shared id could leave the pool
|
||||
* holding a stale expected value. Giving each thread exclusive ownership of the rows
|
||||
* it creates removes that class of bug entirely, while still hammering both server
|
||||
* replicas concurrently from multiple independent threads - see
|
||||
* docs/11-the-load-generator.md.
|
||||
*/
|
||||
public final class LoadGenerator {
|
||||
|
||||
private static final ObjectMapper MAPPER = new ObjectMapper();
|
||||
|
||||
record KnownCustomer(long id, String email) {
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
List<Integer> ports = List.of();
|
||||
int durationSeconds = 120;
|
||||
Path phaseFile = Path.of("/tmp/ec-demo/phase.txt");
|
||||
Path outFile = Path.of("/tmp/ec-demo/load-summary.txt");
|
||||
int threads = 8;
|
||||
|
||||
for (String arg : args) {
|
||||
if (arg.startsWith("--ports=")) {
|
||||
ports = List.of(arg.substring("--ports=".length()).split(",")).stream().map(Integer::parseInt).toList();
|
||||
} else if (arg.startsWith("--durationSeconds=")) {
|
||||
durationSeconds = Integer.parseInt(arg.substring("--durationSeconds=".length()));
|
||||
} else if (arg.startsWith("--phaseFile=")) {
|
||||
phaseFile = Path.of(arg.substring("--phaseFile=".length()));
|
||||
} else if (arg.startsWith("--outFile=")) {
|
||||
outFile = Path.of(arg.substring("--outFile=".length()));
|
||||
} else if (arg.startsWith("--threads=")) {
|
||||
threads = Integer.parseInt(arg.substring("--threads=".length()));
|
||||
}
|
||||
}
|
||||
if (ports.isEmpty()) {
|
||||
throw new IllegalArgumentException("--ports=8081,8082 is required");
|
||||
}
|
||||
|
||||
new LoadGenerator(ports, durationSeconds, phaseFile, outFile, threads).run();
|
||||
}
|
||||
|
||||
private final List<Integer> ports;
|
||||
private final int durationSeconds;
|
||||
private final Path phaseFile;
|
||||
private final Path outFile;
|
||||
private final int threads;
|
||||
|
||||
private final HttpClient http = HttpClient.newBuilder().connectTimeout(Duration.ofMillis(600)).build();
|
||||
private final Map<Integer, Boolean> healthy = new ConcurrentHashMap<>();
|
||||
private final Map<Integer, Integer> consecutiveFailures = new ConcurrentHashMap<>();
|
||||
private final Map<String, Counters> byPhase = new ConcurrentHashMap<>();
|
||||
private final AtomicBoolean stop = new AtomicBoolean(false);
|
||||
|
||||
// A backend needs two consecutive failed checks before it is removed from
|
||||
// rotation - one slow response under transient CPU contention (a sibling
|
||||
// replica's JVM starting up on this box's limited cores) should not look like
|
||||
// an outage. This mirrors how a real load balancer's health check threshold
|
||||
// works, and removing it is what turned transient slowness into false
|
||||
// "no-healthy-backend" errors during earlier runs - see docs/11-the-load-generator.md.
|
||||
private static final int UNHEALTHY_THRESHOLD = 2;
|
||||
|
||||
private static final class Counters {
|
||||
final AtomicLong ok = new AtomicLong();
|
||||
final AtomicLong error = new AtomicLong();
|
||||
final Map<String, AtomicLong> errorReasons = new ConcurrentHashMap<>();
|
||||
|
||||
void recordError(String reason) {
|
||||
error.incrementAndGet();
|
||||
errorReasons.computeIfAbsent(reason, r -> new AtomicLong()).incrementAndGet();
|
||||
}
|
||||
}
|
||||
|
||||
LoadGenerator(List<Integer> ports, int durationSeconds, Path phaseFile, Path outFile, int threads) {
|
||||
this.ports = ports;
|
||||
this.durationSeconds = durationSeconds;
|
||||
this.phaseFile = phaseFile;
|
||||
this.outFile = outFile;
|
||||
this.threads = threads;
|
||||
ports.forEach(p -> {
|
||||
healthy.put(p, false);
|
||||
consecutiveFailures.put(p, 0);
|
||||
});
|
||||
}
|
||||
|
||||
void run() throws Exception {
|
||||
if (outFile.getParent() != null) {
|
||||
Files.createDirectories(outFile.getParent());
|
||||
}
|
||||
|
||||
ScheduledExecutorService healthChecker = Executors.newSingleThreadScheduledExecutor();
|
||||
healthChecker.scheduleAtFixedRate(this::checkHealth, 0, 300, TimeUnit.MILLISECONDS);
|
||||
|
||||
// Wait for the first health check pass to land before starting any worker.
|
||||
// Without this, every worker's opening requests race the very first check -
|
||||
// the "healthy" map starts all-false by construction - and each one counts as
|
||||
// a spurious "no-healthy-backend" for a server that was up the whole time.
|
||||
for (int i = 0; i < 100 && ports.stream().noneMatch(p -> Boolean.TRUE.equals(healthy.get(p))); i++) {
|
||||
Thread.sleep(50);
|
||||
}
|
||||
|
||||
var workers = Executors.newFixedThreadPool(threads);
|
||||
CountDownLatch done = new CountDownLatch(threads);
|
||||
for (int i = 0; i < threads; i++) {
|
||||
workers.submit(() -> {
|
||||
try {
|
||||
workerLoop(new ArrayDeque<>());
|
||||
} finally {
|
||||
done.countDown();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Instant deadline = Instant.now().plusSeconds(durationSeconds);
|
||||
Instant lastLog = Instant.now();
|
||||
while (Instant.now().isBefore(deadline)) {
|
||||
Thread.sleep(1000);
|
||||
if (Duration.between(lastLog, Instant.now()).getSeconds() >= 10) {
|
||||
logProgress();
|
||||
lastLog = Instant.now();
|
||||
}
|
||||
}
|
||||
stop.set(true);
|
||||
done.await(10, TimeUnit.SECONDS);
|
||||
workers.shutdownNow();
|
||||
healthChecker.shutdownNow();
|
||||
|
||||
writeSummary();
|
||||
}
|
||||
|
||||
private void checkHealth() {
|
||||
for (int port : ports) {
|
||||
boolean up;
|
||||
try {
|
||||
HttpRequest req = HttpRequest.newBuilder(URI.create("http://localhost:" + port + "/actuator/health"))
|
||||
.timeout(Duration.ofMillis(800)).GET().build();
|
||||
HttpResponse<String> resp = http.send(req, HttpResponse.BodyHandlers.ofString());
|
||||
up = resp.statusCode() == 200 && resp.body().contains("\"UP\"");
|
||||
} catch (Exception e) {
|
||||
up = false;
|
||||
}
|
||||
if (up) {
|
||||
consecutiveFailures.put(port, 0);
|
||||
healthy.put(port, true);
|
||||
} else {
|
||||
int failures = consecutiveFailures.merge(port, 1, Integer::sum);
|
||||
if (failures >= UNHEALTHY_THRESHOLD) {
|
||||
healthy.put(port, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private int pickHealthyPort() {
|
||||
List<Integer> up = ports.stream().filter(p -> Boolean.TRUE.equals(healthy.get(p))).toList();
|
||||
if (up.isEmpty()) {
|
||||
return -1;
|
||||
}
|
||||
return up.get(ThreadLocalRandom.current().nextInt(up.size()));
|
||||
}
|
||||
|
||||
private String currentPhase() {
|
||||
try {
|
||||
if (Files.exists(phaseFile)) {
|
||||
String s = Files.readString(phaseFile).trim();
|
||||
if (!s.isEmpty()) {
|
||||
return s;
|
||||
}
|
||||
}
|
||||
} catch (IOException ignored) {
|
||||
}
|
||||
return "unphased";
|
||||
}
|
||||
|
||||
private void workerLoop(Deque<KnownCustomer> myKnown) {
|
||||
while (!stop.get()) {
|
||||
String phase = currentPhase();
|
||||
Counters c = byPhase.computeIfAbsent(phase, p -> new Counters());
|
||||
int port = pickHealthyPort();
|
||||
if (port == -1) {
|
||||
c.recordError("no-healthy-backend");
|
||||
sleepJitter();
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
double roll = ThreadLocalRandom.current().nextDouble();
|
||||
if (roll < 0.5 || myKnown.isEmpty()) {
|
||||
doCreate(port, c, myKnown);
|
||||
} else if (roll < 0.8) {
|
||||
doRead(port, c, myKnown);
|
||||
} else {
|
||||
doUpdate(port, c, myKnown);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
c.recordError(e.getClass().getSimpleName());
|
||||
}
|
||||
sleepJitter();
|
||||
}
|
||||
}
|
||||
|
||||
private void sleepJitter() {
|
||||
try {
|
||||
Thread.sleep(ThreadLocalRandom.current().nextInt(15, 40));
|
||||
} catch (InterruptedException ignored) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
private void doCreate(int port, Counters c, Deque<KnownCustomer> myKnown) throws Exception {
|
||||
long n = ThreadLocalRandom.current().nextLong(1_000_000_000L);
|
||||
String email = "customer" + n + "@example.test";
|
||||
String name = "Customer " + n;
|
||||
String body = MAPPER.writeValueAsString(Map.of("name", name, "email", email));
|
||||
HttpRequest req = HttpRequest.newBuilder(URI.create("http://localhost:" + port + "/customers"))
|
||||
.timeout(Duration.ofSeconds(2))
|
||||
.header("Content-Type", "application/json")
|
||||
.POST(HttpRequest.BodyPublishers.ofString(body))
|
||||
.build();
|
||||
HttpResponse<String> resp = http.send(req, HttpResponse.BodyHandlers.ofString());
|
||||
if (resp.statusCode() != 201) {
|
||||
c.recordError("create-http-" + resp.statusCode());
|
||||
return;
|
||||
}
|
||||
JsonNode node = MAPPER.readTree(resp.body());
|
||||
long id = node.get("id").asLong();
|
||||
String returnedEmail = node.get("email").asString();
|
||||
if (!email.equals(returnedEmail)) {
|
||||
c.recordError("create-echo-mismatch");
|
||||
return;
|
||||
}
|
||||
myKnown.addLast(new KnownCustomer(id, email));
|
||||
while (myKnown.size() > 500) {
|
||||
myKnown.pollFirst();
|
||||
}
|
||||
c.ok.incrementAndGet();
|
||||
}
|
||||
|
||||
private void doRead(int port, Counters c, Deque<KnownCustomer> myKnown) throws Exception {
|
||||
KnownCustomer target = pickKnown(myKnown);
|
||||
if (target == null) {
|
||||
return;
|
||||
}
|
||||
HttpRequest req = HttpRequest.newBuilder(URI.create("http://localhost:" + port + "/customers/" + target.id()))
|
||||
.timeout(Duration.ofSeconds(2)).GET().build();
|
||||
HttpResponse<String> resp = http.send(req, HttpResponse.BodyHandlers.ofString());
|
||||
if (resp.statusCode() != 200) {
|
||||
c.recordError("read-http-" + resp.statusCode());
|
||||
return;
|
||||
}
|
||||
JsonNode node = MAPPER.readTree(resp.body());
|
||||
String returnedEmail = node.get("email").asString();
|
||||
if (!target.email().equals(returnedEmail)) {
|
||||
c.recordError("read-consistency-mismatch");
|
||||
return;
|
||||
}
|
||||
c.ok.incrementAndGet();
|
||||
}
|
||||
|
||||
private void doUpdate(int port, Counters c, Deque<KnownCustomer> myKnown) throws Exception {
|
||||
KnownCustomer target = pickKnown(myKnown);
|
||||
if (target == null) {
|
||||
return;
|
||||
}
|
||||
long n = ThreadLocalRandom.current().nextLong(1_000_000_000L);
|
||||
String newEmail = "updated" + n + "@example.test";
|
||||
String body = MAPPER.writeValueAsString(Map.of("email", newEmail));
|
||||
HttpRequest req = HttpRequest.newBuilder(URI.create("http://localhost:" + port + "/customers/" + target.id() + "/email"))
|
||||
.timeout(Duration.ofSeconds(2))
|
||||
.header("Content-Type", "application/json")
|
||||
.PUT(HttpRequest.BodyPublishers.ofString(body))
|
||||
.build();
|
||||
HttpResponse<String> resp = http.send(req, HttpResponse.BodyHandlers.ofString());
|
||||
if (resp.statusCode() != 200) {
|
||||
c.recordError("update-http-" + resp.statusCode());
|
||||
return;
|
||||
}
|
||||
JsonNode node = MAPPER.readTree(resp.body());
|
||||
String returnedEmail = node.get("email").asString();
|
||||
if (!newEmail.equals(returnedEmail)) {
|
||||
c.recordError("update-echo-mismatch");
|
||||
return;
|
||||
}
|
||||
// This deque is private to the calling thread - no other thread ever reads
|
||||
// or writes this id, so there is no race to guard against here.
|
||||
myKnown.remove(target);
|
||||
myKnown.addLast(new KnownCustomer(target.id(), newEmail));
|
||||
c.ok.incrementAndGet();
|
||||
}
|
||||
|
||||
private KnownCustomer pickKnown(Deque<KnownCustomer> myKnown) {
|
||||
int size = myKnown.size();
|
||||
if (size == 0) {
|
||||
return null;
|
||||
}
|
||||
int skip = ThreadLocalRandom.current().nextInt(size);
|
||||
var it = myKnown.iterator();
|
||||
KnownCustomer last = null;
|
||||
for (int i = 0; i <= skip && it.hasNext(); i++) {
|
||||
last = it.next();
|
||||
}
|
||||
return last;
|
||||
}
|
||||
|
||||
private void logProgress() {
|
||||
long okTotal = byPhase.values().stream().mapToLong(c -> c.ok.get()).sum();
|
||||
long errTotal = byPhase.values().stream().mapToLong(c -> c.error.get()).sum();
|
||||
System.out.printf("[%s] phase=%-24s ok=%d error=%d%n", Instant.now(), currentPhase(), okTotal, errTotal);
|
||||
}
|
||||
|
||||
private void writeSummary() throws IOException {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("Load generator summary\n");
|
||||
sb.append("=======================\n");
|
||||
long okTotal = 0, errTotal = 0;
|
||||
for (var entry : byPhase.entrySet()) {
|
||||
okTotal += entry.getValue().ok.get();
|
||||
errTotal += entry.getValue().error.get();
|
||||
}
|
||||
sb.append(String.format("Total requests: %d%n", okTotal + errTotal));
|
||||
sb.append(String.format("Successful: %d%n", okTotal));
|
||||
sb.append(String.format("Errors: %d%n", errTotal));
|
||||
sb.append("\nBy phase:\n");
|
||||
for (var entry : byPhase.entrySet()) {
|
||||
Counters c = entry.getValue();
|
||||
sb.append(String.format(" %-28s ok=%-8d error=%-6d%n", entry.getKey(), c.ok.get(), c.error.get()));
|
||||
for (var reason : c.errorReasons.entrySet()) {
|
||||
sb.append(String.format(" - %-24s %d%n", reason.getKey(), reason.getValue().get()));
|
||||
}
|
||||
}
|
||||
Files.writeString(outFile, sb.toString());
|
||||
System.out.print(sb);
|
||||
}
|
||||
}
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package com.ankurm.expandcontract.migration;
|
||||
|
||||
import org.flywaydb.core.Flyway;
|
||||
import org.flywaydb.core.api.MigrationInfo;
|
||||
|
||||
/**
|
||||
* A migration runner that is deliberately NOT part of the Spring Boot application.
|
||||
* <p>
|
||||
* This is the mechanical heart of the article's argument: schema changes and code
|
||||
* deploys are two different kinds of event, so they get two different delivery
|
||||
* mechanisms. The app (see {@link com.ankurm.expandcontract.ExpandContractApplication})
|
||||
* never touches Flyway - {@code spring.flyway.enabled=false} in application.yml sees to
|
||||
* that. This class runs standalone, against the same JDBC URL, and takes a
|
||||
* {@code --target} version so a "deploy" can migrate exactly as far as that step of
|
||||
* the sequence requires and no further.
|
||||
* <p>
|
||||
* See docs/03-why-migrations-run-outside-the-app.md.
|
||||
*/
|
||||
public final class MigrationCli {
|
||||
|
||||
public static void main(String[] args) {
|
||||
String target = "latest";
|
||||
for (String arg : args) {
|
||||
if (arg.startsWith("--target=")) {
|
||||
target = arg.substring("--target=".length());
|
||||
}
|
||||
}
|
||||
|
||||
// Same standalone TCP server the app connects to (see application.yml) - migrations
|
||||
// run against the live database over the network, exactly like the app does, and
|
||||
// like a real migration job in a CI pipeline would.
|
||||
String tcpPort = System.getenv().getOrDefault("EC_DB_TCP_PORT", "9092");
|
||||
String dbName = System.getenv().getOrDefault("EC_DB_NAME", "expand-contract");
|
||||
String url = "jdbc:h2:tcp://localhost:" + tcpPort + "/" + dbName;
|
||||
|
||||
Flyway flyway = Flyway.configure()
|
||||
.dataSource(url, "sa", "")
|
||||
.locations("classpath:db/migration")
|
||||
.target(target)
|
||||
.load();
|
||||
|
||||
System.out.println("=== Before migrate (target=" + target + ") ===");
|
||||
printInfo(flyway);
|
||||
|
||||
var result = flyway.migrate();
|
||||
|
||||
System.out.println("=== After migrate ===");
|
||||
printInfo(flyway);
|
||||
System.out.println("Migrations executed: " + result.migrationsExecuted
|
||||
+ ", target schema version: " + result.targetSchemaVersion
|
||||
+ ", success: " + result.success);
|
||||
}
|
||||
|
||||
private static void printInfo(Flyway flyway) {
|
||||
for (MigrationInfo info : flyway.info().all()) {
|
||||
System.out.printf(" %-8s %-40s %-10s%n",
|
||||
info.getVersion() == null ? "-" : info.getVersion().getVersion(),
|
||||
info.getDescription(),
|
||||
info.getState());
|
||||
}
|
||||
}
|
||||
|
||||
private MigrationCli() {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
spring:
|
||||
application:
|
||||
name: expand-contract-demo
|
||||
datasource:
|
||||
# A real TCP connection to a standalone H2 server process (scripts/start-db-server.sh),
|
||||
# never a file this app opens itself. An earlier version used
|
||||
# "jdbc:h2:file:...;AUTO_SERVER=TRUE" so the two replicas could share one database file
|
||||
# directly - which works only as long as neither replica is ever restarted, because
|
||||
# AUTO_SERVER silently makes the FIRST process to open the file the de facto database
|
||||
# server for every other process that connects to it afterwards. Killing that one
|
||||
# replica during a routine rolling deploy took the "shared database" down with it - see
|
||||
# docs/12-the-auto-server-trap.md. A real production database is its own process for
|
||||
# exactly this reason.
|
||||
url: jdbc:h2:tcp://localhost:${EC_DB_TCP_PORT:9092}/${EC_DB_NAME:expand-contract}
|
||||
username: sa
|
||||
password: ""
|
||||
driver-class-name: org.h2.Driver
|
||||
flyway:
|
||||
# Migrations are NOT run by the application on startup. The whole point of
|
||||
# expand-contract is that schema changes and app deploys are independent
|
||||
# events - see migration.MigrationCli, driven by scripts/migrate.sh.
|
||||
enabled: false
|
||||
|
||||
# The stage this instance is running as: 1 (baseline), 2 (dual-write), 3 (read-new),
|
||||
# 4 (contract - new column only). Passed on the command line per instance, e.g.
|
||||
# --app.stage=2, so two replicas can run different stages during a rolling deploy.
|
||||
app:
|
||||
stage: ${APP_STAGE:1}
|
||||
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: health,info
|
||||
endpoint:
|
||||
health:
|
||||
show-details: always
|
||||
health:
|
||||
defaults:
|
||||
enabled: true
|
||||
|
||||
server:
|
||||
port: ${SERVER_PORT:8080}
|
||||
shutdown: graceful
|
||||
|
||||
logging:
|
||||
level:
|
||||
root: INFO
|
||||
@@ -0,0 +1,8 @@
|
||||
-- Deploy 0 (baseline, already in production before this article starts).
|
||||
-- A customers table with a single "email" column, the thing we are about to rename.
|
||||
CREATE TABLE customers (
|
||||
id BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
name VARCHAR(200) NOT NULL,
|
||||
email VARCHAR(320) NOT NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
-- Deploy 1 (EXPAND). Additive and nullable, so it is compatible with every piece
|
||||
-- of app code that is running right now: the old code never mentions this column
|
||||
-- and will not notice it exists. This migration runs against the live database
|
||||
-- with zero application deploy and zero restart of any replica.
|
||||
ALTER TABLE customers ADD COLUMN email_address VARCHAR(320);
|
||||
|
||||
-- Backfill every row that existed before dual-write code shipped. Rows created
|
||||
-- *after* this point but before the dual-write code (Deploy 2) is fully rolled
|
||||
-- out are handled separately - see docs/07-the-backfill-window-bug.md.
|
||||
UPDATE customers SET email_address = email WHERE email_address IS NULL;
|
||||
|
||||
-- The other half of "expand": relax the constraint on the column we are about to
|
||||
-- retire. "email" is NOT NULL from V1. Leave that in place and Deploy 4's Stage 4
|
||||
-- code - which never writes "email" - fails every single INSERT with a NOT NULL
|
||||
-- violation the moment it starts, because the column it ignores is still mandatory.
|
||||
-- Forgetting this line is a genuine, reproducible failure - see
|
||||
-- docs/06-the-not-null-trap.md, which captures the exact exception it produces.
|
||||
ALTER TABLE customers ALTER COLUMN email DROP NOT NULL;
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
-- Deploy 4 (CONTRACT). Only safe once every replica in the fleet is confirmed
|
||||
-- running Stage 4 code, which never reads or writes "email". Run this too early
|
||||
-- and any Stage 1/2/3 instance still in the rolling deploy fails on its next
|
||||
-- write - see docs/10-what-happens-if-you-drop-too-soon.md.
|
||||
ALTER TABLE customers DROP COLUMN email;
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import com.ankurm.expandcontract.customer.Customer;
|
||||
import com.ankurm.expandcontract.customer.CustomerService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.jdbc.core.simple.JdbcClient;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* The gap between "the expand migration ran" and "every Stage 1 instance in the fleet
|
||||
* has been replaced by Stage 2" is not instantaneous - a real rolling deploy takes
|
||||
* minutes, and every row a lingering Stage 1 instance writes during that window has
|
||||
* {@code email_address = NULL}. The migration's one-time backfill (V2) only ever sees
|
||||
* rows that existed *before* Deploy 1 ran; it cannot see rows a Stage 1 instance writes
|
||||
* *after* that, during its own rollout window.
|
||||
* <p>
|
||||
* This test reproduces the resulting bug with a naive Stage 3 read (email_address alone)
|
||||
* and then shows the fix that ships in
|
||||
* {@link com.ankurm.expandcontract.customer.CustomerService#findById}: a plain
|
||||
* {@code COALESCE(email_address, email)}. See docs/07-the-backfill-window-bug.md.
|
||||
*/
|
||||
class BackfillWindowBugTest {
|
||||
|
||||
@Test
|
||||
void naiveStage3ReadReturnsNullForARowStage1WroteDuringTheRollout(@TempDir Path tmp) throws Exception {
|
||||
Path db = tmp.resolve("backfill-window");
|
||||
TestSupport.migrateTo(db, "2");
|
||||
Transcript t = Transcript.start("07-backfill-window-bug",
|
||||
"The backfill window: a Stage 1 write after Deploy 1, read by a naive Stage 3");
|
||||
|
||||
// A Stage 1 instance is still serving traffic during the Deploy 2 rollout and
|
||||
// writes a brand new row exactly as it always has - it has never heard of
|
||||
// email_address. This is not a hypothetical; it is guaranteed to happen for
|
||||
// however long the rollout takes.
|
||||
JdbcClient jdbc = TestSupport.jdbcClient(db);
|
||||
jdbc.sql("INSERT INTO customers(name, email) VALUES (?, ?)")
|
||||
.param("Katherine Johnson").param("[email protected]")
|
||||
.update();
|
||||
long insertedId = jdbc.sql("SELECT id FROM customers WHERE name = ?")
|
||||
.param("Katherine Johnson").query(Long.class).single();
|
||||
|
||||
t.section("the row a lingering Stage 1 instance just wrote");
|
||||
String naiveRead = jdbc.sql("SELECT name, email, email_address FROM customers WHERE id = ?")
|
||||
.param(insertedId).query().listOfRows().toString();
|
||||
t.line(naiveRead);
|
||||
assertThat(naiveRead).containsIgnoringCase("email_address=null");
|
||||
|
||||
t.section("a NAIVE Stage 3 read (email_address alone) - the bug");
|
||||
String naiveEmail = jdbc.sql("SELECT email_address FROM customers WHERE id = ?")
|
||||
.param(insertedId).query(String.class).optional().orElse(null);
|
||||
t.line("naive Stage 3 email column value: " + naiveEmail);
|
||||
assertThat(naiveEmail).isNull();
|
||||
|
||||
t.section("the SHIPPED Stage 3 read (CustomerService, COALESCE) - the fix");
|
||||
CustomerService stage3 = new CustomerService(jdbc, 3);
|
||||
Optional<Customer> fixed = stage3.findById(insertedId);
|
||||
t.line("CustomerService (stage 3) result: " + fixed);
|
||||
assertThat(fixed).isPresent();
|
||||
assertThat(fixed.get().email()).isEqualTo("[email protected]");
|
||||
|
||||
t.write();
|
||||
}
|
||||
}
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import com.ankurm.expandcontract.customer.Customer;
|
||||
import com.ankurm.expandcontract.customer.CustomerService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.jdbc.core.simple.JdbcClient;
|
||||
|
||||
import java.nio.file.Path;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* Deploy 4 (CONTRACT), in both directions. First the case that must work: once every
|
||||
* replica is confirmed on Stage 4 and the drop migration (V3) has run, Stage 4 code
|
||||
* keeps working exactly as before - it never referenced "email" to begin with. Second
|
||||
* the case that must fail loudly: a Stage 1, 2 or 3 instance that is somehow still
|
||||
* running against the post-drop schema (a rollback gone wrong, a forgotten canary)
|
||||
* gets a real SQL error the moment it tries to touch the column that no longer exists.
|
||||
* That failure is not a bug in this article's design - it is *why* Deploy 4 has to wait
|
||||
* for confirmation that the fleet is 100% on Stage 4 first. See
|
||||
* docs/09-the-contract-migration.md and docs/10-what-happens-if-you-drop-too-soon.md.
|
||||
*/
|
||||
class ContractSafetyTest {
|
||||
|
||||
@Test
|
||||
void stage4KeepsWorkingAfterTheColumnIsDropped(@TempDir Path tmp) throws Exception {
|
||||
Path db = tmp.resolve("contract-ok");
|
||||
TestSupport.migrateTo(db, "2");
|
||||
JdbcClient jdbc = TestSupport.jdbcClient(db);
|
||||
Transcript t = Transcript.start("09-contract-safety",
|
||||
"Deploy 4 (CONTRACT): Stage 4 after the drop, and what breaks if you drop too soon");
|
||||
|
||||
CustomerService stage4 = new CustomerService(jdbc, 4);
|
||||
long id = stage4.create("Annie Easley", "[email protected]");
|
||||
|
||||
// Deploy 4b: drop the old column, live, with Stage 4 already the only code running.
|
||||
TestSupport.migrateTo(db, "latest");
|
||||
|
||||
Customer afterDrop = stage4.findById(id).orElseThrow();
|
||||
t.section("Stage 4 read, after V3 dropped the email column");
|
||||
t.line(afterDrop.toString());
|
||||
assertThat(afterDrop.email()).isEqualTo("[email protected]");
|
||||
|
||||
long postDropId = stage4.create("Mary Allen Wilkes", "[email protected]");
|
||||
Customer postDrop = stage4.findById(postDropId).orElseThrow();
|
||||
t.section("Stage 4 create + read, entirely after the drop");
|
||||
t.line(postDrop.toString());
|
||||
assertThat(postDrop.email()).isEqualTo("[email protected]");
|
||||
|
||||
t.write();
|
||||
}
|
||||
|
||||
@Test
|
||||
void stage1CodeFailsLoudlyIfItIsStillRunningAfterTheColumnIsDropped(@TempDir Path tmp) throws Exception {
|
||||
Path db = tmp.resolve("contract-too-soon");
|
||||
TestSupport.migrateTo(db, "latest");
|
||||
JdbcClient jdbc = TestSupport.jdbcClient(db);
|
||||
Transcript t = Transcript.start("10-drop-too-soon",
|
||||
"What a lingering Stage 1 instance sees if the drop runs before it is retired");
|
||||
|
||||
CustomerService stage1 = new CustomerService(jdbc, 1);
|
||||
|
||||
Throwable thrown = catchThrowable(() -> stage1.create("Too Late", "[email protected]"));
|
||||
Throwable root = thrown;
|
||||
while (root.getCause() != null) {
|
||||
root = root.getCause();
|
||||
}
|
||||
t.line("Stage 1 create() after V3 dropped \"email\": " + thrown.getClass().getName());
|
||||
t.line("message: " + thrown.getMessage());
|
||||
t.line("root cause: " + root.getClass().getName() + ": " + root.getMessage());
|
||||
assertThat(root.getMessage()).contains("EMAIL").containsIgnoringCase("not found");
|
||||
|
||||
t.write();
|
||||
}
|
||||
|
||||
private static Throwable catchThrowable(org.assertj.core.api.ThrowableAssert.ThrowingCallable callable) {
|
||||
try {
|
||||
callable.call();
|
||||
return new AssertionError("expected an exception but none was thrown");
|
||||
} catch (Throwable t) {
|
||||
return t;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.ResultSetMetaData;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/** Renders a JDBC query as a plain-text table for transcripts - no ORM, no formatting library. */
|
||||
public final class DbDump {
|
||||
|
||||
private DbDump() {
|
||||
}
|
||||
|
||||
public static String table(Connection conn, String sql) {
|
||||
try (Statement st = conn.createStatement(); ResultSet rs = st.executeQuery(sql)) {
|
||||
ResultSetMetaData meta = rs.getMetaData();
|
||||
int cols = meta.getColumnCount();
|
||||
List<String> headers = new ArrayList<>();
|
||||
for (int i = 1; i <= cols; i++) {
|
||||
headers.add(meta.getColumnLabel(i));
|
||||
}
|
||||
List<List<String>> rows = new ArrayList<>();
|
||||
while (rs.next()) {
|
||||
List<String> row = new ArrayList<>();
|
||||
for (int i = 1; i <= cols; i++) {
|
||||
Object v = rs.getObject(i);
|
||||
row.add(v == null ? "NULL" : v.toString());
|
||||
}
|
||||
rows.add(row);
|
||||
}
|
||||
int[] widths = new int[cols];
|
||||
for (int i = 0; i < cols; i++) {
|
||||
widths[i] = headers.get(i).length();
|
||||
}
|
||||
for (List<String> row : rows) {
|
||||
for (int i = 0; i < cols; i++) {
|
||||
widths[i] = Math.max(widths[i], row.get(i).length());
|
||||
}
|
||||
}
|
||||
StringBuilder sb = new StringBuilder();
|
||||
appendRow(sb, headers, widths);
|
||||
StringBuilder sep = new StringBuilder();
|
||||
for (int i = 0; i < cols; i++) {
|
||||
sep.append("-".repeat(widths[i])).append(i < cols - 1 ? "-+-" : "");
|
||||
}
|
||||
sb.append(sep).append(System.lineSeparator());
|
||||
for (List<String> row : rows) {
|
||||
appendRow(sb, row, widths);
|
||||
}
|
||||
sb.append("(").append(rows.size()).append(" row").append(rows.size() == 1 ? "" : "s").append(")");
|
||||
return sb.toString();
|
||||
}
|
||||
catch (SQLException ex) {
|
||||
return "query failed: " + ex.getMessage();
|
||||
}
|
||||
}
|
||||
|
||||
private static void appendRow(StringBuilder sb, List<String> values, int[] widths) {
|
||||
for (int i = 0; i < values.size(); i++) {
|
||||
sb.append(pad(values.get(i), widths[i]));
|
||||
sb.append(i < values.size() - 1 ? " | " : "");
|
||||
}
|
||||
sb.append(System.lineSeparator());
|
||||
}
|
||||
|
||||
private static String pad(String s, int width) {
|
||||
return s + " ".repeat(Math.max(0, width - s.length()));
|
||||
}
|
||||
}
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import com.ankurm.expandcontract.customer.CustomerService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.jdbc.core.simple.JdbcClient;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* The finding that {@link com.ankurm.expandcontract.customer.CustomerService#withRetryForConcurrentDdl}
|
||||
* cannot fix, because nothing throws. The retry helper exists for the case where a concurrent
|
||||
* statement collides with an in-progress {@code ALTER TABLE} and H2 answers with
|
||||
* "table not found" - a real error the caller can see and retry. This test demonstrates a second,
|
||||
* stranger failure mode found the same way the first one was: by running this module's own load
|
||||
* generator against a live rolling deploy and noticing a handful of reads and updates come back
|
||||
* 404 for a customer id that a 201 response had already confirmed existed.
|
||||
* <p>
|
||||
* H2 implements both {@code ALTER TABLE ... ADD COLUMN} and {@code ALTER TABLE ... DROP COLUMN}
|
||||
* by rebuilding the table: copying every row into a new table with the new column layout and
|
||||
* swapping it in. If an ordinary {@code INSERT} on another connection commits - with no error,
|
||||
* with a generated key handed back to the caller - while that rebuild is in flight, the inserted
|
||||
* row can be copied into the new table or left behind in the old one depending on exactly when
|
||||
* the rebuild's internal scan ran relative to the commit. When it is left behind, the row is gone
|
||||
* the instant the rebuild finishes, and nothing on the inserting connection was ever told.
|
||||
* <p>
|
||||
* This is not how every database implements {@code ADD COLUMN} and {@code DROP COLUMN}. PostgreSQL's
|
||||
* reference manual is explicit that both are metadata-only operations on tables like this one -
|
||||
* no non-volatile default and no immediate space reclamation requested - so this is a property of
|
||||
* H2's implementation, not of the expand-contract technique itself. See
|
||||
* docs/14-the-ddl-lock-window.md for the full explanation, the standalone reproduction this test
|
||||
* is built from, and what it implies for choosing a target database for a real migration.
|
||||
*/
|
||||
class DdlSilentDataLossTest {
|
||||
|
||||
// Whether the rebuild's internal scan actually passes a given row before or after that
|
||||
// row's INSERT commits is OS scheduling, not application logic - so a single attempt at
|
||||
// this race can genuinely land on either side of it. A single-attempt version of this test
|
||||
// failed about 1 run in 5 while writing it. Rather than assert on one attempt (flaky either
|
||||
// way) or loosen the assertion to "zero or more" (which would silently stop proving anything
|
||||
// the day this stops reproducing), this test repeats the race, on a fresh table each time,
|
||||
// until it reproduces - the same thing a human would do at a terminal to confirm a suspected
|
||||
// race is real. Twenty attempts reproduced it within the first 4 in every run made while
|
||||
// writing this test.
|
||||
private static final int MAX_ATTEMPTS = 20;
|
||||
|
||||
@Test
|
||||
void concurrentAlterTableAddColumnCanSilentlyDropAnAlreadyCommittedInsert(@TempDir Path tmp) throws Exception {
|
||||
Transcript t = Transcript.start("14-ddl-silent-data-loss",
|
||||
"The failure the retry cannot catch: a committed INSERT that ALTER TABLE loses silently");
|
||||
|
||||
for (int attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
|
||||
Path db = tmp.resolve("silent-data-loss-" + attempt);
|
||||
TestSupport.migrateTo(db, "1");
|
||||
JdbcClient jdbc = TestSupport.jdbcClient(db);
|
||||
CustomerService stage1 = new CustomerService(jdbc, 1);
|
||||
|
||||
AtomicBoolean stop = new AtomicBoolean(false);
|
||||
List<Long> confirmedIds = new CopyOnWriteArrayList<>();
|
||||
List<String> insertErrors = new CopyOnWriteArrayList<>();
|
||||
CountDownLatch started = new CountDownLatch(1);
|
||||
|
||||
Thread inserter = new Thread(() -> {
|
||||
started.countDown();
|
||||
int n = 0;
|
||||
while (!stop.get()) {
|
||||
n++;
|
||||
try {
|
||||
long id = stage1.create("Concurrent " + n, "concurrent" + n + "@example.test");
|
||||
confirmedIds.add(id);
|
||||
} catch (Exception ex) {
|
||||
// The already-documented, already-fixed failure mode: a statement that
|
||||
// collides with the DDL and is told so. Counted here only to show it is
|
||||
// rare and separate from the silent loss this test is isolating.
|
||||
insertErrors.add(ex.getClass().getSimpleName());
|
||||
}
|
||||
}
|
||||
});
|
||||
inserter.start();
|
||||
started.await();
|
||||
|
||||
// The DDL runs on its own connection, exactly as Deploy 1 (EXPAND) runs V2 live
|
||||
// while both replicas keep taking traffic - see scripts/run-all.sh.
|
||||
TestSupport.migrateTo(db, "2");
|
||||
|
||||
stop.set(true);
|
||||
inserter.join();
|
||||
|
||||
int missing = 0;
|
||||
List<Long> missingIds = new java.util.ArrayList<>();
|
||||
for (Long id : confirmedIds) {
|
||||
if (jdbc.sql("SELECT id FROM customers WHERE id = ?").param(id).query().listOfRows().isEmpty()) {
|
||||
missing++;
|
||||
missingIds.add(id);
|
||||
}
|
||||
}
|
||||
|
||||
if (missing == 0 && attempt < MAX_ATTEMPTS) {
|
||||
continue;
|
||||
}
|
||||
|
||||
t.line("attempts needed to reproduce the race: " + attempt + " of " + MAX_ATTEMPTS);
|
||||
t.line("customer creates that returned a generated id with no error: " + confirmedIds.size());
|
||||
t.line("customer creates that got the already-documented, already-fixed DDL-collision error: " + insertErrors.size());
|
||||
t.line("of the ids that came back with no error, missing from the table once V2 finished: " + missing);
|
||||
if (!missingIds.isEmpty()) {
|
||||
t.line("example missing ids: " + missingIds.subList(0, Math.min(5, missingIds.size())));
|
||||
}
|
||||
t.line("");
|
||||
t.line("This is why the retry in CustomerService cannot be the whole fix: these inserts");
|
||||
t.line("never threw anything to retry. The row was committed, then discarded when the");
|
||||
t.line("ADD COLUMN rebuild swapped in a new table that had already been scanned.");
|
||||
t.write();
|
||||
|
||||
// If this fails on the very last attempt, either H2's rebuild strategy changed
|
||||
// (worth its own investigation) or this environment schedules threads differently
|
||||
// enough that this test needs a heavier inserter - not that the finding is wrong.
|
||||
assertThat(missing)
|
||||
.withFailMessage("expected at least one silently-lost row within %d attempts; "
|
||||
+ "either H2's ALTER TABLE implementation changed, or this environment "
|
||||
+ "needs a heavier concurrent inserter to reproduce the race", MAX_ATTEMPTS)
|
||||
.isGreaterThan(0);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import com.ankurm.expandcontract.customer.CustomerService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* Deploy 2 (MIGRATE WRITES): Stage 2 code writes every create and update to both
|
||||
* columns. This is the deploy that makes Deploy 3's read switch safe later - if this
|
||||
* one is wrong, nothing downstream can be trusted. See docs/04-the-dual-write.md.
|
||||
*/
|
||||
class DualWriteConsistencyTest {
|
||||
|
||||
@Test
|
||||
void createAndUpdatePopulateBothColumnsIdentically(@TempDir Path tmp) throws Exception {
|
||||
Path db = tmp.resolve("dual-write");
|
||||
TestSupport.migrateTo(db, "2");
|
||||
Transcript t = Transcript.start("04-dual-write-consistency",
|
||||
"Deploy 2 (MIGRATE WRITES): Stage 2 writes land in both columns");
|
||||
|
||||
CustomerService stage2 = new CustomerService(TestSupport.jdbcClient(db), 2);
|
||||
long id = stage2.create("Margaret Hamilton", "[email protected]");
|
||||
|
||||
try (Connection conn = DriverManager.getConnection(TestSupport.jdbcUrl(db), "sa", "")) {
|
||||
t.section("after create()");
|
||||
String row = DbDump.table(conn, "select name, email, email_address from customers where id = " + id);
|
||||
t.line(row);
|
||||
assertThat(row).contains("[email protected]");
|
||||
assertThat(row.indexOf("[email protected]")).isNotEqualTo(row.lastIndexOf("[email protected]"));
|
||||
}
|
||||
|
||||
stage2.updateEmail(id, "[email protected]");
|
||||
|
||||
try (Connection conn = DriverManager.getConnection(TestSupport.jdbcUrl(db), "sa", "")) {
|
||||
t.section("after updateEmail() - the old value is gone from BOTH columns, not just one");
|
||||
String row = DbDump.table(conn, "select name, email, email_address from customers where id = " + id);
|
||||
t.line(row);
|
||||
assertThat(row).contains("[email protected]").doesNotContain("[email protected]");
|
||||
}
|
||||
|
||||
t.write();
|
||||
}
|
||||
}
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.jdbc.core.simple.JdbcClient;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* Deploy 1 (EXPAND) in isolation: migrate to V2 and confirm two things a real rollout
|
||||
* depends on. First, that every pre-existing row got backfilled in the same migration.
|
||||
* Second - the part that is easy to get wrong - that Stage 1 code, completely unaware
|
||||
* the new column exists, still inserts rows exactly as it always has. If this second
|
||||
* assertion ever failed, the migration would not be additive and expand-contract would
|
||||
* not apply to it. See docs/02-the-expand-migration.md.
|
||||
*/
|
||||
class ExpandMigrationBackwardCompatibleTest {
|
||||
|
||||
@Test
|
||||
void backfillsExistingRowsAndStaysCompatibleWithStage1Code(@TempDir Path tmp) throws Exception {
|
||||
Path db = tmp.resolve("expand");
|
||||
Transcript t = Transcript.start("02-expand-backward-compatible",
|
||||
"Deploy 1 (EXPAND): additive column + backfill, Stage 1 code untouched");
|
||||
|
||||
// Seed one row the way Stage 1 always has, before the expand migration exists.
|
||||
TestSupport.migrateTo(db, "1");
|
||||
JdbcClient preExpand = TestSupport.jdbcClient(db);
|
||||
preExpand.sql("INSERT INTO customers(name, email) VALUES (?, ?)")
|
||||
.param("Ada Lovelace").param("[email protected]").update();
|
||||
|
||||
t.section("schema before Deploy 1");
|
||||
try (Connection conn = DriverManager.getConnection(TestSupport.jdbcUrl(db), "sa", "")) {
|
||||
t.line(DbDump.table(conn, "select column_name from information_schema.columns "
|
||||
+ "where table_name = 'CUSTOMERS' order by ordinal_position"));
|
||||
}
|
||||
|
||||
// Deploy 1: the expand migration runs against the live database. No app restart.
|
||||
TestSupport.migrateTo(db, "2");
|
||||
|
||||
t.section("schema after Deploy 1 (email_address added)");
|
||||
try (Connection conn = DriverManager.getConnection(TestSupport.jdbcUrl(db), "sa", "")) {
|
||||
t.line(DbDump.table(conn, "select column_name from information_schema.columns "
|
||||
+ "where table_name = 'CUSTOMERS' order by ordinal_position"));
|
||||
|
||||
t.section("Ada's row was backfilled by the migration itself");
|
||||
String row = DbDump.table(conn, "select name, email, email_address from customers where name = 'Ada Lovelace'");
|
||||
t.line(row);
|
||||
// Backfilled: both columns hold the same value for a row that predates dual-write code.
|
||||
assertThat(row).contains("[email protected]");
|
||||
assertThat(row.indexOf("[email protected]")).isNotEqualTo(row.lastIndexOf("[email protected]"));
|
||||
}
|
||||
|
||||
// Stage 1 code has not been redeployed and does not know email_address exists.
|
||||
// Its insert statement is byte-for-byte what it was before Deploy 1.
|
||||
JdbcClient postExpand = TestSupport.jdbcClient(db);
|
||||
postExpand.sql("INSERT INTO customers(name, email) VALUES (?, ?)")
|
||||
.param("Grace Hopper").param("[email protected]").update();
|
||||
|
||||
t.section("Stage 1's original INSERT still works, unmodified, after the migration");
|
||||
try (Connection conn = DriverManager.getConnection(TestSupport.jdbcUrl(db), "sa", "")) {
|
||||
String row = DbDump.table(conn, "select name, email, email_address from customers where name = 'Grace Hopper'");
|
||||
t.line(row);
|
||||
assertThat(row).contains("[email protected]").contains("NULL");
|
||||
}
|
||||
|
||||
t.write();
|
||||
}
|
||||
}
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import com.ankurm.expandcontract.customer.Customer;
|
||||
import com.ankurm.expandcontract.customer.CustomerService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.jdbc.core.simple.JdbcClient;
|
||||
|
||||
import java.nio.file.Path;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* The property the whole article rests on: during a rolling deploy, two adjacent
|
||||
* stages are serving traffic against the SAME database at the SAME time, for however
|
||||
* long the rollout takes. If a customer written by one stage cannot be read correctly
|
||||
* by the other, the deploy is not zero-downtime - it is a race with the rollout clock.
|
||||
* <p>
|
||||
* This test builds two {@link CustomerService} instances that share one
|
||||
* {@link JdbcClient} (standing in for one shared database, hit by two replicas on
|
||||
* consecutive stages) and cross-checks every write/read direction for the three
|
||||
* rollouts this article performs: Stage 1↔2, Stage 2↔3, and Stage 3↔4.
|
||||
* See docs/08-the-rolling-window-proof.md - this is the test the live load generator
|
||||
* run in the article is reproducing under real HTTP and real timing.
|
||||
*/
|
||||
class MixedStageRollingWindowTest {
|
||||
|
||||
@Test
|
||||
void everyAdjacentStagePairReadsWhatTheOtherWrote(@TempDir Path tmp) throws Exception {
|
||||
Path db = tmp.resolve("rolling-window");
|
||||
Transcript t = Transcript.start("08-mixed-stage-rolling-window",
|
||||
"Cross-stage consistency during each of the three rolling deploys");
|
||||
|
||||
// Deploy 2's rollout: some replicas still on Stage 1, some already on Stage 2.
|
||||
TestSupport.migrateTo(db, "2");
|
||||
JdbcClient jdbc = TestSupport.jdbcClient(db);
|
||||
CustomerService stage1 = new CustomerService(jdbc, 1);
|
||||
CustomerService stage2 = new CustomerService(jdbc, 2);
|
||||
|
||||
t.section("Stage 1 writes, Stage 2 reads");
|
||||
long a = stage1.create("Radia Perlman", "[email protected]");
|
||||
Customer readByStage2 = stage2.findById(a).orElseThrow();
|
||||
t.line(readByStage2.toString());
|
||||
assertThat(readByStage2.email()).isEqualTo("[email protected]");
|
||||
|
||||
t.section("Stage 2 writes, Stage 1 reads");
|
||||
long b = stage2.create("Barbara Liskov", "[email protected]");
|
||||
Customer readByStage1 = stage1.findById(b).orElseThrow();
|
||||
t.line(readByStage1.toString());
|
||||
assertThat(readByStage1.email()).isEqualTo("[email protected]");
|
||||
|
||||
// Deploy 3's rollout: some replicas on Stage 2, some already on Stage 3.
|
||||
CustomerService stage3 = new CustomerService(jdbc, 3);
|
||||
|
||||
t.section("Stage 2 writes, Stage 3 reads");
|
||||
long c = stage2.create("Shafi Goldwasser", "[email protected]");
|
||||
Customer readByStage3 = stage3.findById(c).orElseThrow();
|
||||
t.line(readByStage3.toString());
|
||||
assertThat(readByStage3.email()).isEqualTo("[email protected]");
|
||||
|
||||
t.section("Stage 3 writes, Stage 2 reads");
|
||||
long d = stage3.create("Frances Allen", "[email protected]");
|
||||
Customer readByStage2Again = stage2.findById(d).orElseThrow();
|
||||
t.line(readByStage2Again.toString());
|
||||
assertThat(readByStage2Again.email()).isEqualTo("[email protected]");
|
||||
|
||||
// Deploy 4a's rollout: some replicas on Stage 3, some already on Stage 4.
|
||||
// email_address has been fully backfilled and dual-written for two whole
|
||||
// deploys by this point - the precondition Stage 4 relies on. The old "email"
|
||||
// column is still physically present (Deploy 4b, the drop, has not run yet)
|
||||
// but Stage 4 code never looks at it.
|
||||
CustomerService stage4 = new CustomerService(jdbc, 4);
|
||||
|
||||
t.section("Stage 3 writes, Stage 4 reads");
|
||||
long e = stage3.create("Adele Goldberg", "[email protected]");
|
||||
Customer readByStage4 = stage4.findById(e).orElseThrow();
|
||||
t.line(readByStage4.toString());
|
||||
assertThat(readByStage4.email()).isEqualTo("[email protected]");
|
||||
|
||||
t.section("Stage 4 writes, Stage 3 reads");
|
||||
long f = stage4.create("Karen Sparck Jones", "[email protected]");
|
||||
Customer readByStage3Again = stage3.findById(f).orElseThrow();
|
||||
t.line(readByStage3Again.toString());
|
||||
assertThat(readByStage3Again.email()).isEqualTo("[email protected]");
|
||||
|
||||
t.write();
|
||||
}
|
||||
}
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import com.ankurm.expandcontract.customer.CustomerService;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.springframework.jdbc.core.simple.JdbcClient;
|
||||
|
||||
import java.nio.file.Path;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* A trap this article's own first draft of the V2 migration fell into: adding
|
||||
* {@code email_address} is not the whole story of "expand" when the column being
|
||||
* retired is {@code NOT NULL}. Stage 4 code never writes "email" - so if "email" is
|
||||
* still mandatory when Stage 4 starts, every create fails, in production, the moment
|
||||
* that deploy reaches its first replica. The fix is one more line in the SAME
|
||||
* migration: {@code ALTER TABLE customers ALTER COLUMN email DROP NOT NULL}. See
|
||||
* docs/06-the-not-null-trap.md, and compare
|
||||
* {@link com.ankurm.expandcontract.customer.CustomerService#create} - the shipped
|
||||
* V2 migration (with the fix) is what every other test in this module runs against.
|
||||
*/
|
||||
class NotNullConstraintTrapTest {
|
||||
|
||||
@Test
|
||||
void stage4FailsIfTheOldColumnIsStillMandatory(@TempDir Path tmp) throws Exception {
|
||||
Path db = tmp.resolve("not-null-trap");
|
||||
TestSupport.migrateTo(db, "1");
|
||||
JdbcClient jdbc = TestSupport.jdbcClient(db);
|
||||
Transcript t = Transcript.start("06-not-null-trap",
|
||||
"The NOT NULL trap: expand without relaxing the old column's constraint");
|
||||
|
||||
// The NAIVE version of V2: add the column, backfill, stop there. This is
|
||||
// exactly V2 minus its last line.
|
||||
jdbc.sql("ALTER TABLE customers ADD COLUMN email_address VARCHAR(320)").update();
|
||||
jdbc.sql("UPDATE customers SET email_address = email WHERE email_address IS NULL").update();
|
||||
|
||||
CustomerService stage4 = new CustomerService(jdbc, 4);
|
||||
Throwable thrown = null;
|
||||
try {
|
||||
stage4.create("Too Early", "[email protected]");
|
||||
} catch (Throwable ex) {
|
||||
thrown = ex;
|
||||
}
|
||||
t.section("Stage 4 create() against the NAIVE migration (no DROP NOT NULL)");
|
||||
assertThat(thrown).isNotNull();
|
||||
Throwable root = thrown;
|
||||
while (root.getCause() != null) {
|
||||
root = root.getCause();
|
||||
}
|
||||
t.line(thrown.getClass().getName() + ": " + thrown.getMessage());
|
||||
t.line("root cause: " + root.getClass().getName() + ": " + root.getMessage());
|
||||
assertThat(root.getMessage()).contains("NULL not allowed for column \"EMAIL\"");
|
||||
|
||||
t.section("Stage 4 create() against the SHIPPED V2 migration (DROP NOT NULL included)");
|
||||
Path fixedDb = tmp.resolve("not-null-fixed");
|
||||
TestSupport.migrateTo(fixedDb, "2");
|
||||
CustomerService fixedStage4 = new CustomerService(TestSupport.jdbcClient(fixedDb), 4);
|
||||
long id = fixedStage4.create("On Time", "[email protected]");
|
||||
var result = fixedStage4.findById(id).orElseThrow();
|
||||
t.line(result.toString());
|
||||
assertThat(result.email()).isEqualTo("[email protected]");
|
||||
|
||||
t.write();
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import org.springframework.boot.autoconfigure.AutoConfigurations;
|
||||
import org.springframework.boot.flyway.autoconfigure.FlywayAutoConfiguration;
|
||||
import org.springframework.boot.jdbc.autoconfigure.DataSourceAutoConfiguration;
|
||||
import org.springframework.boot.test.context.runner.ApplicationContextRunner;
|
||||
import org.springframework.jdbc.core.simple.JdbcClient;
|
||||
import org.springframework.jdbc.datasource.DriverManagerDataSource;
|
||||
|
||||
import java.nio.file.Path;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* Runs Flyway to a given target version through Boot's own {@code FlywayAutoConfiguration}
|
||||
* (the same {@code spring.flyway.target} property a real deploy pipeline would set), then
|
||||
* hands the test a plain {@link JdbcClient} against the same file - so a test can migrate a
|
||||
* database to "however far Deploy N has gotten" and then exercise
|
||||
* {@link com.ankurm.expandcontract.customer.CustomerService} instances directly, with no
|
||||
* Spring context of their own, exactly as the app constructs them.
|
||||
*/
|
||||
final class TestSupport {
|
||||
|
||||
static void migrateTo(Path dbPath, String target) {
|
||||
new ApplicationContextRunner()
|
||||
.withConfiguration(AutoConfigurations.of(DataSourceAutoConfiguration.class, FlywayAutoConfiguration.class))
|
||||
.withPropertyValues(
|
||||
"spring.datasource.url=" + jdbcUrl(dbPath),
|
||||
"spring.datasource.username=sa",
|
||||
"spring.flyway.locations=classpath:db/migration",
|
||||
"spring.flyway.target=" + target)
|
||||
.run(ctx -> assertThat(ctx).hasNotFailed());
|
||||
}
|
||||
|
||||
static JdbcClient jdbcClient(Path dbPath) {
|
||||
DriverManagerDataSource ds = new DriverManagerDataSource(jdbcUrl(dbPath), "sa", "");
|
||||
return JdbcClient.create(ds);
|
||||
}
|
||||
|
||||
static String jdbcUrl(Path dbPath) {
|
||||
return "jdbc:h2:file:" + dbPath + ";AUTO_SERVER=TRUE";
|
||||
}
|
||||
|
||||
private TestSupport() {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.time.Instant;
|
||||
|
||||
/**
|
||||
* Writes a named transcript under {@code docs/output/} while the test that produced it runs.
|
||||
* Every console line quoted in the companion article for this module comes from one of these
|
||||
* files, and the file is only ever written by a test assertion - never hand-typed.
|
||||
*/
|
||||
public final class Transcript {
|
||||
|
||||
private static final Path OUTPUT_DIR = Paths.get("docs", "output");
|
||||
|
||||
private final StringBuilder buffer = new StringBuilder();
|
||||
private final String name;
|
||||
|
||||
private Transcript(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public static Transcript start(String name, String heading) {
|
||||
Transcript t = new Transcript(name);
|
||||
t.line("=".repeat(Math.min(78, heading.length() + 4)));
|
||||
t.line(heading);
|
||||
t.line("=".repeat(Math.min(78, heading.length() + 4)));
|
||||
t.line("captured: " + Instant.now());
|
||||
t.line("");
|
||||
return t;
|
||||
}
|
||||
|
||||
public Transcript line(String text) {
|
||||
buffer.append(text).append(System.lineSeparator());
|
||||
System.out.println(text);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Transcript section(String title) {
|
||||
line("");
|
||||
line("-- " + title + " --");
|
||||
return this;
|
||||
}
|
||||
|
||||
public void write() {
|
||||
try {
|
||||
Files.createDirectories(OUTPUT_DIR);
|
||||
Path target = OUTPUT_DIR.resolve(name + ".txt");
|
||||
Files.writeString(target, buffer.toString(), StandardOpenOption.CREATE,
|
||||
StandardOpenOption.TRUNCATE_EXISTING);
|
||||
}
|
||||
catch (IOException ex) {
|
||||
throw new UncheckedIOException(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user