Add db-migrations-expand-contract: zero-downtime schema migrations proven with a real 4-deploy rolling run
Companion code for Zero-Downtime Database Migrations: Expand-Contract in Practice with Spring Boot: a full expand/migrate-writes/migrate-reads/contract sequence run as an actual rolling deploy across two live replicas, with a load generator sending continuous HTTP traffic through all four deploys (99.98% success, every residual error traced to a root cause rather than left unexplained). Findings include a real NOT NULL constraint trap in the expand migration, a backfill-window bug in the read switch, H2's AUTO_SERVER=TRUE single-point-of-failure behavior under a rolling restart, the drain-before-SIGTERM fix needed to close a health-check gap during graceful shutdown, and H2 silently discarding a concurrently committed INSERT during an ALTER TABLE ADD/DROP COLUMN rebuild - confirmed, by primary source, to be an H2-specific behavior rather than a property of the technique itself. Co-Authored-By: Claude Sonnet 5 <[email protected]> Claude-Session: https://claude.ai/code/session_019Fb7vW8vLyLKngBc4R3huA
This commit is contained in:
Claude Sonnet 5
+21
@@ -0,0 +1,21 @@
|
||||
package com.ankurm.expandcontract;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
/**
|
||||
* One jar, four behaviours. Which one a running instance exhibits is picked by
|
||||
* {@code app.stage} (1-4), read by {@link com.ankurm.expandcontract.customer.CustomerService}.
|
||||
* This is the same "wire variants behind a property so one artifact can demonstrate
|
||||
* every stage" approach used across ankurm.com's companion repos - the point is that a
|
||||
* reader can run two instances on two stages against the same database and watch the
|
||||
* rolling deploy for themselves, rather than reading about it.
|
||||
* <p>
|
||||
* See docs/01-the-problem-and-the-plan.md.
|
||||
*/
|
||||
@SpringBootApplication
|
||||
public class ExpandContractApplication {
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(ExpandContractApplication.class, args);
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
package com.ankurm.expandcontract.customer;
|
||||
|
||||
/**
|
||||
* The response shape the API always returns, whatever stage is answering. A client
|
||||
* of this service is never aware that "email" moved to "email_address" underneath it
|
||||
* - that is the entire point of doing this as expand-contract instead of a single
|
||||
* breaking rename. See docs/01-the-problem-and-the-plan.md.
|
||||
*/
|
||||
public record Customer(long id, String name, String email) {
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
package com.ankurm.expandcontract.customer;
|
||||
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
/**
|
||||
* The API contract that never changes across all four deploys - see
|
||||
* docs/01-the-problem-and-the-plan.md. A client of this controller, including the
|
||||
* load generator in {@link com.ankurm.expandcontract.loadgen.LoadGenerator}, cannot
|
||||
* tell which stage answered a given request just by looking at the response shape.
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping("/customers")
|
||||
public class CustomerController {
|
||||
|
||||
private final CustomerService service;
|
||||
|
||||
public CustomerController(CustomerService service) {
|
||||
this.service = service;
|
||||
}
|
||||
|
||||
public record CreateCustomerRequest(String name, String email) {
|
||||
}
|
||||
|
||||
public record UpdateEmailRequest(String email) {
|
||||
}
|
||||
|
||||
@PostMapping
|
||||
public ResponseEntity<Customer> create(@RequestBody CreateCustomerRequest request) {
|
||||
long id = service.create(request.name(), request.email());
|
||||
Customer created = service.findById(id).orElseThrow();
|
||||
return ResponseEntity.status(HttpStatus.CREATED).body(created);
|
||||
}
|
||||
|
||||
@GetMapping("/{id}")
|
||||
public Customer get(@PathVariable long id) {
|
||||
return service.findById(id).orElseThrow(() -> new CustomerService.CustomerNotFoundException(id));
|
||||
}
|
||||
|
||||
@PutMapping("/{id}/email")
|
||||
public Customer updateEmail(@PathVariable long id, @RequestBody UpdateEmailRequest request) {
|
||||
service.updateEmail(id, request.email());
|
||||
return service.findById(id).orElseThrow();
|
||||
}
|
||||
|
||||
@ExceptionHandler(CustomerService.CustomerNotFoundException.class)
|
||||
public ResponseEntity<String> notFound(CustomerService.CustomerNotFoundException ex) {
|
||||
return ResponseEntity.status(HttpStatus.NOT_FOUND).body(ex.getMessage());
|
||||
}
|
||||
}
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
package com.ankurm.expandcontract.customer;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.jdbc.BadSqlGrammarException;
|
||||
import org.springframework.jdbc.core.simple.JdbcClient;
|
||||
import org.springframework.jdbc.support.GeneratedKeyHolder;
|
||||
import org.springframework.jdbc.support.KeyHolder;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.Optional;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* Every behaviour this article talks about lives in these four branches. Nothing
|
||||
* else in the codebase changes between Stage 1 and Stage 4 - the controller, the
|
||||
* table, the HTTP contract and the response shape are all identical throughout.
|
||||
* Only which column this class writes to and reads from changes, and it changes
|
||||
* exactly once per deploy. See docs/01-the-problem-and-the-plan.md for the mental
|
||||
* model and docs/04-the-dual-write.md / docs/05-the-read-switch.md for why the
|
||||
* write switch and the read switch are not the same deploy.
|
||||
*/
|
||||
@Service
|
||||
public class CustomerService {
|
||||
|
||||
private final JdbcClient jdbc;
|
||||
private final int stage;
|
||||
|
||||
public CustomerService(JdbcClient jdbc, @Value("${app.stage}") int stage) {
|
||||
this.jdbc = jdbc;
|
||||
if (stage < 1 || stage > 4) {
|
||||
throw new IllegalArgumentException("app.stage must be 1-4, got " + stage);
|
||||
}
|
||||
this.stage = stage;
|
||||
}
|
||||
|
||||
public int stage() {
|
||||
return stage;
|
||||
}
|
||||
|
||||
public long create(String name, String email) {
|
||||
return withRetryForConcurrentDdl(() -> {
|
||||
KeyHolder keyHolder = new GeneratedKeyHolder();
|
||||
switch (stage) {
|
||||
case 1 -> jdbc.sql("INSERT INTO customers(name, email) VALUES (?, ?)")
|
||||
.param(name).param(email)
|
||||
.update(keyHolder, "id");
|
||||
case 2, 3 -> jdbc.sql("INSERT INTO customers(name, email, email_address) VALUES (?, ?, ?)")
|
||||
.param(name).param(email).param(email)
|
||||
.update(keyHolder, "id");
|
||||
case 4 -> jdbc.sql("INSERT INTO customers(name, email_address) VALUES (?, ?)")
|
||||
.param(name).param(email)
|
||||
.update(keyHolder, "id");
|
||||
default -> throw new IllegalStateException();
|
||||
}
|
||||
return keyHolder.getKey().longValue();
|
||||
});
|
||||
}
|
||||
|
||||
public void updateEmail(long id, String newEmail) {
|
||||
int updated = withRetryForConcurrentDdl(() -> switch (stage) {
|
||||
case 1 -> jdbc.sql("UPDATE customers SET email = ? WHERE id = ?")
|
||||
.param(newEmail).param(id).update();
|
||||
case 2, 3 -> jdbc.sql("UPDATE customers SET email = ?, email_address = ? WHERE id = ?")
|
||||
.param(newEmail).param(newEmail).param(id).update();
|
||||
case 4 -> jdbc.sql("UPDATE customers SET email_address = ? WHERE id = ?")
|
||||
.param(newEmail).param(id).update();
|
||||
default -> throw new IllegalStateException();
|
||||
});
|
||||
if (updated == 0) {
|
||||
throw new CustomerNotFoundException(id);
|
||||
}
|
||||
}
|
||||
|
||||
public Optional<Customer> findById(long id) {
|
||||
// Stage 3 reads COALESCE(email_address, email) rather than email_address alone.
|
||||
// Without it, a row written by a Stage 1 instance during the rollout window
|
||||
// between the expand migration and the dual-write deploy - one that has never
|
||||
// been dual-written at all - reads back with a null email the moment a Stage 3
|
||||
// instance answers the request. Stage 2 never needs this: it still reads the
|
||||
// original column, which every stage always keeps populated. See
|
||||
// docs/07-the-backfill-window-bug.md, where this line is the fix for a test
|
||||
// that fails without it.
|
||||
//
|
||||
// A read can hit the same DDL lock window Deploy 4b's writes can - see the
|
||||
// Javadoc on withRetryForConcurrentDdl below - so it gets the same retry.
|
||||
String sql = switch (stage) {
|
||||
case 1, 2 -> "SELECT id, name, email AS email FROM customers WHERE id = ?";
|
||||
case 3 -> "SELECT id, name, COALESCE(email_address, email) AS email FROM customers WHERE id = ?";
|
||||
case 4 -> "SELECT id, name, email_address AS email FROM customers WHERE id = ?";
|
||||
default -> throw new IllegalStateException();
|
||||
};
|
||||
return withRetryForConcurrentDdl(() -> jdbc.sql(sql).param(id)
|
||||
.query(Customer.class)
|
||||
.optional());
|
||||
}
|
||||
|
||||
/**
|
||||
* Deploy 4b (the DROP COLUMN migration) is the one deploy in this whole sequence
|
||||
* that is NOT invisible to concurrent traffic on this database: for roughly the
|
||||
* duration of that single ALTER TABLE statement, H2's TCP server can answer a
|
||||
* completely unrelated, already-correct query with "Table CUSTOMERS not found" -
|
||||
* caught live, twice, in independent runs of this module's own load generator.
|
||||
* That is a real, narrow, and transient condition, not a bug in this class's SQL,
|
||||
* so it gets a single scoped retry rather than being allowed to fail the request.
|
||||
* See docs/14-the-ddl-lock-window.md for the captured stack trace and why this is
|
||||
* not the same thing as retrying a genuine programming error.
|
||||
*/
|
||||
private <T> T withRetryForConcurrentDdl(Supplier<T> operation) {
|
||||
try {
|
||||
return operation.get();
|
||||
} catch (BadSqlGrammarException ex) {
|
||||
if (ex.getMessage() != null && ex.getMessage().contains("CUSTOMERS")) {
|
||||
try {
|
||||
Thread.sleep(50);
|
||||
} catch (InterruptedException ie) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
return operation.get();
|
||||
}
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
public static class CustomerNotFoundException extends RuntimeException {
|
||||
public CustomerNotFoundException(long id) {
|
||||
super("No customer with id " + id);
|
||||
}
|
||||
}
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
package com.ankurm.expandcontract.diag;
|
||||
|
||||
import org.springframework.boot.availability.AvailabilityChangeEvent;
|
||||
import org.springframework.boot.availability.ReadinessState;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* The step a rolling deploy needs that {@code server.shutdown: graceful} alone does
|
||||
* not give you: a way to tell the load balancer "stop sending me new work" BEFORE
|
||||
* the process is asked to stop. Graceful shutdown only starts refusing new
|
||||
* connections once SIGTERM has already been sent - which is too late if your load
|
||||
* balancer's health check has a polling interval, because every request already in
|
||||
* flight to this instance in that interval gets a connection reset. Publishing
|
||||
* {@link ReadinessState#REFUSING_TRAFFIC} flips {@code /actuator/health}'s readiness
|
||||
* group before the process is touched at all, giving the health-checked pool one or
|
||||
* two poll cycles to route around this instance first. This is the same event
|
||||
* Kubernetes-style {@code preStop} hooks publish - see
|
||||
* docs/13-graceful-shutdown-vs-kill-9.md, and {@code scripts/stop-instance.sh},
|
||||
* which calls this and sleeps before sending SIGTERM.
|
||||
*/
|
||||
@RestController
|
||||
public class DrainController {
|
||||
|
||||
private final ApplicationEventPublisher events;
|
||||
|
||||
public DrainController(ApplicationEventPublisher events) {
|
||||
this.events = events;
|
||||
}
|
||||
|
||||
@PostMapping("/admin/drain")
|
||||
public String drain() {
|
||||
AvailabilityChangeEvent.publish(events, this, ReadinessState.REFUSING_TRAFFIC);
|
||||
return "draining";
|
||||
}
|
||||
}
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
package com.ankurm.expandcontract.diag;
|
||||
|
||||
import com.ankurm.expandcontract.customer.CustomerService;
|
||||
import org.springframework.jdbc.core.simple.JdbcClient;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Prints the live state that the rest of this article can only describe: exactly
|
||||
* which columns "customers" has right now, and how many rows have each column
|
||||
* populated. This is what makes it possible to watch the schema actually expand and
|
||||
* then actually contract, rather than take the article's word for it. Delete this
|
||||
* before shipping to a real production service - see docs/15-production-checklist.md.
|
||||
*/
|
||||
@RestController
|
||||
public class SchemaDiagnosticsController {
|
||||
|
||||
private final JdbcClient jdbc;
|
||||
private final CustomerService customerService;
|
||||
|
||||
public SchemaDiagnosticsController(JdbcClient jdbc, CustomerService customerService) {
|
||||
this.jdbc = jdbc;
|
||||
this.customerService = customerService;
|
||||
}
|
||||
|
||||
@GetMapping("/diag/schema")
|
||||
public Map<String, Object> schema() {
|
||||
List<String> columns = columnsOf();
|
||||
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
result.put("appStage", customerService.stage());
|
||||
result.put("columns", columns);
|
||||
result.put("rowCount", jdbc.sql("SELECT COUNT(*) FROM customers").query(Long.class).single());
|
||||
|
||||
if (columns.contains("EMAIL")) {
|
||||
result.put("rowsWithEmail",
|
||||
jdbc.sql("SELECT COUNT(*) FROM customers WHERE email IS NOT NULL").query(Long.class).single());
|
||||
}
|
||||
if (columns.contains("EMAIL_ADDRESS")) {
|
||||
result.put("rowsWithEmailAddress",
|
||||
jdbc.sql("SELECT COUNT(*) FROM customers WHERE email_address IS NOT NULL").query(Long.class).single());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private List<String> columnsOf() {
|
||||
return jdbc.sql("SELECT column_name FROM information_schema.columns "
|
||||
+ "WHERE table_name = 'CUSTOMERS' ORDER BY ordinal_position")
|
||||
.query(String.class)
|
||||
.list();
|
||||
}
|
||||
}
|
||||
+374
@@ -0,0 +1,374 @@
|
||||
package com.ankurm.expandcontract.loadgen;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import tools.jackson.databind.ObjectMapper;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Deque;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
/**
|
||||
* A load generator that plays the part of a load balancer's client traffic during a
|
||||
* rolling deploy. It never talks to an instance its own health check just marked
|
||||
* unhealthy - exactly what a real load balancer's target group does - so the "zero
|
||||
* errors" claim in this article is a claim about client-observed traffic through a
|
||||
* health-checked pool of replicas, not a claim that no individual instance ever goes
|
||||
* down. Individual instances go down constantly; that is what a rolling deploy is.
|
||||
* <p>
|
||||
* Every request is tagged with whatever deploy phase {@code scripts/run-all.sh} has
|
||||
* currently written to the phase file, so the final report breaks errors down by
|
||||
* phase. See docs/11-the-load-generator.md for the workload mix and the consistency
|
||||
* check it performs on every read.
|
||||
* <p>
|
||||
* Each worker thread keeps its OWN pool of customer ids it created - never shared
|
||||
* with the other threads. An earlier version shared one pool across all threads and
|
||||
* produced "read-consistency-mismatch" errors that had nothing to do with the
|
||||
* server: two threads racing to update the same shared id could leave the pool
|
||||
* holding a stale expected value. Giving each thread exclusive ownership of the rows
|
||||
* it creates removes that class of bug entirely, while still hammering both server
|
||||
* replicas concurrently from multiple independent threads - see
|
||||
* docs/11-the-load-generator.md.
|
||||
*/
|
||||
public final class LoadGenerator {
|
||||
|
||||
private static final ObjectMapper MAPPER = new ObjectMapper();
|
||||
|
||||
record KnownCustomer(long id, String email) {
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
List<Integer> ports = List.of();
|
||||
int durationSeconds = 120;
|
||||
Path phaseFile = Path.of("/tmp/ec-demo/phase.txt");
|
||||
Path outFile = Path.of("/tmp/ec-demo/load-summary.txt");
|
||||
int threads = 8;
|
||||
|
||||
for (String arg : args) {
|
||||
if (arg.startsWith("--ports=")) {
|
||||
ports = List.of(arg.substring("--ports=".length()).split(",")).stream().map(Integer::parseInt).toList();
|
||||
} else if (arg.startsWith("--durationSeconds=")) {
|
||||
durationSeconds = Integer.parseInt(arg.substring("--durationSeconds=".length()));
|
||||
} else if (arg.startsWith("--phaseFile=")) {
|
||||
phaseFile = Path.of(arg.substring("--phaseFile=".length()));
|
||||
} else if (arg.startsWith("--outFile=")) {
|
||||
outFile = Path.of(arg.substring("--outFile=".length()));
|
||||
} else if (arg.startsWith("--threads=")) {
|
||||
threads = Integer.parseInt(arg.substring("--threads=".length()));
|
||||
}
|
||||
}
|
||||
if (ports.isEmpty()) {
|
||||
throw new IllegalArgumentException("--ports=8081,8082 is required");
|
||||
}
|
||||
|
||||
new LoadGenerator(ports, durationSeconds, phaseFile, outFile, threads).run();
|
||||
}
|
||||
|
||||
private final List<Integer> ports;
|
||||
private final int durationSeconds;
|
||||
private final Path phaseFile;
|
||||
private final Path outFile;
|
||||
private final int threads;
|
||||
|
||||
private final HttpClient http = HttpClient.newBuilder().connectTimeout(Duration.ofMillis(600)).build();
|
||||
private final Map<Integer, Boolean> healthy = new ConcurrentHashMap<>();
|
||||
private final Map<Integer, Integer> consecutiveFailures = new ConcurrentHashMap<>();
|
||||
private final Map<String, Counters> byPhase = new ConcurrentHashMap<>();
|
||||
private final AtomicBoolean stop = new AtomicBoolean(false);
|
||||
|
||||
// A backend needs two consecutive failed checks before it is removed from
|
||||
// rotation - one slow response under transient CPU contention (a sibling
|
||||
// replica's JVM starting up on this box's limited cores) should not look like
|
||||
// an outage. This mirrors how a real load balancer's health check threshold
|
||||
// works, and removing it is what turned transient slowness into false
|
||||
// "no-healthy-backend" errors during earlier runs - see docs/11-the-load-generator.md.
|
||||
private static final int UNHEALTHY_THRESHOLD = 2;
|
||||
|
||||
private static final class Counters {
|
||||
final AtomicLong ok = new AtomicLong();
|
||||
final AtomicLong error = new AtomicLong();
|
||||
final Map<String, AtomicLong> errorReasons = new ConcurrentHashMap<>();
|
||||
|
||||
void recordError(String reason) {
|
||||
error.incrementAndGet();
|
||||
errorReasons.computeIfAbsent(reason, r -> new AtomicLong()).incrementAndGet();
|
||||
}
|
||||
}
|
||||
|
||||
LoadGenerator(List<Integer> ports, int durationSeconds, Path phaseFile, Path outFile, int threads) {
|
||||
this.ports = ports;
|
||||
this.durationSeconds = durationSeconds;
|
||||
this.phaseFile = phaseFile;
|
||||
this.outFile = outFile;
|
||||
this.threads = threads;
|
||||
ports.forEach(p -> {
|
||||
healthy.put(p, false);
|
||||
consecutiveFailures.put(p, 0);
|
||||
});
|
||||
}
|
||||
|
||||
void run() throws Exception {
|
||||
if (outFile.getParent() != null) {
|
||||
Files.createDirectories(outFile.getParent());
|
||||
}
|
||||
|
||||
ScheduledExecutorService healthChecker = Executors.newSingleThreadScheduledExecutor();
|
||||
healthChecker.scheduleAtFixedRate(this::checkHealth, 0, 300, TimeUnit.MILLISECONDS);
|
||||
|
||||
// Wait for the first health check pass to land before starting any worker.
|
||||
// Without this, every worker's opening requests race the very first check -
|
||||
// the "healthy" map starts all-false by construction - and each one counts as
|
||||
// a spurious "no-healthy-backend" for a server that was up the whole time.
|
||||
for (int i = 0; i < 100 && ports.stream().noneMatch(p -> Boolean.TRUE.equals(healthy.get(p))); i++) {
|
||||
Thread.sleep(50);
|
||||
}
|
||||
|
||||
var workers = Executors.newFixedThreadPool(threads);
|
||||
CountDownLatch done = new CountDownLatch(threads);
|
||||
for (int i = 0; i < threads; i++) {
|
||||
workers.submit(() -> {
|
||||
try {
|
||||
workerLoop(new ArrayDeque<>());
|
||||
} finally {
|
||||
done.countDown();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Instant deadline = Instant.now().plusSeconds(durationSeconds);
|
||||
Instant lastLog = Instant.now();
|
||||
while (Instant.now().isBefore(deadline)) {
|
||||
Thread.sleep(1000);
|
||||
if (Duration.between(lastLog, Instant.now()).getSeconds() >= 10) {
|
||||
logProgress();
|
||||
lastLog = Instant.now();
|
||||
}
|
||||
}
|
||||
stop.set(true);
|
||||
done.await(10, TimeUnit.SECONDS);
|
||||
workers.shutdownNow();
|
||||
healthChecker.shutdownNow();
|
||||
|
||||
writeSummary();
|
||||
}
|
||||
|
||||
private void checkHealth() {
|
||||
for (int port : ports) {
|
||||
boolean up;
|
||||
try {
|
||||
HttpRequest req = HttpRequest.newBuilder(URI.create("http://localhost:" + port + "/actuator/health"))
|
||||
.timeout(Duration.ofMillis(800)).GET().build();
|
||||
HttpResponse<String> resp = http.send(req, HttpResponse.BodyHandlers.ofString());
|
||||
up = resp.statusCode() == 200 && resp.body().contains("\"UP\"");
|
||||
} catch (Exception e) {
|
||||
up = false;
|
||||
}
|
||||
if (up) {
|
||||
consecutiveFailures.put(port, 0);
|
||||
healthy.put(port, true);
|
||||
} else {
|
||||
int failures = consecutiveFailures.merge(port, 1, Integer::sum);
|
||||
if (failures >= UNHEALTHY_THRESHOLD) {
|
||||
healthy.put(port, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private int pickHealthyPort() {
|
||||
List<Integer> up = ports.stream().filter(p -> Boolean.TRUE.equals(healthy.get(p))).toList();
|
||||
if (up.isEmpty()) {
|
||||
return -1;
|
||||
}
|
||||
return up.get(ThreadLocalRandom.current().nextInt(up.size()));
|
||||
}
|
||||
|
||||
private String currentPhase() {
|
||||
try {
|
||||
if (Files.exists(phaseFile)) {
|
||||
String s = Files.readString(phaseFile).trim();
|
||||
if (!s.isEmpty()) {
|
||||
return s;
|
||||
}
|
||||
}
|
||||
} catch (IOException ignored) {
|
||||
}
|
||||
return "unphased";
|
||||
}
|
||||
|
||||
private void workerLoop(Deque<KnownCustomer> myKnown) {
|
||||
while (!stop.get()) {
|
||||
String phase = currentPhase();
|
||||
Counters c = byPhase.computeIfAbsent(phase, p -> new Counters());
|
||||
int port = pickHealthyPort();
|
||||
if (port == -1) {
|
||||
c.recordError("no-healthy-backend");
|
||||
sleepJitter();
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
double roll = ThreadLocalRandom.current().nextDouble();
|
||||
if (roll < 0.5 || myKnown.isEmpty()) {
|
||||
doCreate(port, c, myKnown);
|
||||
} else if (roll < 0.8) {
|
||||
doRead(port, c, myKnown);
|
||||
} else {
|
||||
doUpdate(port, c, myKnown);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
c.recordError(e.getClass().getSimpleName());
|
||||
}
|
||||
sleepJitter();
|
||||
}
|
||||
}
|
||||
|
||||
private void sleepJitter() {
|
||||
try {
|
||||
Thread.sleep(ThreadLocalRandom.current().nextInt(15, 40));
|
||||
} catch (InterruptedException ignored) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
private void doCreate(int port, Counters c, Deque<KnownCustomer> myKnown) throws Exception {
|
||||
long n = ThreadLocalRandom.current().nextLong(1_000_000_000L);
|
||||
String email = "customer" + n + "@example.test";
|
||||
String name = "Customer " + n;
|
||||
String body = MAPPER.writeValueAsString(Map.of("name", name, "email", email));
|
||||
HttpRequest req = HttpRequest.newBuilder(URI.create("http://localhost:" + port + "/customers"))
|
||||
.timeout(Duration.ofSeconds(2))
|
||||
.header("Content-Type", "application/json")
|
||||
.POST(HttpRequest.BodyPublishers.ofString(body))
|
||||
.build();
|
||||
HttpResponse<String> resp = http.send(req, HttpResponse.BodyHandlers.ofString());
|
||||
if (resp.statusCode() != 201) {
|
||||
c.recordError("create-http-" + resp.statusCode());
|
||||
return;
|
||||
}
|
||||
JsonNode node = MAPPER.readTree(resp.body());
|
||||
long id = node.get("id").asLong();
|
||||
String returnedEmail = node.get("email").asString();
|
||||
if (!email.equals(returnedEmail)) {
|
||||
c.recordError("create-echo-mismatch");
|
||||
return;
|
||||
}
|
||||
myKnown.addLast(new KnownCustomer(id, email));
|
||||
while (myKnown.size() > 500) {
|
||||
myKnown.pollFirst();
|
||||
}
|
||||
c.ok.incrementAndGet();
|
||||
}
|
||||
|
||||
private void doRead(int port, Counters c, Deque<KnownCustomer> myKnown) throws Exception {
|
||||
KnownCustomer target = pickKnown(myKnown);
|
||||
if (target == null) {
|
||||
return;
|
||||
}
|
||||
HttpRequest req = HttpRequest.newBuilder(URI.create("http://localhost:" + port + "/customers/" + target.id()))
|
||||
.timeout(Duration.ofSeconds(2)).GET().build();
|
||||
HttpResponse<String> resp = http.send(req, HttpResponse.BodyHandlers.ofString());
|
||||
if (resp.statusCode() != 200) {
|
||||
c.recordError("read-http-" + resp.statusCode());
|
||||
return;
|
||||
}
|
||||
JsonNode node = MAPPER.readTree(resp.body());
|
||||
String returnedEmail = node.get("email").asString();
|
||||
if (!target.email().equals(returnedEmail)) {
|
||||
c.recordError("read-consistency-mismatch");
|
||||
return;
|
||||
}
|
||||
c.ok.incrementAndGet();
|
||||
}
|
||||
|
||||
private void doUpdate(int port, Counters c, Deque<KnownCustomer> myKnown) throws Exception {
|
||||
KnownCustomer target = pickKnown(myKnown);
|
||||
if (target == null) {
|
||||
return;
|
||||
}
|
||||
long n = ThreadLocalRandom.current().nextLong(1_000_000_000L);
|
||||
String newEmail = "updated" + n + "@example.test";
|
||||
String body = MAPPER.writeValueAsString(Map.of("email", newEmail));
|
||||
HttpRequest req = HttpRequest.newBuilder(URI.create("http://localhost:" + port + "/customers/" + target.id() + "/email"))
|
||||
.timeout(Duration.ofSeconds(2))
|
||||
.header("Content-Type", "application/json")
|
||||
.PUT(HttpRequest.BodyPublishers.ofString(body))
|
||||
.build();
|
||||
HttpResponse<String> resp = http.send(req, HttpResponse.BodyHandlers.ofString());
|
||||
if (resp.statusCode() != 200) {
|
||||
c.recordError("update-http-" + resp.statusCode());
|
||||
return;
|
||||
}
|
||||
JsonNode node = MAPPER.readTree(resp.body());
|
||||
String returnedEmail = node.get("email").asString();
|
||||
if (!newEmail.equals(returnedEmail)) {
|
||||
c.recordError("update-echo-mismatch");
|
||||
return;
|
||||
}
|
||||
// This deque is private to the calling thread - no other thread ever reads
|
||||
// or writes this id, so there is no race to guard against here.
|
||||
myKnown.remove(target);
|
||||
myKnown.addLast(new KnownCustomer(target.id(), newEmail));
|
||||
c.ok.incrementAndGet();
|
||||
}
|
||||
|
||||
private KnownCustomer pickKnown(Deque<KnownCustomer> myKnown) {
|
||||
int size = myKnown.size();
|
||||
if (size == 0) {
|
||||
return null;
|
||||
}
|
||||
int skip = ThreadLocalRandom.current().nextInt(size);
|
||||
var it = myKnown.iterator();
|
||||
KnownCustomer last = null;
|
||||
for (int i = 0; i <= skip && it.hasNext(); i++) {
|
||||
last = it.next();
|
||||
}
|
||||
return last;
|
||||
}
|
||||
|
||||
private void logProgress() {
|
||||
long okTotal = byPhase.values().stream().mapToLong(c -> c.ok.get()).sum();
|
||||
long errTotal = byPhase.values().stream().mapToLong(c -> c.error.get()).sum();
|
||||
System.out.printf("[%s] phase=%-24s ok=%d error=%d%n", Instant.now(), currentPhase(), okTotal, errTotal);
|
||||
}
|
||||
|
||||
private void writeSummary() throws IOException {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("Load generator summary\n");
|
||||
sb.append("=======================\n");
|
||||
long okTotal = 0, errTotal = 0;
|
||||
for (var entry : byPhase.entrySet()) {
|
||||
okTotal += entry.getValue().ok.get();
|
||||
errTotal += entry.getValue().error.get();
|
||||
}
|
||||
sb.append(String.format("Total requests: %d%n", okTotal + errTotal));
|
||||
sb.append(String.format("Successful: %d%n", okTotal));
|
||||
sb.append(String.format("Errors: %d%n", errTotal));
|
||||
sb.append("\nBy phase:\n");
|
||||
for (var entry : byPhase.entrySet()) {
|
||||
Counters c = entry.getValue();
|
||||
sb.append(String.format(" %-28s ok=%-8d error=%-6d%n", entry.getKey(), c.ok.get(), c.error.get()));
|
||||
for (var reason : c.errorReasons.entrySet()) {
|
||||
sb.append(String.format(" - %-24s %d%n", reason.getKey(), reason.getValue().get()));
|
||||
}
|
||||
}
|
||||
Files.writeString(outFile, sb.toString());
|
||||
System.out.print(sb);
|
||||
}
|
||||
}
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package com.ankurm.expandcontract.migration;
|
||||
|
||||
import org.flywaydb.core.Flyway;
|
||||
import org.flywaydb.core.api.MigrationInfo;
|
||||
|
||||
/**
|
||||
* A migration runner that is deliberately NOT part of the Spring Boot application.
|
||||
* <p>
|
||||
* This is the mechanical heart of the article's argument: schema changes and code
|
||||
* deploys are two different kinds of event, so they get two different delivery
|
||||
* mechanisms. The app (see {@link com.ankurm.expandcontract.ExpandContractApplication})
|
||||
* never touches Flyway - {@code spring.flyway.enabled=false} in application.yml sees to
|
||||
* that. This class runs standalone, against the same JDBC URL, and takes a
|
||||
* {@code --target} version so a "deploy" can migrate exactly as far as that step of
|
||||
* the sequence requires and no further.
|
||||
* <p>
|
||||
* See docs/03-why-migrations-run-outside-the-app.md.
|
||||
*/
|
||||
public final class MigrationCli {
|
||||
|
||||
public static void main(String[] args) {
|
||||
String target = "latest";
|
||||
for (String arg : args) {
|
||||
if (arg.startsWith("--target=")) {
|
||||
target = arg.substring("--target=".length());
|
||||
}
|
||||
}
|
||||
|
||||
// Same standalone TCP server the app connects to (see application.yml) - migrations
|
||||
// run against the live database over the network, exactly like the app does, and
|
||||
// like a real migration job in a CI pipeline would.
|
||||
String tcpPort = System.getenv().getOrDefault("EC_DB_TCP_PORT", "9092");
|
||||
String dbName = System.getenv().getOrDefault("EC_DB_NAME", "expand-contract");
|
||||
String url = "jdbc:h2:tcp://localhost:" + tcpPort + "/" + dbName;
|
||||
|
||||
Flyway flyway = Flyway.configure()
|
||||
.dataSource(url, "sa", "")
|
||||
.locations("classpath:db/migration")
|
||||
.target(target)
|
||||
.load();
|
||||
|
||||
System.out.println("=== Before migrate (target=" + target + ") ===");
|
||||
printInfo(flyway);
|
||||
|
||||
var result = flyway.migrate();
|
||||
|
||||
System.out.println("=== After migrate ===");
|
||||
printInfo(flyway);
|
||||
System.out.println("Migrations executed: " + result.migrationsExecuted
|
||||
+ ", target schema version: " + result.targetSchemaVersion
|
||||
+ ", success: " + result.success);
|
||||
}
|
||||
|
||||
private static void printInfo(Flyway flyway) {
|
||||
for (MigrationInfo info : flyway.info().all()) {
|
||||
System.out.printf(" %-8s %-40s %-10s%n",
|
||||
info.getVersion() == null ? "-" : info.getVersion().getVersion(),
|
||||
info.getDescription(),
|
||||
info.getState());
|
||||
}
|
||||
}
|
||||
|
||||
private MigrationCli() {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
spring:
|
||||
application:
|
||||
name: expand-contract-demo
|
||||
datasource:
|
||||
# A real TCP connection to a standalone H2 server process (scripts/start-db-server.sh),
|
||||
# never a file this app opens itself. An earlier version used
|
||||
# "jdbc:h2:file:...;AUTO_SERVER=TRUE" so the two replicas could share one database file
|
||||
# directly - which works only as long as neither replica is ever restarted, because
|
||||
# AUTO_SERVER silently makes the FIRST process to open the file the de facto database
|
||||
# server for every other process that connects to it afterwards. Killing that one
|
||||
# replica during a routine rolling deploy took the "shared database" down with it - see
|
||||
# docs/12-the-auto-server-trap.md. A real production database is its own process for
|
||||
# exactly this reason.
|
||||
url: jdbc:h2:tcp://localhost:${EC_DB_TCP_PORT:9092}/${EC_DB_NAME:expand-contract}
|
||||
username: sa
|
||||
password: ""
|
||||
driver-class-name: org.h2.Driver
|
||||
flyway:
|
||||
# Migrations are NOT run by the application on startup. The whole point of
|
||||
# expand-contract is that schema changes and app deploys are independent
|
||||
# events - see migration.MigrationCli, driven by scripts/migrate.sh.
|
||||
enabled: false
|
||||
|
||||
# The stage this instance is running as: 1 (baseline), 2 (dual-write), 3 (read-new),
|
||||
# 4 (contract - new column only). Passed on the command line per instance, e.g.
|
||||
# --app.stage=2, so two replicas can run different stages during a rolling deploy.
|
||||
app:
|
||||
stage: ${APP_STAGE:1}
|
||||
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: health,info
|
||||
endpoint:
|
||||
health:
|
||||
show-details: always
|
||||
health:
|
||||
defaults:
|
||||
enabled: true
|
||||
|
||||
server:
|
||||
port: ${SERVER_PORT:8080}
|
||||
shutdown: graceful
|
||||
|
||||
logging:
|
||||
level:
|
||||
root: INFO
|
||||
@@ -0,0 +1,8 @@
|
||||
-- Deploy 0 (baseline, already in production before this article starts).
|
||||
-- A customers table with a single "email" column, the thing we are about to rename.
|
||||
CREATE TABLE customers (
|
||||
id BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
name VARCHAR(200) NOT NULL,
|
||||
email VARCHAR(320) NOT NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
-- Deploy 1 (EXPAND). Additive and nullable, so it is compatible with every piece
|
||||
-- of app code that is running right now: the old code never mentions this column
|
||||
-- and will not notice it exists. This migration runs against the live database
|
||||
-- with zero application deploy and zero restart of any replica.
|
||||
ALTER TABLE customers ADD COLUMN email_address VARCHAR(320);
|
||||
|
||||
-- Backfill every row that existed before dual-write code shipped. Rows created
|
||||
-- *after* this point but before the dual-write code (Deploy 2) is fully rolled
|
||||
-- out are handled separately - see docs/07-the-backfill-window-bug.md.
|
||||
UPDATE customers SET email_address = email WHERE email_address IS NULL;
|
||||
|
||||
-- The other half of "expand": relax the constraint on the column we are about to
|
||||
-- retire. "email" is NOT NULL from V1. Leave that in place and Deploy 4's Stage 4
|
||||
-- code - which never writes "email" - fails every single INSERT with a NOT NULL
|
||||
-- violation the moment it starts, because the column it ignores is still mandatory.
|
||||
-- Forgetting this line is a genuine, reproducible failure - see
|
||||
-- docs/06-the-not-null-trap.md, which captures the exact exception it produces.
|
||||
ALTER TABLE customers ALTER COLUMN email DROP NOT NULL;
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
-- Deploy 4 (CONTRACT). Only safe once every replica in the fleet is confirmed
|
||||
-- running Stage 4 code, which never reads or writes "email". Run this too early
|
||||
-- and any Stage 1/2/3 instance still in the rolling deploy fails on its next
|
||||
-- write - see docs/10-what-happens-if-you-drop-too-soon.md.
|
||||
ALTER TABLE customers DROP COLUMN email;
|
||||
Reference in New Issue
Block a user