Add kubernetes-deployment: probes, shutdown, JVM ergonomics, HPA
Companion code for "Deploying Spring Boot 4 on Kubernetes: Probes, Graceful Shutdown, Limits and JVM Ergonomics". A dependency outage under three probe-group setups, a rolling restart under load four ways (three runs each), the JVM's ergonomic choices for nine pod shapes, one GC-heavy load under five CPU limits with throttling counters, and an HPA driven by a Micrometer gauge through prometheus-adapter. Measured on k3s v1.36.4. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01C3TETMrqVUWeFkNtz3Jbo3
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,12 @@
|
||||
package com.ankurm.k8s;
|
||||
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
|
||||
@SpringBootApplication
|
||||
public class K8sApplication {
|
||||
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(K8sApplication.class, args);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package com.ankurm.k8s.diag;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.lang.management.GarbageCollectorMXBean;
|
||||
import java.lang.management.ManagementFactory;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.sun.management.HotSpotDiagnosticMXBean;
|
||||
import com.sun.management.OperatingSystemMXBean;
|
||||
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* What the JVM decided about the container it is in, and what the kernel did to it.
|
||||
*
|
||||
* <ul>
|
||||
* <li>{@code /diag/jvm} - the ergonomic choices (processors, collector, heap, thread counts) next
|
||||
* to the cgroup limits they were derived from.</li>
|
||||
* <li>{@code /diag/gc} - cumulative GC counts/time and the CFS throttling counters. Take it
|
||||
* before and after a load run and subtract (scripts/demo-gc-throttling.sh does).</li>
|
||||
* </ul>
|
||||
* Reads both cgroup v1 and v2 layouts. Delete before shipping anything real.
|
||||
*/
|
||||
@RestController
|
||||
public class JvmController {
|
||||
|
||||
private static final List<String> FLAGS = List.of("UseSerialGC", "UseParallelGC", "UseG1GC", "UseZGC",
|
||||
"ActiveProcessorCount", "ParallelGCThreads", "ConcGCThreads", "CICompilerCount", "MaxHeapSize",
|
||||
"MaxRAMPercentage", "InitialHeapSize");
|
||||
|
||||
@GetMapping("/diag/jvm")
|
||||
public Map<String, Object> jvm() {
|
||||
Map<String, Object> out = new LinkedHashMap<>();
|
||||
Runtime rt = Runtime.getRuntime();
|
||||
out.put("availableProcessors", rt.availableProcessors());
|
||||
out.put("maxHeapMiB", rt.maxMemory() / (1024 * 1024));
|
||||
out.put("collectors", ManagementFactory.getGarbageCollectorMXBeans().stream()
|
||||
.map(GarbageCollectorMXBean::getName).toList());
|
||||
HotSpotDiagnosticMXBean hs = ManagementFactory.getPlatformMXBean(HotSpotDiagnosticMXBean.class);
|
||||
Map<String, String> flags = new LinkedHashMap<>();
|
||||
for (String flag : FLAGS) {
|
||||
var option = hs.getVMOption(flag);
|
||||
flags.put(flag, option.getValue() + (option.getOrigin().name().equals("DEFAULT") ? "" : " (" + option.getOrigin() + ")"));
|
||||
}
|
||||
out.put("flags", flags);
|
||||
OperatingSystemMXBean os = ManagementFactory.getPlatformMXBean(OperatingSystemMXBean.class);
|
||||
out.put("osTotalMemoryMiB (container-aware)", os.getTotalMemorySize() / (1024 * 1024));
|
||||
out.put("cgroup", cgroup());
|
||||
return out;
|
||||
}
|
||||
|
||||
@GetMapping("/diag/gc")
|
||||
public Map<String, Object> gc() {
|
||||
Map<String, Object> out = new LinkedHashMap<>();
|
||||
for (GarbageCollectorMXBean gc : ManagementFactory.getGarbageCollectorMXBeans()) {
|
||||
out.put(gc.getName(), Map.of("count", gc.getCollectionCount(), "timeMs", gc.getCollectionTime()));
|
||||
}
|
||||
out.put("cpuStat", cpuStat());
|
||||
OperatingSystemMXBean os = ManagementFactory.getPlatformMXBean(OperatingSystemMXBean.class);
|
||||
out.put("processCpuTimeMs", os.getProcessCpuTime() / 1_000_000);
|
||||
return out;
|
||||
}
|
||||
|
||||
private static Map<String, Object> cgroup() {
|
||||
Map<String, Object> out = new LinkedHashMap<>();
|
||||
if (Files.exists(Path.of("/sys/fs/cgroup/cpu.max"))) {
|
||||
out.put("version", 2);
|
||||
out.put("cpu.max", read("/sys/fs/cgroup/cpu.max"));
|
||||
out.put("memory.max", read("/sys/fs/cgroup/memory.max"));
|
||||
}
|
||||
else {
|
||||
out.put("version", 1);
|
||||
out.put("cpu.cfs_quota_us", read("/sys/fs/cgroup/cpu/cpu.cfs_quota_us"));
|
||||
out.put("cpu.cfs_period_us", read("/sys/fs/cgroup/cpu/cpu.cfs_period_us"));
|
||||
out.put("cpu.shares", read("/sys/fs/cgroup/cpu/cpu.shares"));
|
||||
out.put("memory.limit_in_bytes", read("/sys/fs/cgroup/memory/memory.limit_in_bytes"));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private static Map<String, Long> cpuStat() {
|
||||
Path v2 = Path.of("/sys/fs/cgroup/cpu.stat");
|
||||
Path v1 = Path.of("/sys/fs/cgroup/cpu/cpu.stat");
|
||||
Path file = Files.exists(Path.of("/sys/fs/cgroup/cpu.max")) ? v2 : v1;
|
||||
Map<String, Long> out = new LinkedHashMap<>();
|
||||
try {
|
||||
for (String line : Files.readAllLines(file)) {
|
||||
String[] kv = line.trim().split("\\s+");
|
||||
if (kv.length == 2) {
|
||||
out.put(kv[0], Long.parseLong(kv[1]));
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (IOException | NumberFormatException ex) {
|
||||
out.put("unreadable", -1L);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private static String read(String file) {
|
||||
try {
|
||||
return Files.readString(Path.of(file)).trim();
|
||||
}
|
||||
catch (IOException ex) {
|
||||
return "n/a";
|
||||
}
|
||||
}
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
package com.ankurm.k8s.health;
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.time.Duration;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.boot.health.contributor.Health;
|
||||
import org.springframework.boot.health.contributor.HealthIndicator;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* Reports whether a downstream HTTP service answers. Which probe it belongs to is decided by
|
||||
* configuration, not code - and putting it in the liveness group is the mistake that turns one
|
||||
* dependency outage into a restart storm across every replica (docs/02-probes.md).
|
||||
*/
|
||||
@Component("downstream")
|
||||
public class DownstreamHealthIndicator implements HealthIndicator {
|
||||
|
||||
private final HttpClient client = HttpClient.newBuilder().connectTimeout(Duration.ofMillis(500)).build();
|
||||
private final URI url;
|
||||
|
||||
public DownstreamHealthIndicator(@Value("${demo.downstream-url:}") String url) {
|
||||
this.url = url.isBlank() ? null : URI.create(url);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Health health() {
|
||||
if (url == null) {
|
||||
return Health.up().withDetail("downstream", "not configured").build();
|
||||
}
|
||||
try {
|
||||
HttpResponse<Void> response = client.send(
|
||||
HttpRequest.newBuilder(url).timeout(Duration.ofMillis(800)).GET().build(),
|
||||
HttpResponse.BodyHandlers.discarding());
|
||||
return response.statusCode() < 500
|
||||
? Health.up().withDetail("status", response.statusCode()).build()
|
||||
: Health.down().withDetail("status", response.statusCode()).build();
|
||||
}
|
||||
catch (Exception ex) {
|
||||
return Health.down().withDetail("error", ex.getClass().getSimpleName()).build();
|
||||
}
|
||||
finally {
|
||||
if (Thread.interrupted()) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package com.ankurm.k8s.loadgen;
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
/**
|
||||
* A closed-loop load generator that runs inside the cluster, from the same image:
|
||||
* {@code java -cp application.jar com.ankurm.k8s.loadgen.LoadGen <url> <concurrency> <seconds>}.
|
||||
*
|
||||
* <p>Prints one line per second - successes, and failures grouped by cause - then a summary with
|
||||
* latency percentiles. A "failure" is anything that is not HTTP 200: a refused or reset
|
||||
* connection, a 5xx, a timeout. That is exactly what a rolling update is not supposed to produce.
|
||||
* Concurrency can be changed mid-run by giving a schedule instead: {@code 0:5,60:40,180:0}
|
||||
* (seconds:concurrency).
|
||||
*
|
||||
* <p>{@code -Dloadgen.method=POST} sends POST instead of GET. That matters: the JDK HttpClient
|
||||
* transparently retries an idempotent GET whose connection was closed under it, so a GET load test
|
||||
* under-counts requests the server dropped. A POST is never retried - what fails is what you see.
|
||||
*/
|
||||
public final class LoadGen {
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
URI uri = URI.create(args[0]);
|
||||
String method = System.getProperty("loadgen.method", "GET");
|
||||
String schedule = args[1];
|
||||
int seconds = Integer.parseInt(args[2]);
|
||||
HttpClient client = HttpClient.newBuilder()
|
||||
.version(HttpClient.Version.HTTP_1_1)
|
||||
.connectTimeout(Duration.ofSeconds(2))
|
||||
.build();
|
||||
|
||||
Map<Long, Map<String, AtomicInteger>> perSecond = new ConcurrentHashMap<>();
|
||||
List<Long> latencies = Collections.synchronizedList(new ArrayList<>());
|
||||
AtomicInteger target = new AtomicInteger();
|
||||
AtomicBoolean running = new AtomicBoolean(true);
|
||||
long start = System.nanoTime();
|
||||
TreeMap<Integer, Integer> plan = parse(schedule);
|
||||
|
||||
try (var executor = Executors.newVirtualThreadPerTaskExecutor()) {
|
||||
AtomicInteger workers = new AtomicInteger();
|
||||
// One controller thread adjusts the number of workers to the schedule.
|
||||
while (running.get()) {
|
||||
long elapsed = (System.nanoTime() - start) / 1_000_000_000;
|
||||
if (elapsed >= seconds) {
|
||||
running.set(false);
|
||||
break;
|
||||
}
|
||||
target.set(plan.floorEntry((int) elapsed).getValue());
|
||||
while (workers.get() < target.get()) {
|
||||
int id = workers.incrementAndGet();
|
||||
executor.submit(() -> {
|
||||
while (running.get() && id <= target.get()) {
|
||||
long t0 = System.nanoTime();
|
||||
String outcome;
|
||||
try {
|
||||
HttpResponse<Void> r = client.send(HttpRequest.newBuilder(uri)
|
||||
.timeout(Duration.ofSeconds(10))
|
||||
.method(method, HttpRequest.BodyPublishers.noBody()).build(),
|
||||
HttpResponse.BodyHandlers.discarding());
|
||||
outcome = r.statusCode() == 200 ? "ok" : "http_" + r.statusCode();
|
||||
}
|
||||
catch (Exception ex) {
|
||||
outcome = ex.getClass().getSimpleName();
|
||||
}
|
||||
long t1 = System.nanoTime();
|
||||
long second = (t1 - start) / 1_000_000_000;
|
||||
perSecond.computeIfAbsent(second, k -> new ConcurrentHashMap<>())
|
||||
.computeIfAbsent(outcome, k -> new AtomicInteger()).incrementAndGet();
|
||||
if (outcome.equals("ok")) {
|
||||
latencies.add((t1 - t0) / 1_000_000);
|
||||
}
|
||||
}
|
||||
workers.decrementAndGet();
|
||||
return null;
|
||||
});
|
||||
}
|
||||
Thread.sleep(200);
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, Integer> totals = new TreeMap<>();
|
||||
for (long s = 0; s <= seconds; s++) {
|
||||
Map<String, AtomicInteger> row = perSecond.getOrDefault(s, Map.of());
|
||||
StringBuilder line = new StringBuilder(String.format("t=%3ds conc=%-3d", s, plan.floorEntry((int) Math.min(s, seconds - 1)).getValue()));
|
||||
new TreeMap<>(row).forEach((k, v) -> {
|
||||
line.append(' ').append(k).append('=').append(v.get());
|
||||
totals.merge(k, v.get(), Integer::sum);
|
||||
});
|
||||
System.out.println(line);
|
||||
}
|
||||
List<Long> sorted = new ArrayList<>(latencies);
|
||||
Collections.sort(sorted);
|
||||
System.out.println("METHOD " + method);
|
||||
System.out.println("TOTAL " + totals);
|
||||
if (!sorted.isEmpty()) {
|
||||
System.out.printf("LATENCY ms p50=%d p90=%d p99=%d max=%d (n=%d)%n", pct(sorted, 50), pct(sorted, 90),
|
||||
pct(sorted, 99), sorted.get(sorted.size() - 1), sorted.size());
|
||||
}
|
||||
}
|
||||
|
||||
private static TreeMap<Integer, Integer> parse(String schedule) {
|
||||
TreeMap<Integer, Integer> plan = new TreeMap<>();
|
||||
if (!schedule.contains(":")) {
|
||||
plan.put(0, Integer.parseInt(schedule));
|
||||
return plan;
|
||||
}
|
||||
for (String step : schedule.split(",")) {
|
||||
String[] p = step.split(":");
|
||||
plan.put(Integer.parseInt(p[0]), Integer.parseInt(p[1]));
|
||||
}
|
||||
return plan;
|
||||
}
|
||||
|
||||
private static long pct(List<Long> sorted, int p) {
|
||||
return sorted.get(Math.min(sorted.size() - 1, (int) Math.ceil(p / 100.0 * sorted.size()) - 1));
|
||||
}
|
||||
|
||||
private LoadGen() {
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package com.ankurm.k8s.web;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/** Simulates an application that needs {@code demo.startup-delay} to become ready (cache warm-up, migrations). */
|
||||
@Component
|
||||
public class SlowStartup {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(SlowStartup.class);
|
||||
|
||||
private final long seconds;
|
||||
|
||||
public SlowStartup(@Value("${demo.startup-delay:0s}") java.time.Duration delay) {
|
||||
this.seconds = delay.toSeconds();
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
void warmUp() throws InterruptedException {
|
||||
if (seconds > 0) {
|
||||
log.info("Warming up for {} s", seconds);
|
||||
Thread.sleep(seconds * 1000);
|
||||
log.info("Warm-up complete");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package com.ankurm.k8s.web;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import io.micrometer.core.instrument.Gauge;
|
||||
import io.micrometer.core.instrument.MeterRegistry;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMethod;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* The two kinds of work the Kubernetes experiments need.
|
||||
*
|
||||
* <ul>
|
||||
* <li>{@code /work?ms=300} - a request that takes a known time. In-flight requests are exported
|
||||
* as the gauge {@code app.inflight.requests}, which Prometheus sees as
|
||||
* {@code app_inflight_requests} and the HorizontalPodAutoscaler scales on
|
||||
* (docs/06-hpa-custom-metrics.md).</li>
|
||||
* <li>{@code /alloc?mb=64} - allocation-heavy work that keeps the garbage collector busy, for the
|
||||
* CPU-limit experiments (docs/04-cpu-limits-and-gc.md).</li>
|
||||
* </ul>
|
||||
*/
|
||||
@RestController
|
||||
public class WorkController {
|
||||
|
||||
private final AtomicInteger inFlight = new AtomicInteger();
|
||||
private final String pod;
|
||||
private final String revision;
|
||||
/** A slowly churning old generation, so collections have live data to trace, not just garbage. */
|
||||
private final List<byte[]> retained = new ArrayList<>();
|
||||
|
||||
public WorkController(MeterRegistry registry, @Value("${HOSTNAME:local}") String pod,
|
||||
@Value("${demo.revision:1}") String revision) {
|
||||
this.pod = pod;
|
||||
this.revision = revision;
|
||||
Gauge.builder("app.inflight.requests", inFlight, AtomicInteger::get)
|
||||
.description("Requests currently being processed by /work")
|
||||
.register(registry);
|
||||
}
|
||||
|
||||
/** GET and POST: POST is what the shutdown experiment uses, because HTTP clients retry GET. */
|
||||
@RequestMapping(path = "/work", method = {RequestMethod.GET, RequestMethod.POST})
|
||||
public Map<String, Object> work(@RequestParam(defaultValue = "100") long ms) throws InterruptedException {
|
||||
inFlight.incrementAndGet();
|
||||
try {
|
||||
Thread.sleep(ms);
|
||||
return Map.of("pod", pod, "revision", revision, "ms", ms);
|
||||
}
|
||||
finally {
|
||||
inFlight.decrementAndGet();
|
||||
}
|
||||
}
|
||||
|
||||
@GetMapping("/alloc")
|
||||
public Map<String, Object> alloc(@RequestParam(defaultValue = "64") int mb) {
|
||||
long start = System.nanoTime();
|
||||
long checksum = 0;
|
||||
ThreadLocalRandom random = ThreadLocalRandom.current();
|
||||
for (int i = 0; i < mb * 16; i++) { // 64 KiB chunks
|
||||
byte[] chunk = new byte[64 * 1024];
|
||||
chunk[random.nextInt(chunk.length)] = 1;
|
||||
checksum += chunk[0];
|
||||
if (random.nextInt(64) == 0) {
|
||||
synchronized (retained) {
|
||||
retained.add(chunk);
|
||||
if (retained.size() > 400) { // ~25 MiB of long-lived data, replaced over time
|
||||
retained.remove(random.nextInt(retained.size()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return Map.of("pod", pod, "mb", mb, "ms", (System.nanoTime() - start) / 1_000_000, "checksum", checksum);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
spring:
|
||||
application:
|
||||
name: orders
|
||||
threads:
|
||||
virtual:
|
||||
enabled: true
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: health,info,prometheus
|
||||
endpoint:
|
||||
health:
|
||||
probes:
|
||||
# Auto-enabled when Spring Boot detects Kubernetes (the *_SERVICE_HOST/_PORT variables);
|
||||
# set explicitly so the same groups exist when you run the jar on a laptop.
|
||||
enabled: true
|
||||
show-details: always
|
||||
metrics:
|
||||
tags:
|
||||
application: ${spring.application.name}
|
||||
@@ -0,0 +1,31 @@
|
||||
package com.ankurm.k8s;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.test.web.server.LocalServerPort;
|
||||
import org.springframework.web.client.RestClient;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* The probe endpoints and the custom gauge exist with the names the Kubernetes manifests and the
|
||||
* Prometheus adapter rule depend on. If a Spring Boot upgrade renames any of them, this fails
|
||||
* before a rollout does.
|
||||
*/
|
||||
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
class ProbesContractTest {
|
||||
|
||||
@LocalServerPort
|
||||
int port;
|
||||
|
||||
@Test
|
||||
void probeGroupsAndGaugeAreExposed() {
|
||||
RestClient client = RestClient.create("http://localhost:" + port);
|
||||
assertThat(client.get().uri("/actuator/health/liveness").retrieve().body(String.class)).contains("\"UP\"");
|
||||
assertThat(client.get().uri("/actuator/health/readiness").retrieve().body(String.class)).contains("\"UP\"");
|
||||
assertThat(client.get().uri("/actuator/prometheus").retrieve().body(String.class))
|
||||
.contains("app_inflight_requests{application=\"orders\"}");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user