Spring Boot startup time: bean-by-bean diagnosis, and one directory per post

Adds spring-boot-startup-time/, the companion project for BLOG-618: a runnable Spring Boot 4.1.1 application on JDK 25 that installs BufferingApplicationStartup and FlightRecorderApplicationStartup behind a system property, and a /diag/startup endpoint that computes step self time -- the number /actuator/startup does not give you and the one that names the actual culprits. Captured under docs/output/: the step tree sorted both ways, the same startup as JFR events, a +5000-class experiment putting 0.11 ms per scanned class on the classpath scan tax, the silent truncation a 2048-step buffer performs, and JDK 25 AOT cache timings (6.93 s to 4.82 s). Post body and metadata live in post/. Moves the existing Actuator project into actuator-in-production/ so the repository holds one directory per article; the root README is now an index.
2026-09-04 23:58:38 +05:30
parent 4b6cefa60a
commit 958b401f0f
112 changed files with 2744 additions and 154 deletions
--- a/actuator-in-production/src/main/java/com/ankurm/actuator/ActuatorProductionApplication.java
+++ b/actuator-in-production/src/main/java/com/ankurm/actuator/ActuatorProductionApplication.java
@@ -0,0 +1,11 @@
+package com.ankurm.actuator;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class ActuatorProductionApplication {
+    public static void main(String[] args) {
+        SpringApplication.run(ActuatorProductionApplication.class, args);
+    }
+}
--- a/actuator-in-production/src/main/java/com/ankurm/actuator/config/DemoSecurityConfig.java
+++ b/actuator-in-production/src/main/java/com/ankurm/actuator/config/DemoSecurityConfig.java
@@ -0,0 +1,37 @@
+package com.ankurm.actuator.config;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Profile;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.config.http.SessionCreationPolicy;
+import org.springframework.security.web.SecurityFilterChain;
+
+/**
+ * The baseline chain used by every profile that does not bring its own.
+ *
+ * <p>It exists for one reason worth knowing about: Spring Boot's auto-configured security is
+ * browser-shaped. It turns CSRF protection on, which means an authenticated {@code POST} from
+ * curl is rejected before it reaches your controller. While building this repository that
+ * showed up as a {@code 401} on {@code POST /stub/upstream/mode} while {@code GET /orders/count}
+ * with the same credentials returned {@code 200} &mdash; a confusing pair of results that has
+ * nothing to do with the credentials.
+ *
+ * <p>The same trap catches people calling {@code POST /actuator/loggers/{name}} from a script.
+ * Actuator is a machine-to-machine API; give it a stateless, CSRF-free chain.
+ *
+ * <p>See docs/04-securing-actuator.md.
+ */
+@Configuration
+@Profile("!open & !secured")
+public class DemoSecurityConfig {
+
+    @Bean
+    SecurityFilterChain defaultChain(HttpSecurity http) throws Exception {
+        http.authorizeHttpRequests((requests) -> requests.anyRequest().authenticated())
+                .httpBasic((basic) -> { })
+                .csrf((csrf) -> csrf.disable())
+                .sessionManagement((s) -> s.sessionCreationPolicy(SessionCreationPolicy.STATELESS));
+        return http.build();
+    }
+}
--- a/actuator-in-production/src/main/java/com/ankurm/actuator/config/OpenActuatorConfig.java
+++ b/actuator-in-production/src/main/java/com/ankurm/actuator/config/OpenActuatorConfig.java
@@ -0,0 +1,33 @@
+package com.ankurm.actuator.config;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Profile;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.web.SecurityFilterChain;
+
+/**
+ * The misconfiguration, kept in the repository on purpose so the damage can be measured rather
+ * than asserted.
+ *
+ * <p>This is not a strawman. It is what you get when somebody adds Spring Security, finds that
+ * the generated password broke their smoke tests, and reaches for the shortest fix that makes
+ * the tests pass again. Combined with
+ * {@code management.endpoints.web.exposure.include: "*"} it publishes {@code /actuator/env},
+ * {@code /actuator/heapdump} and {@code /actuator/shutdown} to anyone who can reach the port.
+ *
+ * <p>Run {@code ./scripts/demo-open-actuator.sh} to see exactly what leaks.
+ *
+ * <p>See docs/04-securing-actuator.md.
+ */
+@Configuration
+@Profile("open")
+public class OpenActuatorConfig {
+
+    @Bean
+    SecurityFilterChain permitEverything(HttpSecurity http) throws Exception {
+        http.authorizeHttpRequests((requests) -> requests.anyRequest().permitAll())
+                .csrf((csrf) -> csrf.disable());
+        return http.build();
+    }
+}
--- a/actuator-in-production/src/main/java/com/ankurm/actuator/config/SecuredActuatorConfig.java
+++ b/actuator-in-production/src/main/java/com/ankurm/actuator/config/SecuredActuatorConfig.java
@@ -0,0 +1,66 @@
+package com.ankurm.actuator.config;
+
+import org.springframework.boot.security.autoconfigure.actuate.web.servlet.EndpointRequest;
+import org.springframework.boot.health.actuate.endpoint.HealthEndpoint;
+import org.springframework.boot.actuate.info.InfoEndpoint;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Profile;
+import org.springframework.core.annotation.Order;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.web.SecurityFilterChain;
+
+/**
+ * The configuration you actually want in production.
+ *
+ * <p>Three things matter here and each of them is a mistake people make:
+ *
+ * <ol>
+ *   <li><strong>Match on {@code EndpointRequest.toAnyEndpoint()}, not on a path string.</strong>
+ *       A rule written as {@code "/actuator/**"} silently stops matching the moment somebody
+ *       sets {@code management.endpoints.web.base-path}, or moves Actuator to its own port. The
+ *       matcher asks the endpoint registry, so it follows the configuration.
+ *   <li><strong>This chain is ordered ahead of the application's chain.</strong> Without an
+ *       explicit order, whichever chain Spring happens to register first wins for a given
+ *       request, and the application chain's {@code permitAll} can swallow the Actuator paths.
+ *   <li><strong>Only {@code health} and {@code info} are anonymous</strong>, and even health is
+ *       details-free for anonymous callers &mdash; see {@code show-details: when-authorized} in
+ *       application-secured.yaml. Everything else needs the ACTUATOR role.
+ * </ol>
+ *
+ * <p>In Spring Boot 4 {@code EndpointRequest} moved to
+ * {@code org.springframework.boot.security.autoconfigure.actuate.web.servlet}; in 3.x it was
+ * {@code org.springframework.boot.actuate.autoconfigure.security.servlet}. Same class, same
+ * methods, new package. See docs/02-boot-4-changes.md.
+ *
+ * <p>See docs/04-securing-actuator.md.
+ */
+@Configuration
+@Profile("secured")
+public class SecuredActuatorConfig {
+
+    @Bean
+    @Order(1)
+    SecurityFilterChain actuatorChain(HttpSecurity http) throws Exception {
+        http.securityMatcher(EndpointRequest.toAnyEndpoint())
+                .authorizeHttpRequests((requests) -> requests
+                        .requestMatchers(EndpointRequest.to(HealthEndpoint.class, InfoEndpoint.class)).permitAll()
+                        .anyRequest().hasRole("ACTUATOR"))
+                .httpBasic((basic) -> { })
+                // Actuator is a machine-to-machine API. Sessions and CSRF tokens are for
+                // browsers; a stateless chain avoids the 403-on-POST that catches people
+                // calling /actuator/loggers or /actuator/shutdown from curl.
+                .csrf((csrf) -> csrf.disable())
+                .sessionManagement((session) -> session.sessionCreationPolicy(
+                        org.springframework.security.config.http.SessionCreationPolicy.STATELESS));
+        return http.build();
+    }
+
+    @Bean
+    @Order(2)
+    SecurityFilterChain applicationChain(HttpSecurity http) throws Exception {
+        http.authorizeHttpRequests((requests) -> requests.anyRequest().permitAll())
+                .csrf((csrf) -> csrf.disable());
+        return http.build();
+    }
+}
--- a/actuator-in-production/src/main/java/com/ankurm/actuator/health/ExternalApiHealthIndicator.java
+++ b/actuator-in-production/src/main/java/com/ankurm/actuator/health/ExternalApiHealthIndicator.java
@@ -0,0 +1,80 @@
+package com.ankurm.actuator.health;
+
+import java.time.Duration;
+import java.time.Instant;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.health.contributor.AbstractHealthIndicator;
+import org.springframework.boot.health.contributor.Health;
+import org.springframework.stereotype.Component;
+import org.springframework.web.client.RestClient;
+
+/**
+ * Probes a third-party HTTP dependency.
+ *
+ * <p>Two things here are the whole point of the chapter:
+ *
+ * <ul>
+ *   <li>The client carries explicit connect and read timeouts. A health indicator without a
+ *       timeout inherits the JVM default, which is "wait forever" &mdash; and a hung indicator
+ *       hangs {@code /actuator/health}, which hangs the load balancer probe, which takes the
+ *       whole fleet out. See docs/06-health-indicator-failure-modes.md.
+ *   <li>It is registered under the group {@code readiness} only, never {@code liveness}. A
+ *       third party being down must not restart your pod. See docs/07-groups-and-probes.md.
+ * </ul>
+ *
+ * <p>In Spring Boot 4 the base class moved: {@code AbstractHealthIndicator} and {@code Health}
+ * are in {@code org.springframework.boot.health.contributor}, not
+ * {@code org.springframework.boot.actuate.health}. See docs/02-boot-4-changes.md.
+ */
+@Component("externalApi")
+public class ExternalApiHealthIndicator extends AbstractHealthIndicator {
+
+    private final RestClient client;
+    private final String url;
+    private final Duration budget;
+
+    public ExternalApiHealthIndicator(
+            RestClient.Builder builder,
+            @Value("${demo.upstream.url:http://localhost:8080/stub/upstream/ping}") String url,
+            @Value("${demo.upstream.timeout-ms:750}") long timeoutMs,
+            @Value("${demo.upstream.username:}") String username,
+            @Value("${demo.upstream.password:}") String password) {
+        this.url = url;
+        this.budget = Duration.ofMillis(timeoutMs);
+        var factory = new org.springframework.http.client.SimpleClientHttpRequestFactory();
+        factory.setConnectTimeout(this.budget);
+        factory.setReadTimeout(this.budget);
+        if (!username.isEmpty()) {
+            // Real upstreams are authenticated, and a health indicator that forgets its
+            // credentials reports DOWN for a reason that has nothing to do with the upstream.
+            // This repository hit exactly that: the first captured run showed
+            //   "error": "HttpClientErrorException$Unauthorized: 401"
+            // which looks like an outage and was actually a missing Authorization header.
+            builder = builder.defaultHeaders((headers) -> headers.setBasicAuth(username, password));
+        }
+        this.client = builder.requestFactory(factory).build();
+    }
+
+    @Override
+    protected void doHealthCheck(Health.Builder builder) {
+        Instant start = Instant.now();
+        try {
+            String body = this.client.get().uri(this.url).retrieve().body(String.class);
+            builder.up()
+                    .withDetail("url", this.url)
+                    .withDetail("response", body)
+                    .withDetail("latencyMs", Duration.between(start, Instant.now()).toMillis())
+                    .withDetail("timeoutMs", this.budget.toMillis());
+        }
+        catch (Exception ex) {
+            // down(ex) records the exception under the "error" detail key. That detail is only
+            // rendered when show-details permits it, which is why an unauthenticated caller
+            // still sees a bare {"status":"DOWN"}.
+            builder.down(ex)
+                    .withDetail("url", this.url)
+                    .withDetail("latencyMs", Duration.between(start, Instant.now()).toMillis())
+                    .withDetail("timeoutMs", this.budget.toMillis());
+        }
+    }
+}
--- a/actuator-in-production/src/main/java/com/ankurm/actuator/health/KafkaHealthIndicator.java
+++ b/actuator-in-production/src/main/java/com/ankurm/actuator/health/KafkaHealthIndicator.java
@@ -0,0 +1,121 @@
+package com.ankurm.actuator.health;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.AdminClientConfig;
+import org.apache.kafka.clients.admin.DescribeClusterOptions;
+import org.apache.kafka.clients.admin.DescribeClusterResult;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.health.contributor.AbstractHealthIndicator;
+import org.springframework.boot.health.contributor.Health;
+import org.springframework.stereotype.Component;
+
+/**
+ * Checks a Kafka cluster with {@code AdminClient.describeCluster}, in two configurations, so
+ * the difference between them can be measured instead of argued about.
+ *
+ * <p><strong>Naive ({@code demo.kafka.tuned=false}).</strong> Create an {@code AdminClient}
+ * with only the bootstrap servers set and call {@code describeCluster().nodes().get()}. Against
+ * an unreachable broker that call returns after {@code default.api.timeout.ms}, whose Kafka
+ * default is <strong>60000</strong>. Not {@code request.timeout.ms} (30000) &mdash; that bounds
+ * one attempt, and {@code retries} defaults to {@code Integer.MAX_VALUE}, so attempts keep
+ * happening until the API timeout fires. A health endpoint that blocks for a minute is worse
+ * than no health endpoint: every probe piles another thread onto the container.
+ *
+ * <p><strong>Tuned ({@code demo.kafka.tuned=true}, the default here).</strong> Every one of the
+ * four bounds is set, and there is a hard {@code KafkaFuture.get(timeout)} outside them all.
+ * {@code metadata.recovery.strategy} is pinned to {@code none}: Kafka 4's default is
+ * {@code rebootstrap}, and a long-lived AdminClient pointed at a dead broker will otherwise
+ * fill your logs with rebootstrap lines from its background thread &mdash; this repository's
+ * first run produced 192 of them in a few seconds.
+ *
+ * <p>The {@code AdminClient} is created once and reused. Creating one per probe opens a fresh
+ * set of broker connections on every probe interval, which across a fleet is a denial of
+ * service against your own brokers.
+ *
+ * <p>No broker runs in this repository, so this indicator reports DOWN. That is deliberate:
+ * the transcripts in docs/output/ are real.
+ *
+ * <p>See docs/05-custom-health-indicators.md and docs/06-health-indicator-failure-modes.md.
+ */
+@Component("kafka")
+public class KafkaHealthIndicator extends AbstractHealthIndicator implements AutoCloseable {
+
+    private final AdminClient admin;
+    private final String bootstrap;
+    private final Duration budget;
+    private final boolean tuned;
+
+    public KafkaHealthIndicator(
+            @Value("${demo.kafka.bootstrap:localhost:9092}") String bootstrap,
+            @Value("${demo.kafka.timeout-ms:1500}") long timeoutMs,
+            @Value("${demo.kafka.tuned:true}") boolean tuned) {
+        this.bootstrap = bootstrap;
+        this.budget = Duration.ofMillis(timeoutMs);
+        this.tuned = tuned;
+
+        Map<String, Object> config = new HashMap<>();
+        config.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrap);
+        config.put(AdminClientConfig.CLIENT_ID_CONFIG, "health-" + (tuned ? "tuned" : "naive"));
+        if (tuned) {
+            // (1) how long one request may take
+            config.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, (int) timeoutMs);
+            // (2) how long the whole API call may take, retries included. Without this the
+            //     call runs for 60s regardless of (1).
+            config.put(AdminClientConfig.DEFAULT_API_TIMEOUT_MS_CONFIG, (int) timeoutMs);
+            // (3) how long a TCP connect may take
+            config.put(AdminClientConfig.SOCKET_CONNECTION_SETUP_TIMEOUT_MS_CONFIG, (int) timeoutMs);
+            // (4) do not retry on the health path; the caller will probe again shortly
+            config.put(AdminClientConfig.RETRIES_CONFIG, 0);
+            config.put(CommonClientConfigs.RECONNECT_BACKOFF_MAX_MS_CONFIG, 5000);
+            config.put(CommonClientConfigs.METADATA_RECOVERY_STRATEGY_CONFIG, "none");
+        }
+        this.admin = AdminClient.create(config);
+    }
+
+    @Override
+    protected void doHealthCheck(Health.Builder builder) {
+        Instant start = Instant.now();
+        try {
+            // The naive path is deliberately the textbook one: no per-call timeout override,
+            // and a bare get(). Whatever bounds it is Kafka's own default, which is the point.
+            DescribeClusterResult result = this.tuned
+                    ? this.admin.describeCluster(
+                            new DescribeClusterOptions().timeoutMs((int) this.budget.toMillis()))
+                    : this.admin.describeCluster();
+            int nodes = this.tuned
+                    ? result.nodes().get(this.budget.toMillis(), TimeUnit.MILLISECONDS).size()
+                    : result.nodes().get().size();
+            String clusterId = this.tuned
+                    ? result.clusterId().get(this.budget.toMillis(), TimeUnit.MILLISECONDS)
+                    : result.clusterId().get();
+            builder.up()
+                    .withDetail("bootstrap", this.bootstrap)
+                    .withDetail("tuned", this.tuned)
+                    .withDetail("clusterId", clusterId)
+                    .withDetail("nodes", nodes)
+                    .withDetail("probeMs", Duration.between(start, Instant.now()).toMillis());
+        }
+        catch (Exception ex) {
+            Throwable cause = (ex.getCause() != null) ? ex.getCause() : ex;
+            builder.down()
+                    .withDetail("bootstrap", this.bootstrap)
+                    .withDetail("tuned", this.tuned)
+                    .withDetail("error", cause.getClass().getSimpleName() + ": " + cause.getMessage())
+                    .withDetail("probeMs", Duration.between(start, Instant.now()).toMillis())
+                    .withDetail("budgetMs", this.budget.toMillis());
+        }
+    }
+
+    @Override
+    public void close() {
+        this.admin.close(Duration.ofSeconds(2));
+    }
+}
--- a/actuator-in-production/src/main/java/com/ankurm/actuator/health/OrdersDatabaseHealthIndicator.java
+++ b/actuator-in-production/src/main/java/com/ankurm/actuator/health/OrdersDatabaseHealthIndicator.java
@@ -0,0 +1,49 @@
+package com.ankurm.actuator.health;
+
+import java.time.Duration;
+import java.time.Instant;
+
+import org.springframework.boot.health.contributor.AbstractHealthIndicator;
+import org.springframework.boot.health.contributor.Health;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Component;
+
+/**
+ * A business-level database check, deliberately different from Boot's built-in
+ * {@code DataSourceHealthIndicator}.
+ *
+ * <p>The built-in one runs a validation query and reports UP if the connection works. That
+ * answers "can I reach the database", which is rarely the question that matters. This one
+ * answers "can I serve orders": it queries the table the application actually depends on, and
+ * it fails if the query is slow enough that requests would time out anyway.
+ *
+ * <p>Note the {@code @Component} name: it becomes the key in the {@code /actuator/health}
+ * response. Boot strips a trailing "HealthIndicator" from the bean name, so this bean could
+ * also have been named {@code ordersDatabaseHealthIndicator} for the same result.
+ *
+ * <p>See docs/05-custom-health-indicators.md.
+ */
+@Component("ordersDatabase")
+public class OrdersDatabaseHealthIndicator extends AbstractHealthIndicator {
+
+    private final JdbcTemplate jdbc;
+    private final Duration slowThreshold = Duration.ofMillis(250);
+
+    public OrdersDatabaseHealthIndicator(JdbcTemplate jdbc) {
+        this.jdbc = jdbc;
+        // A query timeout is not optional. Without it the indicator blocks on the socket for
+        // as long as the driver's default allows, which on some drivers is indefinitely.
+        this.jdbc.setQueryTimeout(2);
+    }
+
+    @Override
+    protected void doHealthCheck(Health.Builder builder) {
+        Instant start = Instant.now();
+        Integer count = this.jdbc.queryForObject("SELECT COUNT(*) FROM orders", Integer.class);
+        Duration took = Duration.between(start, Instant.now());
+        builder.status(took.compareTo(this.slowThreshold) > 0 ? "DEGRADED" : "UP")
+                .withDetail("orders", count)
+                .withDetail("queryMs", took.toMillis())
+                .withDetail("slowThresholdMs", this.slowThreshold.toMillis());
+    }
+}
--- a/actuator-in-production/src/main/java/com/ankurm/actuator/health/UpstreamState.java
+++ b/actuator-in-production/src/main/java/com/ankurm/actuator/health/UpstreamState.java
@@ -0,0 +1,26 @@
+package com.ankurm.actuator.health;
+
+import java.util.concurrent.atomic.AtomicReference;
+import org.springframework.stereotype.Component;
+
+/**
+ * Shared, mutable state for the stub upstream service. Flipping this at runtime is how the
+ * companion scripts make a health indicator go DOWN without needing a real outage.
+ *
+ * <p>See docs/05-custom-health-indicators.md.
+ */
+@Component
+public class UpstreamState {
+
+    public enum Mode { UP, DOWN, SLOW }
+
+    private final AtomicReference<Mode> mode = new AtomicReference<>(Mode.UP);
+
+    public Mode get() {
+        return this.mode.get();
+    }
+
+    public void set(Mode mode) {
+        this.mode.set(mode);
+    }
+}
--- a/actuator-in-production/src/main/java/com/ankurm/actuator/web/DiagnosticsEndpoint.java
+++ b/actuator-in-production/src/main/java/com/ankurm/actuator/web/DiagnosticsEndpoint.java
@@ -0,0 +1,96 @@
+package com.ankurm.actuator.web;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.springframework.boot.actuate.endpoint.annotation.Endpoint;
+import org.springframework.boot.actuate.endpoint.annotation.ReadOperation;
+import org.springframework.boot.actuate.endpoint.web.ExposableWebEndpoint;
+import org.springframework.boot.actuate.endpoint.web.WebEndpointsSupplier;
+import org.springframework.boot.actuate.endpoint.web.WebOperation;
+import org.springframework.boot.health.contributor.HealthContributor;
+import org.springframework.boot.health.contributor.HealthContributors;
+import org.springframework.boot.health.registry.HealthContributorRegistry;
+import org.springframework.core.env.Environment;
+import org.springframework.stereotype.Component;
+
+/**
+ * Prints the runtime Actuator state that no configuration file will tell you: which endpoints
+ * were actually discovered and exposed on the web, which HTTP methods and paths each one
+ * publishes, and which health contributors are registered.
+ *
+ * <p>This exists because guessing at exposure from {@code application.yaml} is how people ship
+ * {@code /actuator/heapdump} to the internet. Ask the running application instead.
+ *
+ * <p><strong>Delete this before you ship.</strong> It is a debugging aid, and the list of
+ * exposed endpoints is itself reconnaissance.
+ *
+ * <p>See docs/08-diagnostics.md.
+ */
+@Component
+@Endpoint(id = "diag")
+public class DiagnosticsEndpoint {
+
+    private final WebEndpointsSupplier webEndpoints;
+    private final HealthContributorRegistry healthRegistry;
+    private final Environment environment;
+
+    public DiagnosticsEndpoint(WebEndpointsSupplier webEndpoints,
+            HealthContributorRegistry healthRegistry, Environment environment) {
+        this.webEndpoints = webEndpoints;
+        this.healthRegistry = healthRegistry;
+        this.environment = environment;
+    }
+
+    @ReadOperation
+    public Map<String, Object> diagnostics() {
+        Map<String, Object> result = new LinkedHashMap<>();
+        result.put("activeProfiles", List.of(this.environment.getActiveProfiles()));
+        result.put("serverPort", this.environment.getProperty("server.port", "8080"));
+        result.put("managementPort",
+                this.environment.getProperty("management.server.port", "(same as server.port)"));
+        result.put("managementBasePath",
+                this.environment.getProperty("management.endpoints.web.base-path", "/actuator"));
+        result.put("exposureInclude",
+                this.environment.getProperty("management.endpoints.web.exposure.include", "health"));
+        result.put("exposureExclude",
+                this.environment.getProperty("management.endpoints.web.exposure.exclude", "(none)"));
+        result.put("healthShowDetails",
+                this.environment.getProperty("management.endpoint.health.show-details", "never"));
+
+        Map<String, List<String>> exposed = new TreeMap<>();
+        for (ExposableWebEndpoint endpoint : this.webEndpoints.getEndpoints()) {
+            List<String> ops = new ArrayList<>();
+            for (WebOperation operation : endpoint.getOperations()) {
+                var predicate = operation.getRequestPredicate();
+                ops.add(predicate.getHttpMethod() + " " + predicate.getPath());
+            }
+            ops.sort(String::compareTo);
+            exposed.put(endpoint.getEndpointId().toString(), ops);
+        }
+        result.put("exposedWebEndpointCount", exposed.size());
+        result.put("exposedWebEndpoints", exposed);
+
+        List<String> contributors = new ArrayList<>();
+        collect("", this.healthRegistry, contributors);
+        contributors.sort(String::compareTo);
+        result.put("healthContributors", contributors);
+        return result;
+    }
+
+    private void collect(String prefix, HealthContributors contributors, List<String> into) {
+        for (HealthContributors.Entry entry : contributors) {
+            String name = prefix + entry.name();
+            HealthContributor contributor = entry.contributor();
+            if (contributor instanceof HealthContributors nested) {
+                collect(name + "/", nested, into);
+            }
+            else {
+                into.add(name + "  (" + contributor.getClass().getSimpleName() + ")");
+            }
+        }
+    }
+}
--- a/actuator-in-production/src/main/java/com/ankurm/actuator/web/OrderController.java
+++ b/actuator-in-production/src/main/java/com/ankurm/actuator/web/OrderController.java
@@ -0,0 +1,22 @@
+package com.ankurm.actuator.web;
+
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+/** A minimal business endpoint, so the application is not only Actuator. */
+@RestController
+public class OrderController {
+
+    private final JdbcTemplate jdbc;
+
+    public OrderController(JdbcTemplate jdbc) {
+        this.jdbc = jdbc;
+    }
+
+    @GetMapping("/orders/count")
+    public String count() {
+        Integer n = this.jdbc.queryForObject("SELECT COUNT(*) FROM orders", Integer.class);
+        return "orders=" + n;
+    }
+}
--- a/actuator-in-production/src/main/java/com/ankurm/actuator/web/StubUpstreamController.java
+++ b/actuator-in-production/src/main/java/com/ankurm/actuator/web/StubUpstreamController.java
@@ -0,0 +1,47 @@
+package com.ankurm.actuator.web;
+
+import com.ankurm.actuator.health.UpstreamState;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+/**
+ * Stands in for the third-party API that {@code ExternalApiHealthIndicator} probes.
+ *
+ * <p>Keeping the "external" dependency inside the same JVM makes every failure in this
+ * repository reproducible offline and deterministic. A real deployment would point the
+ * indicator at a real host; nothing else about the indicator changes.
+ *
+ * <p>See docs/05-custom-health-indicators.md.
+ */
+@RestController
+@RequestMapping("/stub/upstream")
+public class StubUpstreamController {
+
+    private final UpstreamState state;
+
+    public StubUpstreamController(UpstreamState state) {
+        this.state = state;
+    }
+
+    @GetMapping("/ping")
+    public ResponseEntity<String> ping() throws InterruptedException {
+        switch (this.state.get()) {
+            case DOWN -> {
+                return ResponseEntity.status(503).body("upstream unavailable");
+            }
+            case SLOW -> Thread.sleep(30_000L);
+            case UP -> { }
+        }
+        return ResponseEntity.ok("pong");
+    }
+
+    @PostMapping("/mode")
+    public String mode(@RequestParam("value") String value) {
+        this.state.set(UpstreamState.Mode.valueOf(value.toUpperCase()));
+        return "upstream mode = " + this.state.get();
+    }
+}
--- a/actuator-in-production/src/main/resources/application-details.yaml
+++ b/actuator-in-production/src/main/resources/application-details.yaml
@@ -0,0 +1,9 @@
+management:
+  endpoints:
+    web:
+      exposure:
+        include: health,info,diag
+  endpoint:
+    health:
+      show-details: always
+      show-components: always
--- a/actuator-in-production/src/main/resources/application-exposeall.yaml
+++ b/actuator-in-production/src/main/resources/application-exposeall.yaml
@@ -0,0 +1,7 @@
+# The configuration that appears in most tutorials, and the reason Actuator has a reputation
+# for leaking. Exposes every endpoint the classpath provides on the web, including heapdump.
+management:
+  endpoints:
+    web:
+      exposure:
+        include: "*"
--- a/actuator-in-production/src/main/resources/application-groups.yaml
+++ b/actuator-in-production/src/main/resources/application-groups.yaml
@@ -0,0 +1,38 @@
+# Health groups: the fix for "a third-party outage restarted every pod we own".
+#
+# liveness  -> is this JVM broken beyond recovery? Restarting is the only cure.
+# readiness -> should this instance receive traffic right now?
+#
+# The external API and Kafka belong in readiness. They must never appear in liveness: a
+# dependency being down is not a reason for the orchestrator to kill your process, and if it
+# does, every instance restarts at once and you have turned a partial outage into a total one.
+management:
+  endpoints:
+    web:
+      exposure:
+        include: health,info,diag
+  endpoint:
+    health:
+      show-details: always
+      group:
+        liveness:
+          include: livenessState,diskSpace
+          show-details: always
+        readiness:
+          include: readinessState,ordersDatabase,externalApi
+          show-details: always
+          # Kafka being down should degrade, not black-hole, this instance. OUT_OF_SERVICE
+          # still maps to 503 by default; the additional-path below is what the probe hits.
+        startup:
+          include: ordersDatabase
+          show-details: always
+      probes:
+        enabled: true
+
+# Kafka is left out of the readiness group above ONLY so that this demonstration has one
+# moving part. No broker runs in this repository, so including it would pin readiness to 503
+# and hide the effect of the external API flipping. In a real service Kafka belongs in
+# readiness alongside the database.
+demo:
+  kafka:
+    bootstrap: localhost:9092
--- a/actuator-in-production/src/main/resources/application-kafkanaive.yaml
+++ b/actuator-in-production/src/main/resources/application-kafkanaive.yaml
@@ -0,0 +1,17 @@
+# The textbook AdminClient health check: bootstrap servers and nothing else.
+#
+# demo.kafka.timeout-ms is set far above Kafka's own default on purpose. It is NOT what bounds
+# the call here - the point of this profile is to let Kafka's default.api.timeout.ms (60000) be
+# the binding constraint and to measure it. See docs/06-health-indicator-failure-modes.md.
+demo:
+  kafka:
+    tuned: false
+    timeout-ms: 180000
+management:
+  endpoints:
+    web:
+      exposure:
+        include: health,info,diag
+  endpoint:
+    health:
+      show-details: always
--- a/actuator-in-production/src/main/resources/application-mgmtport.yaml
+++ b/actuator-in-production/src/main/resources/application-mgmtport.yaml
@@ -0,0 +1,16 @@
+# Actuator on its own port, on its own path.
+#
+# The point is not tidiness. It is that port 9001 can be bound to the pod network and left out
+# of the ingress/load-balancer configuration entirely, so /actuator is unreachable from the
+# internet by routing rather than by an authorisation rule you have to keep correct.
+management:
+  server:
+    port: 9001
+    # Bind to loopback only. In Kubernetes you would leave this unset and simply not list 9001
+    # as a Service port; here it demonstrates that the address is separately controllable.
+    address: 127.0.0.1
+  endpoints:
+    web:
+      base-path: /manage
+      exposure:
+        include: "*"
--- a/actuator-in-production/src/main/resources/application-nokafka.yaml
+++ b/actuator-in-production/src/main/resources/application-nokafka.yaml
@@ -0,0 +1,6 @@
+# Points the Kafka indicator at nothing at all, with a very short budget, so the DOWN path is
+# fast. Used by the timeout demonstration.
+demo:
+  kafka:
+    bootstrap: 10.255.255.1:9092
+    timeout-ms: 400
--- a/actuator-in-production/src/main/resources/application-open.yaml
+++ b/actuator-in-production/src/main/resources/application-open.yaml
@@ -0,0 +1,9 @@
+# Pairs with OpenActuatorConfig. Deliberately bad. Do not copy.
+management:
+  endpoints:
+    web:
+      exposure:
+        include: "*"
+  endpoint:
+    health:
+      show-details: always
--- a/actuator-in-production/src/main/resources/application-secured.yaml
+++ b/actuator-in-production/src/main/resources/application-secured.yaml
@@ -0,0 +1,13 @@
+# Pairs with SecuredActuatorConfig.
+management:
+  endpoints:
+    web:
+      exposure:
+        include: "*"
+  endpoint:
+    health:
+      # Anonymous callers get {"status":"UP"}. Authenticated ACTUATOR callers get the
+      # per-contributor breakdown. This is the setting that keeps a health endpoint useful to
+      # your operators without telling an attacker which of your dependencies is wobbling.
+      show-details: when-authorized
+      roles: ACTUATOR
--- a/actuator-in-production/src/main/resources/application.yaml
+++ b/actuator-in-production/src/main/resources/application.yaml
@@ -0,0 +1,65 @@
+# Base configuration. Everything Actuator-related here is either a Spring Boot default written
+# out explicitly (so you can see it) or a demo knob. Profile files layer on top.
+
+spring:
+  application:
+    name: actuator-production
+  datasource:
+    url: jdbc:h2:mem:orders;DB_CLOSE_DELAY=-1
+    username: sa
+    password: "not-a-real-password-but-watch-what-/actuator/env-does-with-it"
+  jpa:
+    hibernate:
+      ddl-auto: none
+  sql:
+    init:
+      mode: always
+  security:
+    user:
+      name: ops
+      password: ops-password
+      roles: ACTUATOR
+
+server:
+  port: 8080
+
+demo:
+  upstream:
+    url: http://localhost:8080/stub/upstream/ping
+    timeout-ms: 750
+    # The stub is behind this application's own security. A real upstream would be behind
+    # someone else's. Either way the indicator must present credentials, or it reports DOWN
+    # for the wrong reason - see the note in ExternalApiHealthIndicator.
+    username: ops
+    password: ops-password
+  kafka:
+    bootstrap: localhost:9092
+    timeout-ms: 1500
+
+# A custom property whose name does NOT match Spring Boot's sanitisation patterns.
+# /actuator/env treats it exactly like any other value - see docs/03-endpoint-catalogue.md.
+acme:
+  partner:
+    credential: "S3CRET-partner-credential"
+
+management:
+  info:
+    env:
+      enabled: true
+  endpoint:
+    health:
+      # Boot's default. Spelled out so the contrast with the 'details' profile is visible.
+      show-details: never
+
+info:
+  app:
+    name: actuator-production
+    purpose: companion repository for the ankurm.com Actuator article
+
+logging:
+  level:
+    # The AdminClient's background thread is chatty when the broker is unreachable.
+    # Quietened here so docs/output/ transcripts stay readable; see the note in
+    # KafkaHealthIndicator about metadata.recovery.strategy.
+    org.apache.kafka.clients.admin.internals.AdminMetadataManager: WARN
+    org.apache.kafka.clients.NetworkClient: ERROR
--- a/actuator-in-production/src/main/resources/data.sql
+++ b/actuator-in-production/src/main/resources/data.sql
@@ -0,0 +1,4 @@
+DELETE FROM orders;
+INSERT INTO orders (id, customer, total_cents) VALUES (1, 'acme', 1999);
+INSERT INTO orders (id, customer, total_cents) VALUES (2, 'globex', 24500);
+INSERT INTO orders (id, customer, total_cents) VALUES (3, 'initech', 750);
--- a/actuator-in-production/src/main/resources/schema.sql
+++ b/actuator-in-production/src/main/resources/schema.sql
@@ -0,0 +1,5 @@
+CREATE TABLE IF NOT EXISTS orders (
+  id BIGINT PRIMARY KEY,
+  customer VARCHAR(64) NOT NULL,
+  total_cents BIGINT NOT NULL
+);
--- a/actuator-in-production/src/test/java/com/ankurm/actuator/ActuatorExposureTests.java
+++ b/actuator-in-production/src/test/java/com/ankurm/actuator/ActuatorExposureTests.java
@@ -0,0 +1,74 @@
+package com.ankurm.actuator;
+
+import org.junit.jupiter.api.Test;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.http.MediaType;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import org.springframework.web.context.WebApplicationContext;
+
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+/**
+ * Contract tests. These pin the surprising behaviour, not the happy path &mdash; if a future
+ * Spring Boot upgrade changes any of it, this suite is where you find out.
+ *
+ * <p>See docs/09-testing-actuator.md.
+ */
+@SpringBootTest
+@ActiveProfiles({ "exposeall", "secured" })
+class ActuatorExposureTests {
+
+    @Autowired
+    private WebApplicationContext context;
+
+    private MockMvc mvc() {
+        return MockMvcBuilders.webAppContextSetup(this.context).apply(springSecurity()).build();
+    }
+
+    @Test
+    void heapdumpIsNotExposedEvenWithWildcardExposure() throws Exception {
+        // management.endpoint.heapdump.access defaults to 'none'. Wildcard exposure does not
+        // override access. This is the assertion that fails loudest if someone "fixes" it.
+        mvc().perform(get("/actuator/heapdump")).andExpect(status().isNotFound());
+    }
+
+    @Test
+    void shutdownIsNotExposedEvenWithWildcardExposure() throws Exception {
+        mvc().perform(post("/actuator/shutdown")).andExpect(status().isNotFound());
+    }
+
+    @Test
+    void healthIsAnonymousButEnvIsNot() throws Exception {
+        mvc().perform(get("/actuator/health")).andExpect(status().isServiceUnavailable());
+        mvc().perform(get("/actuator/env")).andExpect(status().isUnauthorized());
+    }
+
+    @Test
+    void envMasksEveryValueRegardlessOfKeyName() throws Exception {
+        // Not just keys that look like secrets. show-values defaults to 'never'.
+        mvc().perform(get("/actuator/env/acme.partner.credential")
+                        .with(org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors
+                                .user("ops").roles("ACTUATOR"))
+                        .accept(MediaType.APPLICATION_JSON))
+                .andExpect(status().isOk())
+                .andExpect(org.springframework.test.web.servlet.result.MockMvcResultMatchers
+                        .jsonPath("$.property.value").value("******"));
+    }
+
+    @Test
+    void anonymousHealthCarriesNoComponentBreakdown() throws Exception {
+        // show-details: when-authorized. An anonymous prober must not learn which dependency
+        // is failing.
+        mvc().perform(get("/actuator/health"))
+                .andExpect(status().isServiceUnavailable())
+                .andExpect(org.springframework.test.web.servlet.result.MockMvcResultMatchers
+                        .jsonPath("$.components").doesNotExist());
+    }
+}
--- a/actuator-in-production/src/test/java/com/ankurm/actuator/HealthGroupTests.java
+++ b/actuator-in-production/src/test/java/com/ankurm/actuator/HealthGroupTests.java
@@ -0,0 +1,81 @@
+package com.ankurm.actuator;
+
+import com.ankurm.actuator.health.UpstreamState;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.health.contributor.Status;
+import org.springframework.boot.health.registry.HealthContributorRegistry;
+import org.springframework.boot.health.contributor.HealthIndicator;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.ActiveProfiles;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * The contract that matters operationally: an upstream outage must move readiness and must NOT
+ * move liveness.
+ *
+ * <p>See docs/07-groups-and-probes.md.
+ */
+// DEFINED_PORT, not the default MOCK environment. ExternalApiHealthIndicator makes a real
+// HTTP call to this application's own stub controller, so a servlet container has to be
+// listening on the port the indicator was configured with. With the mock environment the
+// indicator reports DOWN with "Connection refused" and the test proves nothing.
+@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT)
+@ActiveProfiles("groups")
+class HealthGroupTests {
+
+    @Autowired
+    private HealthContributorRegistry registry;
+
+    @Autowired
+    private UpstreamState upstream;
+
+    @AfterEach
+    void reset() {
+        this.upstream.set(UpstreamState.Mode.UP);
+    }
+
+    private Status statusOf(String name) {
+        var contributor = this.registry.getContributor(name);
+        assertThat(contributor).as("contributor '%s' is registered", name).isNotNull();
+        return ((HealthIndicator) contributor).health().getStatus();
+    }
+
+    @Test
+    void everyCustomIndicatorIsRegisteredUnderTheExpectedName() {
+        // The bean name, minus a trailing "HealthIndicator", is the key in the JSON response.
+        // Rename the bean and you silently break every dashboard that reads it.
+        assertThat(statusOf("ordersDatabase")).isNotNull();
+        assertThat(statusOf("externalApi")).isNotNull();
+        assertThat(statusOf("kafka")).isNotNull();
+    }
+
+    @Test
+    void upstreamOutageMovesTheExternalApiIndicatorButNotLiveness() {
+        assertThat(statusOf("externalApi")).isEqualTo(Status.UP);
+        assertThat(statusOf("livenessState")).isEqualTo(Status.UP);
+
+        this.upstream.set(UpstreamState.Mode.DOWN);
+
+        assertThat(statusOf("externalApi")).isEqualTo(Status.DOWN);
+        assertThat(statusOf("livenessState"))
+                .as("a third-party outage must never make this process look unrecoverable")
+                .isEqualTo(Status.UP);
+    }
+
+    @Test
+    void theExternalApiIndicatorRespectsItsTimeoutBudget() {
+        this.upstream.set(UpstreamState.Mode.SLOW);
+        long start = System.nanoTime();
+        Status status = statusOf("externalApi");
+        long elapsedMs = (System.nanoTime() - start) / 1_000_000;
+
+        assertThat(status).isEqualTo(Status.DOWN);
+        // The stub sleeps 30s. demo.upstream.timeout-ms is 750. If this assertion ever fails,
+        // someone removed the read timeout and the health endpoint can now block a worker.
+        assertThat(elapsedMs).isLessThan(5_000L);
+    }
+}