Add kubernetes-deployment: probes, shutdown, JVM ergonomics, HPA

Companion code for "Deploying Spring Boot 4 on Kubernetes: Probes, Graceful Shutdown, Limits and JVM Ergonomics". A dependency outage under three probe-group setups, a rolling restart under load four ways (three runs each), the JVM's ergonomic choices for nine pod shapes, one GC-heavy load under five CPU limits with throttling counters, and an HPA driven by a Micrometer gauge through prometheus-adapter. Measured on k3s v1.36.4. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01C3TETMrqVUWeFkNtz3Jbo3
2026-09-11 17:12:10 +00:00
parent 644da9e65e
commit a065696478
72 changed files with 28415 additions and 2 deletions
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# The same allocation-heavy load against one pod under different CPU limits and GC settings.
+# Collects GC pauses from the GC log, CFS throttling from cpu.stat (via /diag/gc, before and after),
+# and request latency from the load generator. -> docs/output/gc-throttling.txt (+ raw logs)
+set -uo pipefail
+source "$(dirname "$0")/env.sh"
+kubectl apply -f "$MODULE_DIR/k8s/gc-lab.yaml" > /dev/null
+GCLOG="-Xlog:gc,gc+cpu:stdout:uptime,level,tags"
+VARIANTS=(
+  "A|500m|1Gi||500m CPU, defaults"
+  "B|1|1Gi||1 CPU, defaults"
+  "C|1|1Gi|-XX:ActiveProcessorCount=2 -XX:+UseG1GC|1 CPU, G1 forced with ActiveProcessorCount=2"
+  "D|1500m|2Gi||1.5 CPUs, defaults"
+  "E|2|2Gi||2 CPUs, defaults"
+)
+diag() { curl -s "$(kubectl -n "$NS" get pod -l app=gc-lab -o jsonpath='{.items[0].status.podIP}'):8080/diag/gc"; }
+jvm() { curl -s "$(kubectl -n "$NS" get pod -l app=gc-lab -o jsonpath='{.items[0].status.podIP}'):8080/diag/jvm"; }
+
+printf '%-2s %-48s | %-8s %4s | %6s %9s %8s | %10s %10s | %8s %6s %6s\n' \
+  "" "variant" "GC" "thr" "pauses" "pause sum" "max" "throttled" "thr. time" "requests" "p50" "p99" > "$OUT/gc-throttling.txt"
+for v in "${VARIANTS[@]}"; do
+  IFS='|' read -r id cpu mem opts desc <<< "$v"
+  lid=$(echo "$id" | tr 'A-Z' 'a-z')   # pod names must be lower case
+  kubectl -n "$NS" set resources deploy/gc-lab --limits="cpu=$cpu,memory=$mem" > /dev/null
+  kubectl -n "$NS" set env deploy/gc-lab "JAVA_TOOL_OPTIONS=$opts $GCLOG" > /dev/null
+  kubectl -n "$NS" rollout status deploy/gc-lab --timeout=180s > /dev/null
+  sleep 10
+  pod=$(kubectl -n "$NS" get pod -l app=gc-lab -o jsonpath='{.items[0].metadata.name}')
+  info=$(jvm)
+  before=$(diag)
+  since=$(kubectl -n "$NS" logs "$pod" | wc -l)
+  "$MODULE_DIR/scripts/loadgen.sh" "lg-gc-$lid" "http://gc-lab:8080/alloc?mb=16" 4 60
+  kubectl -n "$NS" wait --for=jsonpath='{.status.phase}'=Succeeded "pod/lg-gc-$lid" --timeout=150s > /dev/null
+  after=$(diag)
+  kubectl -n "$NS" logs "$pod" | tail -n +"$((since + 1))" | grep -E '\[gc' > "$OUT/gc-throttling-$id-gclog.txt"
+  kubectl -n "$NS" logs "lg-gc-$lid" > "$OUT/gc-throttling-$id-load.txt"
+  kubectl -n "$NS" delete pod "lg-gc-$lid" --wait=false > /dev/null
+  python3 - "$id" "$desc" "$info" "$before" "$after" "$OUT/gc-throttling-$id-gclog.txt" "$OUT/gc-throttling-$id-load.txt" >> "$OUT/gc-throttling.txt" <<'PY'
+import json, re, sys
+id, desc, info, before, after, gclog, load = sys.argv[1:]
+info, b, a = json.loads(info), json.loads(before), json.loads(after)
+gc = "G1" if info["flags"]["UseG1GC"].startswith("true") else "Serial" if info["flags"]["UseSerialGC"].startswith("true") else "Parallel"
+threads = info["flags"]["ParallelGCThreads"].split()[0]
+pauses = [float(m.group(1)) for m in re.finditer(r"Pause .*? (\d+\.\d+)ms", open(gclog).read())]
+cs_b, cs_a = b["cpuStat"], a["cpuStat"]
+periods = cs_a["nr_periods"] - cs_b["nr_periods"]
+thr = cs_a["nr_throttled"] - cs_b["nr_throttled"]
+key = "throttled_time" if "throttled_time" in cs_a else "throttled_usec"
+tt = cs_a[key] - cs_b[key]
+tt_ms = tt / 1e6 if key == "throttled_time" else tt / 1e3
+text = open(load).read()
+total = re.search(r"TOTAL \{(.*)\}", text).group(1)
+ok = re.search(r"ok=(\d+)", total)
+lat = re.search(r"p50=(\d+) p90=(\d+) p99=(\d+) max=(\d+)", text)
+print("%-2s %-48s | %-8s %4s | %6d %8.0fms %6.1fms | %4d/%-5d %8.1fs | %8s %5sms %5sms" % (
+    id, desc, gc, threads, len(pauses), sum(pauses), max(pauses) if pauses else 0,
+    thr, periods, tt_ms / 1000, ok.group(1) if ok else "0", lat.group(1) if lat else "-", lat.group(3) if lat else "-"))
+PY
+  tail -1 "$OUT/gc-throttling.txt"
+done
+{
+  echo
+  echo "# thr = ParallelGCThreads. pauses/pause sum/max from the GC log during the 60 s run."
+  echo "# throttled = CFS periods in which the container hit its quota / periods elapsed, from cpu.stat."
+  echo "# requests = successful /alloc?mb=16 calls by 4 closed-loop clients in 60 s."
+} >> "$OUT/gc-throttling.txt"
+kubectl -n "$NS" scale deploy/gc-lab --replicas=0 > /dev/null
+python3 "$MODULE_DIR/scripts/gc-cpu-ratio.py" > /dev/null
+cat "$OUT/gc-throttling.txt"