asmhatreandClaude Opus 5
a065696478
Companion code for "Deploying Spring Boot 4 on Kubernetes: Probes, Graceful Shutdown, Limits and JVM Ergonomics". A dependency outage under three probe-group setups, a rolling restart under load four ways (three runs each), the JVM's ergonomic choices for nine pod shapes, one GC-heavy load under five CPU limits with throttling counters, and an HPA driven by a Micrometer gauge through prometheus-adapter. Measured on k3s v1.36.4. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01C3TETMrqVUWeFkNtz3Jbo3
70 lines
4.0 KiB
Bash
70 lines
4.0 KiB
Bash
#!/usr/bin/env bash
|
|
# The same allocation-heavy load against one pod under different CPU limits and GC settings.
|
|
# Collects GC pauses from the GC log, CFS throttling from cpu.stat (via /diag/gc, before and after),
|
|
# and request latency from the load generator. -> docs/output/gc-throttling.txt (+ raw logs)
|
|
set -uo pipefail
|
|
source "$(dirname "$0")/env.sh"
|
|
kubectl apply -f "$MODULE_DIR/k8s/gc-lab.yaml" > /dev/null
|
|
GCLOG="-Xlog:gc,gc+cpu:stdout:uptime,level,tags"
|
|
VARIANTS=(
|
|
"A|500m|1Gi||500m CPU, defaults"
|
|
"B|1|1Gi||1 CPU, defaults"
|
|
"C|1|1Gi|-XX:ActiveProcessorCount=2 -XX:+UseG1GC|1 CPU, G1 forced with ActiveProcessorCount=2"
|
|
"D|1500m|2Gi||1.5 CPUs, defaults"
|
|
"E|2|2Gi||2 CPUs, defaults"
|
|
)
|
|
diag() { curl -s "$(kubectl -n "$NS" get pod -l app=gc-lab -o jsonpath='{.items[0].status.podIP}'):8080/diag/gc"; }
|
|
jvm() { curl -s "$(kubectl -n "$NS" get pod -l app=gc-lab -o jsonpath='{.items[0].status.podIP}'):8080/diag/jvm"; }
|
|
|
|
printf '%-2s %-48s | %-8s %4s | %6s %9s %8s | %10s %10s | %8s %6s %6s\n' \
|
|
"" "variant" "GC" "thr" "pauses" "pause sum" "max" "throttled" "thr. time" "requests" "p50" "p99" > "$OUT/gc-throttling.txt"
|
|
for v in "${VARIANTS[@]}"; do
|
|
IFS='|' read -r id cpu mem opts desc <<< "$v"
|
|
lid=$(echo "$id" | tr 'A-Z' 'a-z') # pod names must be lower case
|
|
kubectl -n "$NS" set resources deploy/gc-lab --limits="cpu=$cpu,memory=$mem" > /dev/null
|
|
kubectl -n "$NS" set env deploy/gc-lab "JAVA_TOOL_OPTIONS=$opts $GCLOG" > /dev/null
|
|
kubectl -n "$NS" rollout status deploy/gc-lab --timeout=180s > /dev/null
|
|
sleep 10
|
|
pod=$(kubectl -n "$NS" get pod -l app=gc-lab -o jsonpath='{.items[0].metadata.name}')
|
|
info=$(jvm)
|
|
before=$(diag)
|
|
since=$(kubectl -n "$NS" logs "$pod" | wc -l)
|
|
"$MODULE_DIR/scripts/loadgen.sh" "lg-gc-$lid" "http://gc-lab:8080/alloc?mb=16" 4 60
|
|
kubectl -n "$NS" wait --for=jsonpath='{.status.phase}'=Succeeded "pod/lg-gc-$lid" --timeout=150s > /dev/null
|
|
after=$(diag)
|
|
kubectl -n "$NS" logs "$pod" | tail -n +"$((since + 1))" | grep -E '\[gc' > "$OUT/gc-throttling-$id-gclog.txt"
|
|
kubectl -n "$NS" logs "lg-gc-$lid" > "$OUT/gc-throttling-$id-load.txt"
|
|
kubectl -n "$NS" delete pod "lg-gc-$lid" --wait=false > /dev/null
|
|
python3 - "$id" "$desc" "$info" "$before" "$after" "$OUT/gc-throttling-$id-gclog.txt" "$OUT/gc-throttling-$id-load.txt" >> "$OUT/gc-throttling.txt" <<'PY'
|
|
import json, re, sys
|
|
id, desc, info, before, after, gclog, load = sys.argv[1:]
|
|
info, b, a = json.loads(info), json.loads(before), json.loads(after)
|
|
gc = "G1" if info["flags"]["UseG1GC"].startswith("true") else "Serial" if info["flags"]["UseSerialGC"].startswith("true") else "Parallel"
|
|
threads = info["flags"]["ParallelGCThreads"].split()[0]
|
|
pauses = [float(m.group(1)) for m in re.finditer(r"Pause .*? (\d+\.\d+)ms", open(gclog).read())]
|
|
cs_b, cs_a = b["cpuStat"], a["cpuStat"]
|
|
periods = cs_a["nr_periods"] - cs_b["nr_periods"]
|
|
thr = cs_a["nr_throttled"] - cs_b["nr_throttled"]
|
|
key = "throttled_time" if "throttled_time" in cs_a else "throttled_usec"
|
|
tt = cs_a[key] - cs_b[key]
|
|
tt_ms = tt / 1e6 if key == "throttled_time" else tt / 1e3
|
|
text = open(load).read()
|
|
total = re.search(r"TOTAL \{(.*)\}", text).group(1)
|
|
ok = re.search(r"ok=(\d+)", total)
|
|
lat = re.search(r"p50=(\d+) p90=(\d+) p99=(\d+) max=(\d+)", text)
|
|
print("%-2s %-48s | %-8s %4s | %6d %8.0fms %6.1fms | %4d/%-5d %8.1fs | %8s %5sms %5sms" % (
|
|
id, desc, gc, threads, len(pauses), sum(pauses), max(pauses) if pauses else 0,
|
|
thr, periods, tt_ms / 1000, ok.group(1) if ok else "0", lat.group(1) if lat else "-", lat.group(3) if lat else "-"))
|
|
PY
|
|
tail -1 "$OUT/gc-throttling.txt"
|
|
done
|
|
{
|
|
echo
|
|
echo "# thr = ParallelGCThreads. pauses/pause sum/max from the GC log during the 60 s run."
|
|
echo "# throttled = CFS periods in which the container hit its quota / periods elapsed, from cpu.stat."
|
|
echo "# requests = successful /alloc?mb=16 calls by 4 closed-loop clients in 60 s."
|
|
} >> "$OUT/gc-throttling.txt"
|
|
kubectl -n "$NS" scale deploy/gc-lab --replicas=0 > /dev/null
|
|
python3 "$MODULE_DIR/scripts/gc-cpu-ratio.py" > /dev/null
|
|
cat "$OUT/gc-throttling.txt"
|