asmhatreandClaude Opus 5
a065696478
Companion code for "Deploying Spring Boot 4 on Kubernetes: Probes, Graceful Shutdown, Limits and JVM Ergonomics". A dependency outage under three probe-group setups, a rolling restart under load four ways (three runs each), the JVM's ergonomic choices for nine pod shapes, one GC-heavy load under five CPU limits with throttling counters, and an HPA driven by a Micrometer gauge through prometheus-adapter. Measured on k3s v1.36.4. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01C3TETMrqVUWeFkNtz3Jbo3
57 lines
3.8 KiB
Bash
57 lines
3.8 KiB
Bash
#!/usr/bin/env bash
|
|
# A downstream outage, with the downstream health indicator in the LIVENESS group and then in the
|
|
# READINESS group, and in neither (the default). Outage from t=0 to t=60, then 50 s of recovery.
|
|
# -> docs/output/probes-*.txt
|
|
set -uo pipefail
|
|
source "$(dirname "$0")/env.sh"
|
|
|
|
watch_outage() { # label, file
|
|
local label="$1" file="$2"
|
|
kubectl -n "$NS" rollout status deploy/orders --timeout=180s > /dev/null
|
|
sleep 5
|
|
{
|
|
echo "# $label"
|
|
echo "# health groups: liveness=$(kubectl -n "$NS" get deploy orders -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="MANAGEMENT_ENDPOINT_HEALTH_GROUP_LIVENESS_INCLUDE")].value}') readiness=$(kubectl -n "$NS" get deploy orders -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="MANAGEMENT_ENDPOINT_HEALTH_GROUP_READINESS_INCLUDE")].value}')"
|
|
echo "# t=0: kubectl scale deploy/downstream --replicas=0 t=60: back to 1"
|
|
echo
|
|
kubectl -n "$NS" scale deploy/downstream --replicas=0 > /dev/null
|
|
local start; start=$(date +%s)
|
|
local restored=no
|
|
for _ in $(seq 1 22); do
|
|
local t=$(( $(date +%s) - start ))
|
|
if [ "$t" -ge 60 ] && [ "$restored" = no ]; then
|
|
kubectl -n "$NS" scale deploy/downstream --replicas=1 > /dev/null; restored=yes
|
|
echo "-------- downstream restored --------"
|
|
fi
|
|
local pods; pods=$(kubectl -n "$NS" get pods -l app=orders --no-headers \
|
|
-o custom-columns=N:.metadata.name,R:.status.containerStatuses[0].ready,C:.status.containerStatuses[0].restartCount,S:.status.containerStatuses[0].state \
|
|
| awk '{st=$4; sub(/map\[/,"",st); sub(/:.*/,"",st); printf "%s ready=%s restarts=%s %s | ", substr($1,length($1)-4), $2, $3, st}')
|
|
local eps; eps=$(kubectl -n "$NS" get endpointslices -l kubernetes.io/service-name=orders -o jsonpath='{range .items[*].endpoints[*]}{.conditions.ready}{" "}{end}' | tr ' ' '\n' | grep -c true)
|
|
local work; work=$(kubectl -n "$NS" exec client -- wget -q -T 2 -O - http://orders:8080/work?ms=1 2>/dev/null | grep -c pod || true)
|
|
printf 't=%3ds %s serving endpoints=%s GET /work via Service: %s\n' "$t" "$pods" "$eps" "$([ "$work" = 1 ] && echo ok || echo FAIL)"
|
|
sleep 5
|
|
done
|
|
echo
|
|
echo "# Events (probe failures and kills) for orders pods:"
|
|
kubectl -n "$NS" get events --field-selector involvedObject.kind=Pod --sort-by=.lastTimestamp \
|
|
| grep -E '(Unhealthy|Killing|BackOff).*orders-' | awk '{ $1=""; $4=""; print }' | sed 's/^ //' | sort | uniq -c | sort -rn | head -8
|
|
} | tee "$OUT/$file"
|
|
}
|
|
|
|
kubectl apply -f "$MODULE_DIR/k8s/client.yaml" > /dev/null
|
|
kubectl -n "$NS" wait --for=condition=Ready pod/client --timeout=60s > /dev/null
|
|
kubectl -n "$NS" scale deploy/downstream --replicas=1 > /dev/null
|
|
kubectl -n "$NS" delete events --all > /dev/null 2>&1
|
|
kubectl -n "$NS" set env deploy/orders MANAGEMENT_ENDPOINT_HEALTH_GROUP_LIVENESS_INCLUDE=livenessState,downstream MANAGEMENT_ENDPOINT_HEALTH_GROUP_READINESS_INCLUDE- > /dev/null
|
|
watch_outage "downstream health indicator in the LIVENESS group" probes-liveness-includes-downstream.txt
|
|
kubectl -n "$NS" rollout status deploy/downstream --timeout=60s > /dev/null
|
|
|
|
kubectl -n "$NS" delete events --all > /dev/null 2>&1
|
|
kubectl -n "$NS" set env deploy/orders MANAGEMENT_ENDPOINT_HEALTH_GROUP_LIVENESS_INCLUDE- MANAGEMENT_ENDPOINT_HEALTH_GROUP_READINESS_INCLUDE=readinessState,downstream > /dev/null
|
|
watch_outage "downstream health indicator in the READINESS group" probes-readiness-includes-downstream.txt
|
|
kubectl -n "$NS" rollout status deploy/downstream --timeout=60s > /dev/null
|
|
|
|
kubectl -n "$NS" delete events --all > /dev/null 2>&1
|
|
kubectl -n "$NS" set env deploy/orders MANAGEMENT_ENDPOINT_HEALTH_GROUP_READINESS_INCLUDE- > /dev/null
|
|
watch_outage "downstream health indicator in NEITHER group (Spring Boot's default probe groups)" probes-default-groups.txt
|