Add kubernetes-deployment: probes, shutdown, JVM ergonomics, HPA
Companion code for "Deploying Spring Boot 4 on Kubernetes: Probes, Graceful Shutdown, Limits and JVM Ergonomics". A dependency outage under three probe-group setups, a rolling restart under load four ways (three runs each), the JVM's ergonomic choices for nine pod shapes, one GC-heavy load under five CPU limits with throttling counters, and an HPA driven by a Micrometer gauge through prometheus-adapter. Measured on k3s v1.36.4. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01C3TETMrqVUWeFkNtz3Jbo3
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env bash
|
||||
# What the JVM decides from the pod's resources, with no JVM flags at all.
|
||||
# -> docs/output/jvm-ergonomics.txt
|
||||
set -uo pipefail
|
||||
source "$(dirname "$0")/env.sh"
|
||||
# cpu.limit | cpu.request | mem.request | mem.limit. Requests are deliberately small and constant:
|
||||
# the JVM ignores them (JDK-8281181 removed CPU shares from the calculation in JDK 19), and small
|
||||
# requests let every case schedule on a 2-CPU node.
|
||||
CASES=(
|
||||
"none|100m|256Mi|2Gi"
|
||||
"500m|100m|256Mi|512Mi"
|
||||
"1|100m|256Mi|1Gi"
|
||||
"1|100m|256Mi|2Gi"
|
||||
"1500m|100m|256Mi|2Gi"
|
||||
"2|100m|256Mi|1Gi"
|
||||
"2|100m|256Mi|1700Mi"
|
||||
"2|100m|256Mi|1800Mi"
|
||||
"2|100m|256Mi|4Gi"
|
||||
)
|
||||
flag() { grep -E "^ *[a-z_]+ +$1 " <<< "$2" | awk '{print $4}'; }
|
||||
{
|
||||
echo "# JVM ergonomics per pod resources. Image: $IMAGE, no JVM options. Node: $(nproc) CPUs."
|
||||
echo "# cgroup v$( [ -f /sys/fs/cgroup/cgroup.controllers ] && echo 2 || echo 1 ) node; see docs/03-jvm-ergonomics.md for the v2 equivalents."
|
||||
echo
|
||||
printf '%-7s %-7s %-7s %-7s | %-5s %-10s %9s %6s %6s %6s\n' "cpu.req" "cpu.lim" "mem.req" "mem.lim" "CPUs" "GC" "MaxHeap" "PGCThr" "CGCThr" "JITThr"
|
||||
i=0
|
||||
for c in "${CASES[@]}"; do
|
||||
IFS='|' read -r cl cr mr ml <<< "$c"
|
||||
if [ "$cl" = none ]; then res="{requests: {cpu: $cr, memory: $mr}, limits: {memory: $ml}}"
|
||||
else res="{requests: {cpu: $cr, memory: $mr}, limits: {cpu: $cl, memory: $ml}}"; fi
|
||||
out=$(oneoff "erg-$i" "$res" java -XX:+PrintFlagsFinal -version 2>&1)
|
||||
gc=SerialGC; [ "$(flag UseG1GC "$out")" = true ] && gc=G1GC; [ "$(flag UseParallelGC "$out")" = true ] && gc=ParallelGC
|
||||
cpus=$(oneoff "erg-$i" "$res" java -XshowSettings:system -version 2>&1 | grep -i 'Effective CPU Count' | awk -F: '{gsub(/ /,"",$2); print $2}')
|
||||
heap=$(( $(flag MaxHeapSize "$out") / 1024 / 1024 ))
|
||||
printf '%-7s %-7s %-7s %-7s | %-5s %-10s %8sM %6s %6s %6s\n' "$cr" "$cl" "$mr" "$ml" "$cpus" "$gc" "$heap" \
|
||||
"$(flag ParallelGCThreads "$out")" "$(flag ConcGCThreads "$out")" "$(flag CICompilerCount "$out")"
|
||||
i=$((i+1))
|
||||
done
|
||||
echo
|
||||
echo "# java -XshowSettings:system for the 1500m case:"
|
||||
oneoff erg-show "{requests: {cpu: 100m, memory: 256Mi}, limits: {cpu: 1500m, memory: 2Gi}}" java -XshowSettings:system -version 2>&1 | grep -vE '^(openjdk|OpenJDK)'
|
||||
} | tee "$OUT/jvm-ergonomics.txt"
|
||||
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env bash
|
||||
# The same allocation-heavy load against one pod under different CPU limits and GC settings.
|
||||
# Collects GC pauses from the GC log, CFS throttling from cpu.stat (via /diag/gc, before and after),
|
||||
# and request latency from the load generator. -> docs/output/gc-throttling.txt (+ raw logs)
|
||||
set -uo pipefail
|
||||
source "$(dirname "$0")/env.sh"
|
||||
kubectl apply -f "$MODULE_DIR/k8s/gc-lab.yaml" > /dev/null
|
||||
GCLOG="-Xlog:gc,gc+cpu:stdout:uptime,level,tags"
|
||||
VARIANTS=(
|
||||
"A|500m|1Gi||500m CPU, defaults"
|
||||
"B|1|1Gi||1 CPU, defaults"
|
||||
"C|1|1Gi|-XX:ActiveProcessorCount=2 -XX:+UseG1GC|1 CPU, G1 forced with ActiveProcessorCount=2"
|
||||
"D|1500m|2Gi||1.5 CPUs, defaults"
|
||||
"E|2|2Gi||2 CPUs, defaults"
|
||||
)
|
||||
diag() { curl -s "$(kubectl -n "$NS" get pod -l app=gc-lab -o jsonpath='{.items[0].status.podIP}'):8080/diag/gc"; }
|
||||
jvm() { curl -s "$(kubectl -n "$NS" get pod -l app=gc-lab -o jsonpath='{.items[0].status.podIP}'):8080/diag/jvm"; }
|
||||
|
||||
printf '%-2s %-48s | %-8s %4s | %6s %9s %8s | %10s %10s | %8s %6s %6s\n' \
|
||||
"" "variant" "GC" "thr" "pauses" "pause sum" "max" "throttled" "thr. time" "requests" "p50" "p99" > "$OUT/gc-throttling.txt"
|
||||
for v in "${VARIANTS[@]}"; do
|
||||
IFS='|' read -r id cpu mem opts desc <<< "$v"
|
||||
lid=$(echo "$id" | tr 'A-Z' 'a-z') # pod names must be lower case
|
||||
kubectl -n "$NS" set resources deploy/gc-lab --limits="cpu=$cpu,memory=$mem" > /dev/null
|
||||
kubectl -n "$NS" set env deploy/gc-lab "JAVA_TOOL_OPTIONS=$opts $GCLOG" > /dev/null
|
||||
kubectl -n "$NS" rollout status deploy/gc-lab --timeout=180s > /dev/null
|
||||
sleep 10
|
||||
pod=$(kubectl -n "$NS" get pod -l app=gc-lab -o jsonpath='{.items[0].metadata.name}')
|
||||
info=$(jvm)
|
||||
before=$(diag)
|
||||
since=$(kubectl -n "$NS" logs "$pod" | wc -l)
|
||||
"$MODULE_DIR/scripts/loadgen.sh" "lg-gc-$lid" "http://gc-lab:8080/alloc?mb=16" 4 60
|
||||
kubectl -n "$NS" wait --for=jsonpath='{.status.phase}'=Succeeded "pod/lg-gc-$lid" --timeout=150s > /dev/null
|
||||
after=$(diag)
|
||||
kubectl -n "$NS" logs "$pod" | tail -n +"$((since + 1))" | grep -E '\[gc' > "$OUT/gc-throttling-$id-gclog.txt"
|
||||
kubectl -n "$NS" logs "lg-gc-$lid" > "$OUT/gc-throttling-$id-load.txt"
|
||||
kubectl -n "$NS" delete pod "lg-gc-$lid" --wait=false > /dev/null
|
||||
python3 - "$id" "$desc" "$info" "$before" "$after" "$OUT/gc-throttling-$id-gclog.txt" "$OUT/gc-throttling-$id-load.txt" >> "$OUT/gc-throttling.txt" <<'PY'
|
||||
import json, re, sys
|
||||
id, desc, info, before, after, gclog, load = sys.argv[1:]
|
||||
info, b, a = json.loads(info), json.loads(before), json.loads(after)
|
||||
gc = "G1" if info["flags"]["UseG1GC"].startswith("true") else "Serial" if info["flags"]["UseSerialGC"].startswith("true") else "Parallel"
|
||||
threads = info["flags"]["ParallelGCThreads"].split()[0]
|
||||
pauses = [float(m.group(1)) for m in re.finditer(r"Pause .*? (\d+\.\d+)ms", open(gclog).read())]
|
||||
cs_b, cs_a = b["cpuStat"], a["cpuStat"]
|
||||
periods = cs_a["nr_periods"] - cs_b["nr_periods"]
|
||||
thr = cs_a["nr_throttled"] - cs_b["nr_throttled"]
|
||||
key = "throttled_time" if "throttled_time" in cs_a else "throttled_usec"
|
||||
tt = cs_a[key] - cs_b[key]
|
||||
tt_ms = tt / 1e6 if key == "throttled_time" else tt / 1e3
|
||||
text = open(load).read()
|
||||
total = re.search(r"TOTAL \{(.*)\}", text).group(1)
|
||||
ok = re.search(r"ok=(\d+)", total)
|
||||
lat = re.search(r"p50=(\d+) p90=(\d+) p99=(\d+) max=(\d+)", text)
|
||||
print("%-2s %-48s | %-8s %4s | %6d %8.0fms %6.1fms | %4d/%-5d %8.1fs | %8s %5sms %5sms" % (
|
||||
id, desc, gc, threads, len(pauses), sum(pauses), max(pauses) if pauses else 0,
|
||||
thr, periods, tt_ms / 1000, ok.group(1) if ok else "0", lat.group(1) if lat else "-", lat.group(3) if lat else "-"))
|
||||
PY
|
||||
tail -1 "$OUT/gc-throttling.txt"
|
||||
done
|
||||
{
|
||||
echo
|
||||
echo "# thr = ParallelGCThreads. pauses/pause sum/max from the GC log during the 60 s run."
|
||||
echo "# throttled = CFS periods in which the container hit its quota / periods elapsed, from cpu.stat."
|
||||
echo "# requests = successful /alloc?mb=16 calls by 4 closed-loop clients in 60 s."
|
||||
} >> "$OUT/gc-throttling.txt"
|
||||
kubectl -n "$NS" scale deploy/gc-lab --replicas=0 > /dev/null
|
||||
python3 "$MODULE_DIR/scripts/gc-cpu-ratio.py" > /dev/null
|
||||
cat "$OUT/gc-throttling.txt"
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env bash
|
||||
# HorizontalPodAutoscaler on a Micrometer gauge through Prometheus + prometheus-adapter.
|
||||
# Load: 2 clients, then 30 from t=20 s to t=140 s, then none; each request holds for 500 ms.
|
||||
# -> docs/output/hpa-custom-metric.txt, hpa-custom-metrics-api.txt
|
||||
set -uo pipefail
|
||||
source "$(dirname "$0")/env.sh"
|
||||
H="$MODULE_DIR/k8s/hpa"
|
||||
kubectl apply -f "$H/prometheus.yaml" -f "$H/prometheus-adapter.yaml" > /dev/null
|
||||
kubectl -n monitoring rollout status deploy/prometheus --timeout=120s > /dev/null
|
||||
kubectl -n monitoring rollout status deploy/prometheus-adapter --timeout=120s > /dev/null
|
||||
kubectl -n "$NS" scale deploy/orders --replicas=1 > /dev/null
|
||||
kubectl -n "$NS" rollout status deploy/orders --timeout=120s > /dev/null
|
||||
API=/apis/custom.metrics.k8s.io/v1beta1/namespaces/demo/pods/%2A/app_inflight_requests
|
||||
for _ in $(seq 1 40); do kubectl get --raw "$API" 2>/dev/null | grep -q '"items":\[{' && break; sleep 5; done
|
||||
{
|
||||
echo "# What Micrometer exports (one pod):"
|
||||
echo "\$ curl <pod>:8080/actuator/prometheus | grep app_inflight"
|
||||
curl -s "$(kubectl -n "$NS" get pod -l app=orders -o jsonpath='{.items[0].status.podIP}'):8080/actuator/prometheus" | grep app_inflight
|
||||
echo
|
||||
echo "# What the HPA controller sees through the custom metrics API:"
|
||||
echo "\$ kubectl get --raw $API"
|
||||
kubectl get --raw "$API" | python3 -m json.tool
|
||||
} > "$OUT/hpa-custom-metrics-api.txt"
|
||||
|
||||
kubectl apply -f "$H/hpa.yaml" > /dev/null
|
||||
sleep 20
|
||||
kubectl -n "$NS" delete events --field-selector involvedObject.kind=HorizontalPodAutoscaler > /dev/null 2>&1
|
||||
"$MODULE_DIR/scripts/loadgen.sh" lg-hpa "http://orders:8080/work?ms=500" "0:2,20:30,140:0" 230
|
||||
kubectl -n "$NS" wait --for=condition=Ready pod/lg-hpa --timeout=60s > /dev/null
|
||||
start=$(date +%s)
|
||||
{
|
||||
echo "# HPA orders: target app_inflight_requests averageValue 5, min 1, max 4; scaleDown stabilization 30 s"
|
||||
echo "# load: t=0 2 clients, t=20 30 clients, t=140 0 clients; GET /work?ms=500"
|
||||
echo
|
||||
printf '%-6s %-10s %-9s %-8s %s\n' "t" "clients" "metric" "desired" "ready pods"
|
||||
while [ $(( $(date +%s) - start )) -lt 235 ]; do
|
||||
t=$(( $(date +%s) - start ))
|
||||
clients=2; [ $t -ge 20 ] && clients=30; [ $t -ge 140 ] && clients=0
|
||||
cur=$(kubectl -n "$NS" get hpa orders -o jsonpath='{.status.currentMetrics[0].pods.current.averageValue}')
|
||||
des=$(kubectl -n "$NS" get hpa orders -o jsonpath='{.status.desiredReplicas}')
|
||||
ready=$(kubectl -n "$NS" get deploy orders -o jsonpath='{.status.readyReplicas}')
|
||||
printf '%-6s %-10s %-9s %-8s %s\n' "${t}s" "$clients" "${cur:-?}" "${des:-?}" "${ready:-0}"
|
||||
sleep 10
|
||||
done
|
||||
echo
|
||||
echo "# HPA events:"
|
||||
kubectl -n "$NS" get events --field-selector involvedObject.kind=HorizontalPodAutoscaler --sort-by=.lastTimestamp \
|
||||
| awk 'NR>1 {$1=""; $2=""; $4=""; print}' | sed 's/^ *//'
|
||||
echo
|
||||
echo "# Load generator summary:"
|
||||
kubectl -n "$NS" wait --for=jsonpath='{.status.phase}'=Succeeded pod/lg-hpa --timeout=60s > /dev/null
|
||||
kubectl -n "$NS" logs lg-hpa | grep -E '^TOTAL|^LATENCY'
|
||||
} | tee "$OUT/hpa-custom-metric.txt"
|
||||
kubectl -n "$NS" logs lg-hpa > "$OUT/hpa-load.txt"
|
||||
kubectl -n "$NS" delete pod lg-hpa --wait=false > /dev/null
|
||||
kubectl -n "$NS" delete hpa orders > /dev/null
|
||||
kubectl -n "$NS" scale deploy/orders --replicas=2 > /dev/null
|
||||
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env bash
|
||||
# A downstream outage, with the downstream health indicator in the LIVENESS group and then in the
|
||||
# READINESS group, and in neither (the default). Outage from t=0 to t=60, then 50 s of recovery.
|
||||
# -> docs/output/probes-*.txt
|
||||
set -uo pipefail
|
||||
source "$(dirname "$0")/env.sh"
|
||||
|
||||
watch_outage() { # label, file
|
||||
local label="$1" file="$2"
|
||||
kubectl -n "$NS" rollout status deploy/orders --timeout=180s > /dev/null
|
||||
sleep 5
|
||||
{
|
||||
echo "# $label"
|
||||
echo "# health groups: liveness=$(kubectl -n "$NS" get deploy orders -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="MANAGEMENT_ENDPOINT_HEALTH_GROUP_LIVENESS_INCLUDE")].value}') readiness=$(kubectl -n "$NS" get deploy orders -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="MANAGEMENT_ENDPOINT_HEALTH_GROUP_READINESS_INCLUDE")].value}')"
|
||||
echo "# t=0: kubectl scale deploy/downstream --replicas=0 t=60: back to 1"
|
||||
echo
|
||||
kubectl -n "$NS" scale deploy/downstream --replicas=0 > /dev/null
|
||||
local start; start=$(date +%s)
|
||||
local restored=no
|
||||
for _ in $(seq 1 22); do
|
||||
local t=$(( $(date +%s) - start ))
|
||||
if [ "$t" -ge 60 ] && [ "$restored" = no ]; then
|
||||
kubectl -n "$NS" scale deploy/downstream --replicas=1 > /dev/null; restored=yes
|
||||
echo "-------- downstream restored --------"
|
||||
fi
|
||||
local pods; pods=$(kubectl -n "$NS" get pods -l app=orders --no-headers \
|
||||
-o custom-columns=N:.metadata.name,R:.status.containerStatuses[0].ready,C:.status.containerStatuses[0].restartCount,S:.status.containerStatuses[0].state \
|
||||
| awk '{st=$4; sub(/map\[/,"",st); sub(/:.*/,"",st); printf "%s ready=%s restarts=%s %s | ", substr($1,length($1)-4), $2, $3, st}')
|
||||
local eps; eps=$(kubectl -n "$NS" get endpointslices -l kubernetes.io/service-name=orders -o jsonpath='{range .items[*].endpoints[*]}{.conditions.ready}{" "}{end}' | tr ' ' '\n' | grep -c true)
|
||||
local work; work=$(kubectl -n "$NS" exec client -- wget -q -T 2 -O - http://orders:8080/work?ms=1 2>/dev/null | grep -c pod || true)
|
||||
printf 't=%3ds %s serving endpoints=%s GET /work via Service: %s\n' "$t" "$pods" "$eps" "$([ "$work" = 1 ] && echo ok || echo FAIL)"
|
||||
sleep 5
|
||||
done
|
||||
echo
|
||||
echo "# Events (probe failures and kills) for orders pods:"
|
||||
kubectl -n "$NS" get events --field-selector involvedObject.kind=Pod --sort-by=.lastTimestamp \
|
||||
| grep -E '(Unhealthy|Killing|BackOff).*orders-' | awk '{ $1=""; $4=""; print }' | sed 's/^ //' | sort | uniq -c | sort -rn | head -8
|
||||
} | tee "$OUT/$file"
|
||||
}
|
||||
|
||||
kubectl apply -f "$MODULE_DIR/k8s/client.yaml" > /dev/null
|
||||
kubectl -n "$NS" wait --for=condition=Ready pod/client --timeout=60s > /dev/null
|
||||
kubectl -n "$NS" scale deploy/downstream --replicas=1 > /dev/null
|
||||
kubectl -n "$NS" delete events --all > /dev/null 2>&1
|
||||
kubectl -n "$NS" set env deploy/orders MANAGEMENT_ENDPOINT_HEALTH_GROUP_LIVENESS_INCLUDE=livenessState,downstream MANAGEMENT_ENDPOINT_HEALTH_GROUP_READINESS_INCLUDE- > /dev/null
|
||||
watch_outage "downstream health indicator in the LIVENESS group" probes-liveness-includes-downstream.txt
|
||||
kubectl -n "$NS" rollout status deploy/downstream --timeout=60s > /dev/null
|
||||
|
||||
kubectl -n "$NS" delete events --all > /dev/null 2>&1
|
||||
kubectl -n "$NS" set env deploy/orders MANAGEMENT_ENDPOINT_HEALTH_GROUP_LIVENESS_INCLUDE- MANAGEMENT_ENDPOINT_HEALTH_GROUP_READINESS_INCLUDE=readinessState,downstream > /dev/null
|
||||
watch_outage "downstream health indicator in the READINESS group" probes-readiness-includes-downstream.txt
|
||||
kubectl -n "$NS" rollout status deploy/downstream --timeout=60s > /dev/null
|
||||
|
||||
kubectl -n "$NS" delete events --all > /dev/null 2>&1
|
||||
kubectl -n "$NS" set env deploy/orders MANAGEMENT_ENDPOINT_HEALTH_GROUP_READINESS_INCLUDE- > /dev/null
|
||||
watch_outage "downstream health indicator in NEITHER group (Spring Boot's default probe groups)" probes-default-groups.txt
|
||||
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env bash
|
||||
# A rolling restart under load, four ways. Each run: 20 concurrent clients calling /work?ms=300
|
||||
# through the Service for 45 s, rollout restart at t=8 s. METHOD=POST (default) or GET.
|
||||
# -> docs/output/shutdown-<method>-<variant>.txt
|
||||
set -uo pipefail
|
||||
source "$(dirname "$0")/env.sh"
|
||||
export METHOD="${METHOD:-POST}"
|
||||
RUNS="${RUNS:-3}"
|
||||
M=$(echo "$METHOD" | tr 'A-Z' 'a-z')
|
||||
D="deploy/orders"
|
||||
PRESTOP_PATH=/spec/template/spec/containers/0/lifecycle
|
||||
|
||||
variant() { # name, description - runs it $RUNS times
|
||||
local base="$1" desc="$2" r
|
||||
for r in $(seq 1 "$RUNS"); do run_once "$base-run$r" "$desc (run $r of $RUNS)"; done
|
||||
}
|
||||
|
||||
run_once() { # name, description
|
||||
local name="$1" desc="$2"
|
||||
kubectl -n "$NS" rollout status "$D" --timeout=180s > /dev/null
|
||||
kubectl -n "$NS" delete events --all > /dev/null 2>&1
|
||||
sleep 3
|
||||
"$MODULE_DIR/scripts/loadgen.sh" "lg-$M-$name" "http://orders:8080/work?ms=300" 20 45
|
||||
kubectl -n "$NS" wait --for=condition=Ready "pod/lg-$M-$name" --timeout=60s > /dev/null
|
||||
sleep 8
|
||||
kubectl -n "$NS" rollout restart "$D" > /dev/null
|
||||
kubectl -n "$NS" rollout status "$D" --timeout=180s > /dev/null
|
||||
kubectl -n "$NS" wait --for=jsonpath='{.status.phase}'=Succeeded "pod/lg-$M-$name" --timeout=120s > /dev/null
|
||||
{
|
||||
echo "# $desc"
|
||||
echo "# 20 clients, $METHOD /work?ms=300 via Service orders, 45 s; kubectl rollout restart at ~t=8 s"
|
||||
echo
|
||||
kubectl -n "$NS" logs "lg-$M-$name"
|
||||
echo
|
||||
echo "# Events:"
|
||||
kubectl -n "$NS" get events --sort-by=.lastTimestamp | grep -E 'PreStop|Killing' | awk '{$1=""; print}' | sed 's/^ //' | sort | uniq -c | head -6
|
||||
} > "$OUT/shutdown-$M-$name.txt"
|
||||
grep -E '^TOTAL|^LATENCY' "$OUT/shutdown-$M-$name.txt" | sed "s/^/$M $name: /"
|
||||
kubectl -n "$NS" delete pod "lg-$M-$name" --wait=false > /dev/null
|
||||
}
|
||||
|
||||
# 1. server.shutdown=immediate, no preStop
|
||||
kubectl -n "$NS" patch "$D" --type=json -p "[{\"op\":\"remove\",\"path\":\"$PRESTOP_PATH\"}]" > /dev/null 2>&1
|
||||
kubectl -n "$NS" set env "$D" SERVER_SHUTDOWN=immediate > /dev/null
|
||||
variant 1-immediate-no-prestop "server.shutdown=immediate, no preStop hook"
|
||||
|
||||
# 2. graceful shutdown (the Boot default), no preStop
|
||||
kubectl -n "$NS" set env "$D" SERVER_SHUTDOWN- > /dev/null
|
||||
variant 2-graceful-no-prestop "graceful shutdown (default), no preStop hook"
|
||||
|
||||
# 3. graceful + native sleep preStop (Kubernetes 1.32+) - the baseline manifest
|
||||
kubectl -n "$NS" patch "$D" --type=json -p "[{\"op\":\"add\",\"path\":\"$PRESTOP_PATH\",\"value\":{\"preStop\":{\"sleep\":{\"seconds\":5}}}}]" > /dev/null
|
||||
variant 3-graceful-prestop-sleep "graceful shutdown + preStop: sleep: {seconds: 5}"
|
||||
|
||||
# 4. graceful + exec preStop that needs a shell - on a distroless image
|
||||
kubectl -n "$NS" patch "$D" --type=json -p "[{\"op\":\"replace\",\"path\":\"$PRESTOP_PATH\",\"value\":{\"preStop\":{\"exec\":{\"command\":[\"sh\",\"-c\",\"sleep 5\"]}}}}]" > /dev/null
|
||||
variant 4-graceful-prestop-exec-sh "graceful shutdown + preStop: exec: [sh, -c, sleep 5] on a distroless image"
|
||||
|
||||
# summary across runs
|
||||
{
|
||||
echo "# $METHOD /work?ms=300, 20 clients, rolling restart of 2 replicas. Failed requests per run (successful in brackets)."
|
||||
echo
|
||||
for f in "$OUT"/shutdown-$M-*-run1.txt; do
|
||||
v=$(basename "$f" -run1.txt); v=${v#shutdown-$M-}
|
||||
printf '%-32s' "$v"
|
||||
for r in $(seq 1 "$RUNS"); do
|
||||
t=$(grep '^TOTAL' "$OUT/shutdown-$M-$v-run$r.txt")
|
||||
ok=$(echo "$t" | grep -o 'ok=[0-9]*' | cut -d= -f2)
|
||||
fails=$(echo "$t" | grep -o '[A-Za-z_0-9]*=[0-9]*' | grep -v '^ok=' | tr '\n' ' ')
|
||||
printf ' | run %s: %-38s' "$r" "${fails:-0 failures} [$ok]"
|
||||
done
|
||||
echo
|
||||
done
|
||||
} | tee "$OUT/shutdown-$M-summary.txt"
|
||||
|
||||
# restore the baseline
|
||||
kubectl -n "$NS" patch "$D" --type=json -p "[{\"op\":\"replace\",\"path\":\"$PRESTOP_PATH\",\"value\":{\"preStop\":{\"sleep\":{\"seconds\":5}}}}]" > /dev/null
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
# An application that needs ~45 s to start, with and without a startupProbe. 1 replica; each
|
||||
# phase is ONE patch, so there is one new ReplicaSet per phase. -> docs/output/startup-probe.txt
|
||||
set -uo pipefail
|
||||
source "$(dirname "$0")/env.sh"
|
||||
D="deploy/orders"
|
||||
observe() {
|
||||
local label="$1"
|
||||
echo "## $label"
|
||||
local start; start=$(date +%s)
|
||||
for _ in $(seq 1 20); do
|
||||
local t=$(( $(date +%s) - start ))
|
||||
printf 't=%3ds ' "$t"
|
||||
kubectl -n "$NS" get pods -l app=orders --no-headers \
|
||||
-o custom-columns=N:.metadata.name,H:.metadata.labels.pod-template-hash,R:.status.containerStatuses[0].ready,C:.status.containerStatuses[0].restartCount,W:.status.containerStatuses[0].state.waiting.reason,X:.metadata.deletionTimestamp \
|
||||
| awk '{ if ($6 != "<none>") next; printf "[%s ready=%s restarts=%s%s] ", substr($1,length($1)-4), $3, $4, ($5=="<none>"?"":" "$5)}'
|
||||
echo
|
||||
sleep 6
|
||||
done
|
||||
echo
|
||||
}
|
||||
kubectl -n "$NS" scale "$D" --replicas=1 > /dev/null
|
||||
kubectl -n "$NS" rollout status "$D" --timeout=120s > /dev/null
|
||||
kubectl -n "$NS" delete events --all > /dev/null 2>&1
|
||||
{
|
||||
echo "# DEMO_STARTUP_DELAY=40s: the context takes ~45 s to refresh, and Tomcat only listens after that."
|
||||
echo "# 1 replica, rolling update (maxSurge 1, maxUnavailable 0). Liveness: period 5 s, failureThreshold 3."
|
||||
echo "# Pods being deleted are not listed. The old pod keeps serving while the new one is not ready."
|
||||
echo
|
||||
kubectl -n "$NS" patch "$D" --type=json -p '[
|
||||
{"op":"remove","path":"/spec/template/spec/containers/0/startupProbe"},
|
||||
{"op":"add","path":"/spec/template/spec/containers/0/env/-","value":{"name":"DEMO_STARTUP_DELAY","value":"40s"}}]' > /dev/null
|
||||
observe "no startupProbe"
|
||||
echo "# Events so far:"
|
||||
kubectl -n "$NS" get events --sort-by=.lastTimestamp | grep -E 'Unhealthy|Killing|BackOff' | awk '{$1=""; $4=""; print}' | sed 's/^ *//' | sort | uniq -c | head -6
|
||||
echo
|
||||
kubectl -n "$NS" delete events --all > /dev/null 2>&1
|
||||
kubectl -n "$NS" patch "$D" --type=json -p '[{"op":"add","path":"/spec/template/spec/containers/0/startupProbe","value":{"httpGet":{"path":"/actuator/health/liveness","port":"http"},"periodSeconds":2,"failureThreshold":60}}]' > /dev/null
|
||||
observe "startupProbe: /actuator/health/liveness every 2 s, failureThreshold 60 (a 120 s budget)"
|
||||
echo "# Events during the startupProbe run:"
|
||||
kubectl -n "$NS" get events --sort-by=.lastTimestamp | grep -E 'Unhealthy|Killing|BackOff' | awk '{$1=""; $4=""; print}' | sed 's/^ *//' | sort | uniq -c | head -6
|
||||
} | tee "$OUT/startup-probe.txt"
|
||||
kubectl -n "$NS" set env "$D" DEMO_STARTUP_DELAY- > /dev/null
|
||||
kubectl -n "$NS" scale "$D" --replicas=2 > /dev/null
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env bash
|
||||
MODULE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
OUT="$MODULE_DIR/docs/output"
|
||||
NS=demo
|
||||
IMAGE="${IMAGE:-sbd/k8s-demo:1}"
|
||||
mkdir -p "$OUT"
|
||||
export KUBECONFIG="${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml}"
|
||||
# Run a one-off pod from the app image with an arbitrary command and resources; print its logs.
|
||||
# oneoff <name> '<resources-json>' <command...>
|
||||
oneoff() {
|
||||
local name="$1" resources="$2"; shift 2
|
||||
local cmd; cmd=$(printf '"%s",' "$@"); cmd="[${cmd%,}]"
|
||||
kubectl -n "$NS" delete pod "$name" --ignore-not-found --wait=true > /dev/null
|
||||
kubectl -n "$NS" apply -f - > /dev/null <<YAML
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata: {name: $name, namespace: $NS, labels: {role: oneoff}}
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: app
|
||||
image: $IMAGE
|
||||
imagePullPolicy: Never
|
||||
command: $cmd
|
||||
resources: $resources
|
||||
YAML
|
||||
kubectl -n "$NS" wait --for=jsonpath='{.status.phase}'=Succeeded "pod/$name" --timeout=120s > /dev/null
|
||||
kubectl -n "$NS" logs "$name"
|
||||
kubectl -n "$NS" delete pod "$name" --wait=false > /dev/null
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env python3
|
||||
"""From the raw GC logs of demo-gc-throttling.sh: CPU time the collector got during pauses vs the
|
||||
pauses' wall time (-Xlog:gc+cpu prints User/Sys/Real per collection, at 10 ms resolution).
|
||||
A single GC thread that is never descheduled gives a ratio near 1.0; N parallel threads near N.
|
||||
Well below that means the collector was waiting for CPU - throttled - inside its own pauses.
|
||||
-> docs/output/gc-cpu-ratio.txt"""
|
||||
import os, re, sys
|
||||
out = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "docs", "output")
|
||||
rows = []
|
||||
for v in "ABCDE":
|
||||
path = os.path.join(out, f"gc-throttling-{v}-gclog.txt")
|
||||
if not os.path.exists(path):
|
||||
continue
|
||||
cpu = real = 0.0
|
||||
n = 0
|
||||
for m in re.finditer(r"User=(\d+\.\d+)s Sys=(\d+\.\d+)s Real=(\d+\.\d+)s", open(path).read()):
|
||||
u, s, r = map(float, m.groups())
|
||||
cpu += u + s
|
||||
real += r
|
||||
n += 1
|
||||
rows.append((v, n, cpu, real, cpu / real if real else 0))
|
||||
with open(os.path.join(out, "gc-cpu-ratio.txt"), "w") as f:
|
||||
f.write("# CPU the collector received during its pauses, from -Xlog:gc+cpu (sum over every collection)\n\n")
|
||||
f.write("%-2s %12s %12s %12s %10s\n" % ("", "collections", "user+sys", "real", "cpu/real"))
|
||||
for v, n, cpu, real, ratio in rows:
|
||||
f.write("%-2s %12d %11.2fs %11.2fs %10.2f\n" % (v, n, cpu, real, ratio))
|
||||
print(open(os.path.join(out, "gc-cpu-ratio.txt")).read())
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run the in-cluster load generator as a pod and wait for it; prints its report.
|
||||
# [METHOD=POST] ./scripts/loadgen.sh <name> <url> <concurrency-or-schedule> <seconds>
|
||||
set -uo pipefail
|
||||
source "$(dirname "$0")/env.sh"
|
||||
name="$1" url="$2" conc="$3" secs="$4"
|
||||
kubectl -n "$NS" delete pod "$name" --ignore-not-found --wait=true > /dev/null
|
||||
kubectl -n "$NS" run "$name" --image="$IMAGE" --image-pull-policy=Never --restart=Never --labels=role=loadgen \
|
||||
--overrides='{"spec":{"containers":[{"name":"'"$name"'","image":"'"$IMAGE"'","imagePullPolicy":"Never","resources":{"requests":{"cpu":"100m","memory":"256Mi"},"limits":{"memory":"512Mi"}},"command":["java","-Dloadgen.method='"${METHOD:-GET}"'","-cp","application.jar","com.ankurm.k8s.loadgen.LoadGen","'"$url"'","'"$conc"'","'"$secs"'"]}]}}' > /dev/null
|
||||
Reference in New Issue
Block a user