Add kubernetes-deployment: probes, shutdown, JVM ergonomics, HPA
Companion code for "Deploying Spring Boot 4 on Kubernetes: Probes, Graceful Shutdown, Limits and JVM Ergonomics". A dependency outage under three probe-group setups, a rolling restart under load four ways (three runs each), the JVM's ergonomic choices for nine pod shapes, one GC-heavy load under five CPU limits with throttling counters, and an HPA driven by a Micrometer gauge through prometheus-adapter. Measured on k3s v1.36.4. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01C3TETMrqVUWeFkNtz3Jbo3
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,4 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: demo
|
||||
@@ -0,0 +1,13 @@
|
||||
# A shell inside the cluster for the scripts to call Services from (the app image has none).
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: client
|
||||
namespace: demo
|
||||
spec:
|
||||
terminationGracePeriodSeconds: 1
|
||||
containers:
|
||||
- name: busybox
|
||||
image: busybox:1.37
|
||||
imagePullPolicy: Never
|
||||
command: ["sh", "-c", "trap 'exit 0' TERM; while true; do sleep 1; done"]
|
||||
@@ -0,0 +1,33 @@
|
||||
# A trivially small HTTP service the app's "downstream" health indicator calls. Scale it to zero to
|
||||
# simulate a dependency outage: kubectl -n demo scale deploy/downstream --replicas=0
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: downstream
|
||||
namespace: demo
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels: {app: downstream}
|
||||
template:
|
||||
metadata:
|
||||
labels: {app: downstream}
|
||||
spec:
|
||||
# busybox httpd runs as PID 1 and ignores SIGTERM, so without this a scale-to-zero leaves the
|
||||
# "outage" serving for the full 30 s grace period.
|
||||
terminationGracePeriodSeconds: 2
|
||||
containers:
|
||||
- name: httpd
|
||||
image: busybox:1.37
|
||||
imagePullPolicy: Never
|
||||
command: ["sh", "-c", "mkdir -p /www && echo ok > /www/index.html && exec httpd -f -p 8080 -h /www"]
|
||||
ports: [{containerPort: 8080}]
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: downstream
|
||||
namespace: demo
|
||||
spec:
|
||||
selector: {app: downstream}
|
||||
ports: [{port: 8080, targetPort: 8080}]
|
||||
@@ -0,0 +1,38 @@
|
||||
# One replica used only by the CPU-limit / GC experiment. scripts/demo-gc-throttling.sh patches
|
||||
# resources and JAVA_TOOL_OPTIONS per variant.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: gc-lab
|
||||
namespace: demo
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels: {app: gc-lab}
|
||||
template:
|
||||
metadata:
|
||||
labels: {app: gc-lab}
|
||||
spec:
|
||||
containers:
|
||||
- name: app
|
||||
image: sbd/k8s-demo:1
|
||||
imagePullPolicy: Never
|
||||
ports: [{name: http, containerPort: 8080}]
|
||||
env:
|
||||
- name: JAVA_TOOL_OPTIONS
|
||||
value: ""
|
||||
resources:
|
||||
requests: {cpu: 100m, memory: 256Mi}
|
||||
limits: {cpu: "1", memory: 1Gi}
|
||||
readinessProbe:
|
||||
httpGet: {path: /actuator/health/readiness, port: http}
|
||||
periodSeconds: 2
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: gc-lab
|
||||
namespace: demo
|
||||
spec:
|
||||
selector: {app: gc-lab}
|
||||
ports: [{name: http, port: 8080, targetPort: http}]
|
||||
@@ -0,0 +1,20 @@
|
||||
# Scale orders on in-flight requests per pod, not CPU. Target: 5 in flight on average.
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata: {name: orders, namespace: demo}
|
||||
spec:
|
||||
scaleTargetRef: {apiVersion: apps/v1, kind: Deployment, name: orders}
|
||||
minReplicas: 1
|
||||
maxReplicas: 4
|
||||
metrics:
|
||||
- type: Pods
|
||||
pods:
|
||||
metric: {name: app_inflight_requests}
|
||||
target: {type: AverageValue, averageValue: "5"}
|
||||
behavior:
|
||||
scaleUp:
|
||||
stabilizationWindowSeconds: 0
|
||||
scaleDown:
|
||||
# The default is 300 s. Shortened so the demo shows a scale-down inside a few minutes;
|
||||
# keep the default (or longer) in production.
|
||||
stabilizationWindowSeconds: 30
|
||||
@@ -0,0 +1,112 @@
|
||||
# prometheus-adapter v0.12.0 serving custom.metrics.k8s.io from one rule: the Micrometer gauge
|
||||
# app.inflight.requests, which Prometheus stores as app_inflight_requests.
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata: {name: prometheus-adapter, namespace: monitoring}
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata: {name: prometheus-adapter}
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: [namespaces, pods, services, nodes]
|
||||
verbs: [get, list, watch]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata: {name: prometheus-adapter}
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: prometheus-adapter}
|
||||
subjects: [{kind: ServiceAccount, name: prometheus-adapter, namespace: monitoring}]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata: {name: prometheus-adapter-auth-delegator}
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: system:auth-delegator}
|
||||
subjects: [{kind: ServiceAccount, name: prometheus-adapter, namespace: monitoring}]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata: {name: prometheus-adapter-auth-reader, namespace: kube-system}
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: extension-apiserver-authentication-reader}
|
||||
subjects: [{kind: ServiceAccount, name: prometheus-adapter, namespace: monitoring}]
|
||||
---
|
||||
# The HPA controller reads custom metrics as this service account.
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata: {name: custom-metrics-reader}
|
||||
rules:
|
||||
- apiGroups: [custom.metrics.k8s.io]
|
||||
resources: ["*"]
|
||||
verbs: [get, list]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata: {name: hpa-custom-metrics-reader}
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: custom-metrics-reader}
|
||||
subjects: [{kind: ServiceAccount, name: horizontal-pod-autoscaler, namespace: kube-system}]
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata: {name: prometheus-adapter, namespace: monitoring}
|
||||
data:
|
||||
config.yaml: |
|
||||
rules:
|
||||
# Micrometer "app.inflight.requests" -> Prometheus "app_inflight_requests" (dots become
|
||||
# underscores; a gauge gets no suffix). The series carries namespace/pod labels from the
|
||||
# scrape config, which is how the adapter attributes it to a pod.
|
||||
- seriesQuery: 'app_inflight_requests{namespace!="",pod!=""}'
|
||||
resources:
|
||||
overrides:
|
||||
namespace: {resource: namespace}
|
||||
pod: {resource: pod}
|
||||
name:
|
||||
matches: "^app_inflight_requests$"
|
||||
as: "app_inflight_requests"
|
||||
metricsQuery: 'avg_over_time(<<.Series>>{<<.LabelMatchers>>}[30s])'
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata: {name: prometheus-adapter, namespace: monitoring}
|
||||
spec:
|
||||
replicas: 1
|
||||
selector: {matchLabels: {app: prometheus-adapter}}
|
||||
template:
|
||||
metadata: {labels: {app: prometheus-adapter}}
|
||||
spec:
|
||||
serviceAccountName: prometheus-adapter
|
||||
containers:
|
||||
- name: adapter
|
||||
image: registry.k8s.io/prometheus-adapter/prometheus-adapter:v0.12.0
|
||||
imagePullPolicy: Never
|
||||
args:
|
||||
- --prometheus-url=http://prometheus.monitoring.svc:9090/
|
||||
- --metrics-relist-interval=15s
|
||||
- --config=/etc/adapter/config.yaml
|
||||
- --secure-port=6443
|
||||
- --cert-dir=/tmp/cert
|
||||
ports: [{containerPort: 6443}]
|
||||
resources: {requests: {cpu: 50m, memory: 64Mi}, limits: {memory: 256Mi}}
|
||||
volumeMounts:
|
||||
- {name: config, mountPath: /etc/adapter}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
volumes:
|
||||
- {name: config, configMap: {name: prometheus-adapter}}
|
||||
- {name: tmp, emptyDir: {}}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata: {name: prometheus-adapter, namespace: monitoring}
|
||||
spec:
|
||||
selector: {app: prometheus-adapter}
|
||||
ports: [{port: 443, targetPort: 6443}]
|
||||
---
|
||||
apiVersion: apiregistration.k8s.io/v1
|
||||
kind: APIService
|
||||
metadata: {name: v1beta1.custom.metrics.k8s.io}
|
||||
spec:
|
||||
service: {name: prometheus-adapter, namespace: monitoring}
|
||||
group: custom.metrics.k8s.io
|
||||
version: v1beta1
|
||||
insecureSkipTLSVerify: true # the adapter generated a self-signed cert; use cert-manager in real clusters
|
||||
groupPriorityMinimum: 100
|
||||
versionPriority: 100
|
||||
@@ -0,0 +1,75 @@
|
||||
# Minimal Prometheus: scrapes every pod labelled app=orders on /actuator/prometheus every 5 s and
|
||||
# keeps the namespace and pod as labels - prometheus-adapter needs both to map a series to a pod.
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata: {name: monitoring}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata: {name: prometheus, namespace: monitoring}
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata: {name: prometheus-sd}
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: [pods, endpoints, services]
|
||||
verbs: [get, list, watch]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata: {name: prometheus-sd}
|
||||
roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: prometheus-sd}
|
||||
subjects: [{kind: ServiceAccount, name: prometheus, namespace: monitoring}]
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata: {name: prometheus, namespace: monitoring}
|
||||
data:
|
||||
prometheus.yml: |
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
scrape_configs:
|
||||
- job_name: orders
|
||||
metrics_path: /actuator/prometheus
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
namespaces: {names: [demo]}
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_pod_label_app]
|
||||
regex: orders
|
||||
action: keep
|
||||
- source_labels: [__meta_kubernetes_pod_container_port_number]
|
||||
regex: "8080"
|
||||
action: keep
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
target_label: namespace
|
||||
- source_labels: [__meta_kubernetes_pod_name]
|
||||
target_label: pod
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata: {name: prometheus, namespace: monitoring}
|
||||
spec:
|
||||
replicas: 1
|
||||
selector: {matchLabels: {app: prometheus}}
|
||||
template:
|
||||
metadata: {labels: {app: prometheus}}
|
||||
spec:
|
||||
serviceAccountName: prometheus
|
||||
containers:
|
||||
- name: prometheus
|
||||
image: quay.io/prometheus/prometheus:v3.14.0
|
||||
imagePullPolicy: Never
|
||||
args: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.retention.time=2h"]
|
||||
ports: [{containerPort: 9090}]
|
||||
resources: {requests: {cpu: 50m, memory: 128Mi}, limits: {memory: 512Mi}}
|
||||
volumeMounts: [{name: config, mountPath: /etc/prometheus}]
|
||||
volumes: [{name: config, configMap: {name: prometheus}}]
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata: {name: prometheus, namespace: monitoring}
|
||||
spec:
|
||||
selector: {app: prometheus}
|
||||
ports: [{port: 9090, targetPort: 9090}]
|
||||
@@ -0,0 +1,54 @@
|
||||
# The baseline Deployment every scenario starts from. Scenario scripts patch it with the specific
|
||||
# change under test (see scripts/*.sh), so each transcript differs from this file in one place.
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: orders
|
||||
namespace: demo
|
||||
spec:
|
||||
replicas: 2
|
||||
selector:
|
||||
matchLabels: {app: orders}
|
||||
template:
|
||||
metadata:
|
||||
labels: {app: orders}
|
||||
spec:
|
||||
terminationGracePeriodSeconds: 30
|
||||
containers:
|
||||
- name: app
|
||||
image: sbd/k8s-demo:1
|
||||
imagePullPolicy: Never
|
||||
ports: [{name: http, containerPort: 8080}]
|
||||
env:
|
||||
- name: DEMO_DOWNSTREAM_URL
|
||||
value: http://downstream:8080/
|
||||
- name: JAVA_TOOL_OPTIONS
|
||||
value: "-XX:MaxRAMPercentage=75"
|
||||
resources:
|
||||
requests: {cpu: 250m, memory: 512Mi}
|
||||
limits: {memory: 512Mi}
|
||||
startupProbe:
|
||||
httpGet: {path: /actuator/health/liveness, port: http}
|
||||
periodSeconds: 2
|
||||
failureThreshold: 60
|
||||
livenessProbe:
|
||||
httpGet: {path: /actuator/health/liveness, port: http}
|
||||
periodSeconds: 5
|
||||
failureThreshold: 3
|
||||
readinessProbe:
|
||||
httpGet: {path: /actuator/health/readiness, port: http}
|
||||
periodSeconds: 2
|
||||
failureThreshold: 1
|
||||
lifecycle:
|
||||
preStop:
|
||||
sleep: {seconds: 5}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: orders
|
||||
namespace: demo
|
||||
labels: {app: orders}
|
||||
spec:
|
||||
selector: {app: orders}
|
||||
ports: [{name: http, port: 8080, targetPort: http}]
|
||||
Reference in New Issue
Block a user