#!/usr/bin/env bash
# HorizontalPodAutoscaler on a Micrometer gauge through Prometheus + prometheus-adapter.
# Load: 2 clients, then 30 from t=20 s to t=140 s, then none; each request holds for 500 ms.
# -> docs/output/hpa-custom-metric.txt, hpa-custom-metrics-api.txt
set -uo pipefail
source "$(dirname "$0")/env.sh"
H="$MODULE_DIR/k8s/hpa"
kubectl apply -f "$H/prometheus.yaml" -f "$H/prometheus-adapter.yaml" > /dev/null
kubectl -n monitoring rollout status deploy/prometheus --timeout=120s > /dev/null
kubectl -n monitoring rollout status deploy/prometheus-adapter --timeout=120s > /dev/null
kubectl -n "$NS" scale deploy/orders --replicas=1 > /dev/null
kubectl -n "$NS" rollout status deploy/orders --timeout=120s > /dev/null
API=/apis/custom.metrics.k8s.io/v1beta1/namespaces/demo/pods/%2A/app_inflight_requests
for _ in $(seq 1 40); do kubectl get --raw "$API" 2>/dev/null | grep -q '"items":\[{' && break; sleep 5; done
{
  echo "# What Micrometer exports (one pod):"
  echo "\$ curl <pod>:8080/actuator/prometheus | grep app_inflight"
  curl -s "$(kubectl -n "$NS" get pod -l app=orders -o jsonpath='{.items[0].status.podIP}'):8080/actuator/prometheus" | grep app_inflight
  echo
  echo "# What the HPA controller sees through the custom metrics API:"
  echo "\$ kubectl get --raw $API"
  kubectl get --raw "$API" | python3 -m json.tool
} > "$OUT/hpa-custom-metrics-api.txt"

kubectl apply -f "$H/hpa.yaml" > /dev/null
sleep 20
kubectl -n "$NS" delete events --field-selector involvedObject.kind=HorizontalPodAutoscaler > /dev/null 2>&1
"$MODULE_DIR/scripts/loadgen.sh" lg-hpa "http://orders:8080/work?ms=500" "0:2,20:30,140:0" 230
kubectl -n "$NS" wait --for=condition=Ready pod/lg-hpa --timeout=60s > /dev/null
start=$(date +%s)
{
  echo "# HPA orders: target app_inflight_requests averageValue 5, min 1, max 4; scaleDown stabilization 30 s"
  echo "# load: t=0 2 clients, t=20 30 clients, t=140 0 clients; GET /work?ms=500"
  echo
  printf '%-6s %-10s %-9s %-8s %s\n' "t" "clients" "metric" "desired" "ready pods"
  while [ $(( $(date +%s) - start )) -lt 235 ]; do
    t=$(( $(date +%s) - start ))
    clients=2; [ $t -ge 20 ] && clients=30; [ $t -ge 140 ] && clients=0
    cur=$(kubectl -n "$NS" get hpa orders -o jsonpath='{.status.currentMetrics[0].pods.current.averageValue}')
    des=$(kubectl -n "$NS" get hpa orders -o jsonpath='{.status.desiredReplicas}')
    ready=$(kubectl -n "$NS" get deploy orders -o jsonpath='{.status.readyReplicas}')
    printf '%-6s %-10s %-9s %-8s %s\n' "${t}s" "$clients" "${cur:-?}" "${des:-?}" "${ready:-0}"
    sleep 10
  done
  echo
  echo "# HPA events:"
  kubectl -n "$NS" get events --field-selector involvedObject.kind=HorizontalPodAutoscaler --sort-by=.lastTimestamp \
    | awk 'NR>1 {$1=""; $2=""; $4=""; print}' | sed 's/^ *//'
  echo
  echo "# Load generator summary:"
  kubectl -n "$NS" wait --for=jsonpath='{.status.phase}'=Succeeded pod/lg-hpa --timeout=60s > /dev/null
  kubectl -n "$NS" logs lg-hpa | grep -E '^TOTAL|^LATENCY'
} | tee "$OUT/hpa-custom-metric.txt"
kubectl -n "$NS" logs lg-hpa > "$OUT/hpa-load.txt"
kubectl -n "$NS" delete pod lg-hpa --wait=false > /dev/null
kubectl -n "$NS" delete hpa orders > /dev/null
kubectl -n "$NS" scale deploy/orders --replicas=2 > /dev/null