#!/usr/bin/env bash
# A downstream outage, with the downstream health indicator in the LIVENESS group and then in the
# READINESS group, and in neither (the default). Outage from t=0 to t=60, then 50 s of recovery.
# -> docs/output/probes-*.txt
set -uo pipefail
source "$(dirname "$0")/env.sh"

watch_outage() { # label, file
  local label="$1" file="$2"
  kubectl -n "$NS" rollout status deploy/orders --timeout=180s > /dev/null
  sleep 5
  {
    echo "# $label"
    echo "# health groups: liveness=$(kubectl -n "$NS" get deploy orders -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="MANAGEMENT_ENDPOINT_HEALTH_GROUP_LIVENESS_INCLUDE")].value}') readiness=$(kubectl -n "$NS" get deploy orders -o jsonpath='{.spec.template.spec.containers[0].env[?(@.name=="MANAGEMENT_ENDPOINT_HEALTH_GROUP_READINESS_INCLUDE")].value}')"
    echo "# t=0: kubectl scale deploy/downstream --replicas=0    t=60: back to 1"
    echo
    kubectl -n "$NS" scale deploy/downstream --replicas=0 > /dev/null
    local start; start=$(date +%s)
    local restored=no
    for _ in $(seq 1 22); do
      local t=$(( $(date +%s) - start ))
      if [ "$t" -ge 60 ] && [ "$restored" = no ]; then
        kubectl -n "$NS" scale deploy/downstream --replicas=1 > /dev/null; restored=yes
        echo "-------- downstream restored --------"
      fi
      local pods; pods=$(kubectl -n "$NS" get pods -l app=orders --no-headers \
        -o custom-columns=N:.metadata.name,R:.status.containerStatuses[0].ready,C:.status.containerStatuses[0].restartCount,S:.status.containerStatuses[0].state \
        | awk '{st=$4; sub(/map\[/,"",st); sub(/:.*/,"",st); printf "%s ready=%s restarts=%s %s | ", substr($1,length($1)-4), $2, $3, st}')
      local eps; eps=$(kubectl -n "$NS" get endpointslices -l kubernetes.io/service-name=orders -o jsonpath='{range .items[*].endpoints[*]}{.conditions.ready}{" "}{end}' | tr ' ' '\n' | grep -c true)
      local work; work=$(kubectl -n "$NS" exec client -- wget -q -T 2 -O - http://orders:8080/work?ms=1 2>/dev/null | grep -c pod || true)
      printf 't=%3ds  %s serving endpoints=%s  GET /work via Service: %s\n' "$t" "$pods" "$eps" "$([ "$work" = 1 ] && echo ok || echo FAIL)"
      sleep 5
    done
    echo
    echo "# Events (probe failures and kills) for orders pods:"
    kubectl -n "$NS" get events --field-selector involvedObject.kind=Pod --sort-by=.lastTimestamp \
      | grep -E '(Unhealthy|Killing|BackOff).*orders-' | awk '{ $1=""; $4=""; print }' | sed 's/^ //' | sort | uniq -c | sort -rn | head -8
  } | tee "$OUT/$file"
}

kubectl apply -f "$MODULE_DIR/k8s/client.yaml" > /dev/null
kubectl -n "$NS" wait --for=condition=Ready pod/client --timeout=60s > /dev/null
kubectl -n "$NS" scale deploy/downstream --replicas=1 > /dev/null
kubectl -n "$NS" delete events --all > /dev/null 2>&1
kubectl -n "$NS" set env deploy/orders MANAGEMENT_ENDPOINT_HEALTH_GROUP_LIVENESS_INCLUDE=livenessState,downstream MANAGEMENT_ENDPOINT_HEALTH_GROUP_READINESS_INCLUDE- > /dev/null
watch_outage "downstream health indicator in the LIVENESS group" probes-liveness-includes-downstream.txt
kubectl -n "$NS" rollout status deploy/downstream --timeout=60s > /dev/null

kubectl -n "$NS" delete events --all > /dev/null 2>&1
kubectl -n "$NS" set env deploy/orders MANAGEMENT_ENDPOINT_HEALTH_GROUP_LIVENESS_INCLUDE- MANAGEMENT_ENDPOINT_HEALTH_GROUP_READINESS_INCLUDE=readinessState,downstream > /dev/null
watch_outage "downstream health indicator in the READINESS group" probes-readiness-includes-downstream.txt
kubectl -n "$NS" rollout status deploy/downstream --timeout=60s > /dev/null

kubectl -n "$NS" delete events --all > /dev/null 2>&1
kubectl -n "$NS" set env deploy/orders MANAGEMENT_ENDPOINT_HEALTH_GROUP_READINESS_INCLUDE- > /dev/null
watch_outage "downstream health indicator in NEITHER group (Spring Boot's default probe groups)" probes-default-groups.txt