Add observability: real OTLP metrics/traces to grafana/otel-lgtm, Docker Compose auto-wiring, and a dual-version (Boot 4.0 vs 4.1) proof of the new OTEL_* env var support
Companion module for the rewritten ankurm.com Prometheus/Grafana monitoring post. Verified against a real running grafana/otel-lgtm container (not mocked): 8 real requests produce a real orders_placed_total metric queried back from the bundled Prometheus-compatible API with zero management.otlp.* properties, auto-wired entirely by Boot's Docker Compose service-connection detection. Two real findings surfaced along the way and documented rather than smoothed over: @Observed silently produces no span without an explicit ObservedAspect bean (AspectJ weaving alone is not sufficient, despite Micrometer Tracing being active), and OTEL_EXPORTER_OTLP_ENDPOINT already worked on Boot 4.0 via Micrometer's own OtlpConfig fallback -- what's actually new in 4.1 is the rest of the standard OTEL_* surface (verified with OTEL_METRIC_EXPORT_INTERVAL against identical source compiled on both Boot 4.0.8 and 4.1.1). Also fixes the root README's module table, which was missing a row for resilience4j-circuit-breaker (added in a previous commit but never indexed here).
This commit is contained in:
Claude
Executable
+78
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
# Regenerates every file under docs/output/ from a real run against a real grafana/otel-lgtm
|
||||
# container. Needs Docker, JDK 25 and several minutes -- Tempo's search index lags live traffic
|
||||
# by roughly a minute in this bundled all-in-one image, so this script waits for it rather than
|
||||
# racing it. Leaves the LGTM container running afterwards; `docker compose down` to stop it.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
OUT="docs/output"
|
||||
mkdir -p "$OUT"
|
||||
|
||||
mvn -q -DskipTests package
|
||||
|
||||
echo "--- phase 1: default sampling (0.10), Docker Compose auto-wiring + real metrics ---"
|
||||
mvn -q spring-boot:run > /tmp/obs-run-default.log 2>&1 &
|
||||
APP_PID=$!
|
||||
until curl -sf http://localhost:8080/actuator/health > /dev/null 2>&1; do sleep 2; done
|
||||
|
||||
{
|
||||
echo '$ mvn spring-boot:run'
|
||||
grep -E "DockerComposeLifecycleManager|DockerCli|Publishing metrics|Started Observability" /tmp/obs-run-default.log
|
||||
echo
|
||||
echo '$ docker ps'
|
||||
docker ps --filter name=lgtm
|
||||
} > "$OUT/00-docker-compose-auto-wiring.txt"
|
||||
|
||||
for i in $(seq 1 8); do curl -s -X POST "http://localhost:8080/orders/$i" > /dev/null; done
|
||||
sleep 65
|
||||
{
|
||||
echo "8 requests sent to /orders/{1..8}. Queried from LGTM's own Prometheus-compatible API:"
|
||||
echo '$ curl -s "http://localhost:9090/api/v1/query?query=orders_placed_total"'
|
||||
curl -s "http://localhost:9090/api/v1/query?query=orders_placed_total" | python3 -m json.tool
|
||||
} > "$OUT/01-metrics-and-traces-in-lgtm.txt"
|
||||
|
||||
BASELINE_TRACES=$(curl -s "http://localhost:3200/api/search?limit=50" | python3 -c "import json,sys;print(len(json.load(sys.stdin)['traces']))")
|
||||
for i in $(seq 101 115); do curl -s -X POST "http://localhost:8080/orders/$i" > /dev/null; done
|
||||
sleep 90
|
||||
AFTER_TRACES=$(curl -s "http://localhost:3200/api/search?limit=50" | python3 -c "import json,sys;print(len(json.load(sys.stdin)['traces']))")
|
||||
{
|
||||
echo "Default sampling probability (0.10): 15 requests sent, metric increases by 15,"
|
||||
echo "trace count increases by only $((AFTER_TRACES - BASELINE_TRACES)) (baseline=$BASELINE_TRACES, after=$AFTER_TRACES)."
|
||||
} > "$OUT/02-low-sampling-demo.txt"
|
||||
|
||||
kill "$APP_PID" 2>/dev/null || true
|
||||
wait "$APP_PID" 2>/dev/null || true
|
||||
|
||||
echo "--- phase 2: fulltrace profile (1.0 sampling), @Observed span proof ---"
|
||||
mvn -q -Dspring-boot.run.profiles=fulltrace spring-boot:run > /tmp/obs-run-fulltrace.log 2>&1 &
|
||||
APP_PID=$!
|
||||
until curl -sf http://localhost:8080/actuator/health > /dev/null 2>&1; do sleep 2; done
|
||||
|
||||
curl -s -X POST "http://localhost:8080/orders/777" > /dev/null
|
||||
sleep 90
|
||||
TRACE_ID=$(curl -s "http://localhost:3200/api/search?limit=5" | python3 -c "
|
||||
import json,sys
|
||||
d=json.load(sys.stdin)
|
||||
t=sorted(d['traces'], key=lambda x:int(x['startTimeUnixNano']), reverse=True)[0]
|
||||
print(t['traceID'])
|
||||
")
|
||||
{
|
||||
echo "Trace for the /orders/777 request just sent, fetched from Tempo:"
|
||||
curl -s "http://localhost:3200/api/traces/$TRACE_ID" | python3 -c "
|
||||
import json,sys
|
||||
d=json.load(sys.stdin)
|
||||
for b in d['batches']:
|
||||
for ss in b['scopeSpans']:
|
||||
for sp in ss['spans']:
|
||||
print('span:', sp['name'], '| kind:', sp['kind'], '| parent:', sp.get('parentSpanId','(root)'))
|
||||
"
|
||||
} > "$OUT/03-observed-needs-explicit-bean.txt"
|
||||
|
||||
kill "$APP_PID" 2>/dev/null || true
|
||||
wait "$APP_PID" 2>/dev/null || true
|
||||
docker compose down
|
||||
|
||||
echo "--- phase 3: standard OTEL_* env var support, Boot 4.0 vs 4.1 ---"
|
||||
./scripts/run-env-var-proof.sh > "$OUT/04-otel-env-vars-4.0-vs-4.1.txt" 2>&1
|
||||
|
||||
echo "Done. See $OUT/*.txt"
|
||||
Executable
+37
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
# Rebuilds and runs the env-var-proof/boot40 vs boot41 experiment described in
|
||||
# docs/03-otel-env-vars.md, and regenerates docs/output/04-otel-env-vars-4.0-vs-4.1.txt.
|
||||
# Requires JDK 25 and network access (each module resolves its own Boot parent POM).
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
ROOT="$(pwd)"
|
||||
|
||||
mkdir -p env-var-proof/stub-receiver/out
|
||||
javac env-var-proof/stub-receiver/StubOtlpReceiver.java -d env-var-proof/stub-receiver/out
|
||||
|
||||
(cd env-var-proof/boot40 && mvn -q -DskipTests package)
|
||||
(cd env-var-proof/boot41 && mvn -q -DskipTests package)
|
||||
|
||||
run_case() {
|
||||
local label="$1" jar="$2" port="$3"
|
||||
shift 3
|
||||
local log="/tmp/otel-proof-${label}.log"
|
||||
rm -f "$log"
|
||||
java -cp env-var-proof/stub-receiver/out StubOtlpReceiver "$port" "$log" &
|
||||
local receiver_pid=$!
|
||||
sleep 1
|
||||
echo "=== $label ==="
|
||||
env "$@" timeout 30 java -jar "$jar" | grep -E "Publishing metrics" || true
|
||||
kill "$receiver_pid" 2>/dev/null || true
|
||||
echo "--- receiver log ($label) ---"
|
||||
cat "$log"
|
||||
}
|
||||
|
||||
run_case "boot41-env-vars" "env-var-proof/boot41/target/otel-env-proof-41-1.0.0.jar" 24318 \
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:24318" OTEL_METRIC_EXPORT_INTERVAL=2000
|
||||
run_case "boot40-env-vars" "env-var-proof/boot40/target/otel-env-proof-40-1.0.0.jar" 24319 \
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:24319" OTEL_METRIC_EXPORT_INTERVAL=2000
|
||||
run_case "boot40-management-props" "env-var-proof/boot40/target/otel-env-proof-40-1.0.0.jar" 24320 \
|
||||
MANAGEMENT_OTLP_METRICS_EXPORT_URL="http://localhost:24320/v1/metrics" MANAGEMENT_OTLP_METRICS_EXPORT_STEP=2s
|
||||
|
||||
echo "Compare the three receiver logs above against docs/output/04-otel-env-vars-4.0-vs-4.1.txt"
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
# Starts the app on :8080. Boot's Docker Compose support auto-starts compose.yaml's
|
||||
# grafana/otel-lgtm container the first time this runs (needs Docker) and wires the OTLP
|
||||
# metrics/traces/logs exporters to it -- no management.otlp.* properties needed.
|
||||
# ./scripts/run.sh # default 0.10 trace sampling
|
||||
# ./scripts/run.sh fulltrace # 1.0 trace sampling -- see docs/03-otel-env-vars.md
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
PROFILE="${1:-}"
|
||||
if [ -n "$PROFILE" ]; then
|
||||
mvn -q -Dspring-boot.run.profiles="$PROFILE" spring-boot:run
|
||||
else
|
||||
mvn -q spring-boot:run
|
||||
fi
|
||||
Reference in New Issue
Block a user