framework-benchmarks: Spring Boot 4.1.1 vs Quarkus 3.40.1 vs Micronaut 5.2.2, JVM + native, benchmarked
This commit is contained in:
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
measure.py - launches one framework-benchmarks app, times cold startup, and samples RSS
|
||||
right after the app answers its first successful request.
|
||||
|
||||
No external dependencies: stdlib only, so the measurement script itself is auditable
|
||||
without installing anything. See docs/output/ for the real transcripts this script produced.
|
||||
|
||||
Usage:
|
||||
measure.py <label> <port> <ready-path> <runs> -- <command...>
|
||||
|
||||
Prints one JSON line per run to stdout, plus a final "summary" line. The first run is
|
||||
discarded (cold filesystem cache) exactly as the file-cache warm-up convention that
|
||||
independent Java framework benchmarks (e.g. gillius.org's Java 25 framework comparison)
|
||||
use, and the summary averages the remaining runs.
|
||||
"""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import signal
|
||||
import os
|
||||
|
||||
|
||||
def wait_ready(url, timeout_s=90):
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=0.5) as resp:
|
||||
if resp.status == 200:
|
||||
return True
|
||||
except (urllib.error.URLError, ConnectionError, OSError, TimeoutError):
|
||||
pass
|
||||
time.sleep(0.01)
|
||||
return False
|
||||
|
||||
|
||||
def rss_kb(pid):
|
||||
try:
|
||||
with open(f"/proc/{pid}/status") as f:
|
||||
for line in f:
|
||||
if line.startswith("VmRSS:"):
|
||||
return int(line.split()[1])
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def run_once(label, port, ready_path, cmd, cwd):
|
||||
url = f"http://127.0.0.1:{port}{ready_path}"
|
||||
start = time.monotonic()
|
||||
proc = subprocess.Popen(
|
||||
cmd, cwd=cwd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
start_new_session=True,
|
||||
)
|
||||
ok = wait_ready(url)
|
||||
elapsed = time.monotonic() - start
|
||||
rss = rss_kb(proc.pid) if ok else None
|
||||
# one more request after a short settle, to confirm the app is actually serving,
|
||||
# not just that the socket accepted a connection
|
||||
served_ok = False
|
||||
if ok:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=2) as resp:
|
||||
served_ok = resp.status == 200
|
||||
except Exception:
|
||||
served_ok = False
|
||||
# terminate the whole process group (session leader), not just the Popen pid -
|
||||
# several of these apps fork a child JVM/launcher
|
||||
try:
|
||||
os.killpg(proc.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
proc.wait(timeout=10)
|
||||
return {
|
||||
"label": label,
|
||||
"ready": ok,
|
||||
"served_ok": served_ok,
|
||||
"startup_s": round(elapsed, 3) if ok else None,
|
||||
"rss_kb": rss,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
args = sys.argv[1:]
|
||||
sep = args.index("--")
|
||||
label, port, ready_path, runs = args[0], int(args[1]), args[2], int(args[3])
|
||||
cmd = args[sep + 1:]
|
||||
cwd = os.getcwd()
|
||||
|
||||
results = []
|
||||
for i in range(runs):
|
||||
r = run_once(label, port, ready_path, cmd, cwd)
|
||||
r["run"] = i + 1
|
||||
r["discarded"] = (i == 0)
|
||||
print(json.dumps(r))
|
||||
sys.stdout.flush()
|
||||
results.append(r)
|
||||
time.sleep(1.5)
|
||||
|
||||
kept = [r for r in results if not r["discarded"] and r["ready"]]
|
||||
if kept:
|
||||
avg_startup = sum(r["startup_s"] for r in kept) / len(kept)
|
||||
avg_rss = sum(r["rss_kb"] for r in kept if r["rss_kb"]) / len(kept)
|
||||
summary = {
|
||||
"label": label,
|
||||
"summary": True,
|
||||
"runs_kept": len(kept),
|
||||
"avg_startup_s": round(avg_startup, 3),
|
||||
"avg_rss_kb": round(avg_rss, 1),
|
||||
"avg_rss_mb": round(avg_rss / 1024, 1),
|
||||
}
|
||||
else:
|
||||
summary = {"label": label, "summary": True, "error": "no successful runs"}
|
||||
print(json.dumps(summary))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env bash
|
||||
# Regenerates every file in docs/output/ from a clean build. Run from the repo root.
|
||||
#
|
||||
# Requires: GraalVM for JDK 25 on PATH (JAVA_HOME pointing at it), Maven, Python 3.
|
||||
# Builds JVM jars first, benchmarks them, then builds native images (slow - budget
|
||||
# 20-30 minutes total on a 2-vCPU machine) and benchmarks those too.
|
||||
#
|
||||
# This is exactly the sequence used to produce the committed docs/output/ files -
|
||||
# re-run it yourself and your numbers should land in the same ballpark, modulo your
|
||||
# own hardware. See README.md for the one GraalVM build-arg fix the Micronaut module
|
||||
# needed and why.
|
||||
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
echo "== 1/4: building JVM artifacts =="
|
||||
(cd spring-boot-app && mvn -q -B clean package -DskipTests)
|
||||
(cd quarkus-app && mvn -q -B clean package -DskipTests)
|
||||
(cd micronaut-app && mvn -q -B clean package -DskipTests)
|
||||
|
||||
echo "== 2/4: JVM startup + memory (4 runs each, first discarded) =="
|
||||
{
|
||||
echo "=== Spring Boot JVM startup+memory (4 runs) ==="
|
||||
python3 scripts/measure.py "spring-boot-jvm" 8081 /widgets 4 -- java -jar spring-boot-app/target/spring-boot-app.jar
|
||||
echo "=== Quarkus JVM startup+memory (4 runs) ==="
|
||||
python3 scripts/measure.py "quarkus-jvm" 8082 /widgets 4 -- java -jar quarkus-app/target/quarkus-app/quarkus-run.jar
|
||||
echo "=== Micronaut JVM startup+memory (4 runs) ==="
|
||||
python3 scripts/measure.py "micronaut-jvm" 8083 /widgets 4 -- java -jar micronaut-app/target/micronaut-app.jar
|
||||
} | tee docs/output/01-startup-memory-jvm.txt
|
||||
|
||||
echo "== 3/4: JVM throughput (15s @ concurrency 20, one app at a time) =="
|
||||
run_throughput() {
|
||||
local label="$1" port="$2" ready_path="$3"; shift 3
|
||||
"$@" > "/tmp/${label}.log" 2>&1 < /dev/null &
|
||||
local pid=$!
|
||||
disown
|
||||
for i in $(seq 1 60); do
|
||||
code=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:${port}${ready_path}" --max-time 2 || true)
|
||||
[ "$code" = "200" ] && break
|
||||
sleep 0.2
|
||||
done
|
||||
python3 scripts/throughput.py "$label" localhost "$port" "$ready_path" 15 20
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
sleep 1
|
||||
}
|
||||
{
|
||||
echo "=== Spring Boot JVM throughput ==="
|
||||
run_throughput spring-boot-jvm 8081 /widgets java -jar spring-boot-app/target/spring-boot-app.jar
|
||||
echo "=== Quarkus JVM throughput ==="
|
||||
run_throughput quarkus-jvm 8082 /widgets java -jar quarkus-app/target/quarkus-app/quarkus-run.jar
|
||||
echo "=== Micronaut JVM throughput ==="
|
||||
run_throughput micronaut-jvm 8083 /widgets java -jar micronaut-app/target/micronaut-app.jar
|
||||
} | tee docs/output/02-throughput-jvm.txt
|
||||
|
||||
echo "== 4/4: native builds + benchmarks (slow) =="
|
||||
(cd quarkus-app && mvn -q -B -Pnative clean package -DskipTests && mvn -q -B package -DskipTests)
|
||||
(cd micronaut-app && mvn -q -B -Pnative clean package -DskipTests && \
|
||||
mvn -q -B -Pnative package org.graalvm.buildtools:native-maven-plugin:compile -DskipTests)
|
||||
(cd spring-boot-app && mvn -q -B -Pnative clean package -DskipTests && mvn -q -B -Pnative native:compile)
|
||||
|
||||
{
|
||||
echo "=== Spring Boot NATIVE startup+memory (4 runs) ==="
|
||||
python3 scripts/measure.py "spring-boot-native" 8081 /widgets 4 -- ./spring-boot-app/target/spring-boot-app
|
||||
echo "=== Quarkus NATIVE startup+memory (4 runs) ==="
|
||||
python3 scripts/measure.py "quarkus-native" 8082 /widgets 4 -- ./quarkus-app/target/quarkus-app-runner
|
||||
echo "=== Micronaut NATIVE startup+memory (4 runs) ==="
|
||||
python3 scripts/measure.py "micronaut-native" 8083 /widgets 4 -- ./micronaut-app/target/micronaut-app
|
||||
} | tee docs/output/03-startup-memory-native.txt
|
||||
|
||||
{
|
||||
echo "=== Spring Boot NATIVE throughput ==="
|
||||
run_throughput spring-boot-native 8081 /widgets ./spring-boot-app/target/spring-boot-app
|
||||
echo "=== Quarkus NATIVE throughput ==="
|
||||
run_throughput quarkus-native 8082 /widgets ./quarkus-app/target/quarkus-app-runner
|
||||
echo "=== Micronaut NATIVE throughput ==="
|
||||
run_throughput micronaut-native 8083 /widgets ./micronaut-app/target/micronaut-app
|
||||
} | tee docs/output/04-throughput-native.txt
|
||||
|
||||
echo "Done. See docs/output/05-summary.txt for the consolidated table (written by hand from the runs above - update it if your numbers differ materially)."
|
||||
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
throughput.py - a minimal, dependency-free concurrent HTTP load generator.
|
||||
|
||||
This repo intentionally does not shell out to wrk/hey/ab (neither was available in the
|
||||
sandbox this repo was built in, and a stdlib script means the load generator itself needs
|
||||
no separate install or version callout). Each worker thread keeps one persistent
|
||||
HTTP/1.1 connection (http.client, keep-alive) and fires GET requests back-to-back for a
|
||||
fixed wall-clock duration; the main thread joins all workers and aggregates.
|
||||
|
||||
Usage:
|
||||
throughput.py <label> <host> <port> <path> <duration_s> <concurrency>
|
||||
|
||||
Prints one JSON line: requests, errors, requests_per_sec, latency_ms mean/p50/p95/p99.
|
||||
"""
|
||||
import http.client
|
||||
import json
|
||||
import statistics
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
|
||||
def worker(host, port, path, stop_at, latencies, counters, idx):
|
||||
conn = http.client.HTTPConnection(host, port, timeout=5)
|
||||
local_ok = 0
|
||||
local_err = 0
|
||||
local_latencies = []
|
||||
while time.monotonic() < stop_at:
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
conn.request("GET", path)
|
||||
resp = conn.getresponse()
|
||||
resp.read()
|
||||
if resp.status == 200:
|
||||
local_ok += 1
|
||||
else:
|
||||
local_err += 1
|
||||
except Exception:
|
||||
local_err += 1
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
conn = http.client.HTTPConnection(host, port, timeout=5)
|
||||
local_latencies.append((time.monotonic() - t0) * 1000.0)
|
||||
counters[idx] = (local_ok, local_err)
|
||||
latencies[idx] = local_latencies
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def main():
|
||||
label, host, port, path, duration_s, concurrency = sys.argv[1:7]
|
||||
port = int(port)
|
||||
duration_s = float(duration_s)
|
||||
concurrency = int(concurrency)
|
||||
|
||||
counters = [None] * concurrency
|
||||
latencies = [None] * concurrency
|
||||
stop_at = time.monotonic() + duration_s
|
||||
|
||||
threads = [
|
||||
threading.Thread(target=worker, args=(host, port, path, stop_at, latencies, counters, i))
|
||||
for i in range(concurrency)
|
||||
]
|
||||
wall_start = time.monotonic()
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
wall_elapsed = time.monotonic() - wall_start
|
||||
|
||||
total_ok = sum(c[0] for c in counters)
|
||||
total_err = sum(c[1] for c in counters)
|
||||
all_latencies = [x for sub in latencies for x in sub]
|
||||
all_latencies.sort()
|
||||
|
||||
def pct(p):
|
||||
if not all_latencies:
|
||||
return None
|
||||
k = min(len(all_latencies) - 1, int(len(all_latencies) * p))
|
||||
return round(all_latencies[k], 2)
|
||||
|
||||
result = {
|
||||
"label": label,
|
||||
"path": path,
|
||||
"concurrency": concurrency,
|
||||
"duration_s": round(wall_elapsed, 2),
|
||||
"requests_ok": total_ok,
|
||||
"requests_err": total_err,
|
||||
"requests_per_sec": round(total_ok / wall_elapsed, 1) if wall_elapsed > 0 else None,
|
||||
"latency_ms_mean": round(statistics.mean(all_latencies), 2) if all_latencies else None,
|
||||
"latency_ms_p50": pct(0.50),
|
||||
"latency_ms_p95": pct(0.95),
|
||||
"latency_ms_p99": pct(0.99),
|
||||
}
|
||||
print(json.dumps(result, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user