jmh-basics: JMH tutorial module (dead-code elimination, Blackhole, constant folding, warmup/forks, state scopes, -prof gc)

Eight benchmark/demo classes, a deterministic test suite, and 12 captured output files generated by scripts/run-all.sh on JDK 25.0.5 (Temurin) with JMH 1.37. Co-Authored-By: Claude Sonnet 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_017ic4b6Fj2JNt7dRev4pZPe
2026-11-08 18:31:11 +00:00
parent f6237d4117
commit db129fb1ce
27 changed files with 2018 additions and 0 deletions
@@ -0,0 +1,55 @@
+package com.ankurm.jmhbasics;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Every method below contains the source text {@code new Point(...)} or {@code new int[4]}.
+ * Whether an object is really allocated depends on whether it can escape - and only a profiler
+ * can tell you. Run with the built-in GC profiler:
+ *
+ *   java -jar target/benchmarks.jar AllocationBenchmark -prof gc
+ *
+ * Read the {@code gc.alloc.rate.norm} column: bytes allocated per benchmark call.
+ */
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 3, time = 1)
+@Fork(1)
+@State(Scope.Thread)
+public class AllocationBenchmark {
+
+    record Point(int x, int y) { }
+
+    int a = 3;
+    int b = 4;
+
+    /** The Point never leaves this method: escape analysis can replace it with two locals. */
+    @Benchmark
+    public int doesNotEscape() {
+        Point p = new Point(a, b);
+        return p.x() + p.y();
+    }
+
+    /** The Point is returned to JMH, so it escapes and must exist on the heap. */
+    @Benchmark
+    public Point escapes() {
+        return new Point(a, b);
+    }
+
+    /** An array that is returned: header + 4 ints, rounded up to 8-byte alignment. */
+    @Benchmark
+    public int[] arrayEscapes() {
+        return new int[4];
+    }
+}
@@ -0,0 +1,82 @@
+package com.ankurm.jmhbasics;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Eight benchmarks around Math.log and Math.pow. Only some of them measure what their name says.
+ *
+ *   baseline         empty method: the cost of JMH's own loop. Everything else is read against it.
+ *   deadCode         computes the log and throws it away     -> candidate for dead-code elimination
+ *   returned         computes the log and returns it         -> JMH consumes it for you
+ *   viaBlackhole     computes the log and hands it to Blackhole.consume
+ *   logOfConstant    log of a compile-time constant          -> NOT folded on this JDK (see the post)
+ *   powOfConstant    pow of a compile-time constant          -> folded away by the JIT
+ *   powOfFinalField  pow of a final instance field           -> ALSO folded (JMHSample_10 lists this as a mistake)
+ *   powOfField       pow of a plain mutable field            -> the honest, un-foldable version
+ *
+ * Run: java -jar target/benchmarks.jar DeadCodeBenchmark
+ */
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 5, time = 1)
+@Fork(2)
+@State(Scope.Thread)
+public class DeadCodeBenchmark {
+
+    static final double CONSTANT = Math.PI;
+
+    double field = Math.PI;               // plain mutable field: the JIT cannot assume its value
+    final double finalField = Math.PI;    // final: measured below to be folded exactly like a literal constant
+
+    @Benchmark
+    public void baseline() {
+        // intentionally empty
+    }
+
+    @Benchmark
+    public void deadCode() {
+        Math.log(field);                  // result discarded
+    }
+
+    @Benchmark
+    public double returned() {
+        return Math.log(field);           // JMH feeds the return value to a Blackhole
+    }
+
+    @Benchmark
+    public void viaBlackhole(Blackhole bh) {
+        bh.consume(Math.log(field));
+    }
+
+    @Benchmark
+    public double logOfConstant() {
+        return Math.log(CONSTANT);
+    }
+
+    @Benchmark
+    public double powOfConstant() {
+        return Math.pow(CONSTANT, 10);
+    }
+
+    @Benchmark
+    public double powOfFinalField() {
+        return Math.pow(finalField, 10);
+    }
+
+    @Benchmark
+    public double powOfField() {
+        return Math.pow(field, 10);
+    }
+}
@@ -0,0 +1,64 @@
+package com.ankurm.jmhbasics;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Why forks exist. The three benchmarks do the same work through one call site, {@code measure}.
+ *
+ *   measure_1_c1 and measure_3_c1 pass an {@link AddOne}; measure_2_c2 passes an {@link AddTwo}.
+ *
+ * Fork count is deliberately NOT set here - choose it on the command line:
+ *
+ *   java -jar target/benchmarks.jar ForkingBenchmark -f 0   # all three share one JVM
+ *   java -jar target/benchmarks.jar ForkingBenchmark -f 1   # each benchmark gets a fresh JVM
+ *
+ * With -f 0 the first benchmark teaches HotSpot "this call site only ever sees AddOne". The second
+ * benchmark then breaks that assumption, and the third inherits the damage.
+ */
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 3, time = 1)
+@State(Scope.Thread)
+public class ForkingBenchmark {
+
+    public interface Step { int apply(int x); }
+
+    public static final class AddOne implements Step {
+        @Override public int apply(int x) { return x + 1; }
+    }
+
+    public static final class AddTwo implements Step {
+        @Override public int apply(int x) { return x + 2; }
+    }
+
+    final Step one = new AddOne();
+    final Step two = new AddTwo();
+
+    /** The single shared call site whose type profile gets polluted. */
+    public int measure(Step step) {
+        int acc = 0;
+        for (int i = 0; i < 10; i++) {
+            acc = step.apply(acc);
+        }
+        return acc;
+    }
+
+    @Benchmark
+    public int measure_1_c1() { return measure(one); }
+
+    @Benchmark
+    public int measure_2_c2() { return measure(two); }
+
+    @Benchmark
+    public int measure_3_c1() { return measure(one); }
+}
@@ -0,0 +1,100 @@
+package com.ankurm.jmhbasics;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.options.TimeValue;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Counts how often each {@code @Setup}/{@code @TearDown} level fires.
+ *
+ * It runs JMH with forks(0) on purpose: the benchmark then runs inside this JVM, so plain static
+ * counters are visible here afterwards. With a normal forked run they would live in another JVM.
+ *
+ * Configuration: 1 warmup iteration + 2 measurement iterations, 50 ms each.
+ *
+ * Run: java -cp target/benchmarks.jar com.ankurm.jmhbasics.LifecycleDemo   (after mvn package)
+ */
+public class LifecycleDemo {
+
+    @State(Scope.Benchmark)
+    public static class Lifecycle {
+        static final AtomicLong trialSetup = new AtomicLong();
+        static final AtomicLong iterationSetup = new AtomicLong();
+        static final AtomicLong invocationSetup = new AtomicLong();
+        static final AtomicLong invocationTearDown = new AtomicLong();
+        static final AtomicLong iterationTearDown = new AtomicLong();
+        static final AtomicLong trialTearDown = new AtomicLong();
+        static final AtomicLong benchmarkCalls = new AtomicLong();
+
+        static void reset() {
+            trialSetup.set(0);
+            iterationSetup.set(0);
+            invocationSetup.set(0);
+            invocationTearDown.set(0);
+            iterationTearDown.set(0);
+            trialTearDown.set(0);
+            benchmarkCalls.set(0);
+        }
+
+        @Setup(Level.Trial)
+        public void setupTrial() { trialSetup.incrementAndGet(); }
+
+        @Setup(Level.Iteration)
+        public void setupIteration() { iterationSetup.incrementAndGet(); }
+
+        @Setup(Level.Invocation)
+        public void setupInvocation() { invocationSetup.incrementAndGet(); }
+
+        @TearDown(Level.Invocation)
+        public void tearDownInvocation() { invocationTearDown.incrementAndGet(); }
+
+        @TearDown(Level.Iteration)
+        public void tearDownIteration() { iterationTearDown.incrementAndGet(); }
+
+        @TearDown(Level.Trial)
+        public void tearDownTrial() { trialTearDown.incrementAndGet(); }
+    }
+
+    @Benchmark
+    public void work(Lifecycle state) {
+        Lifecycle.benchmarkCalls.incrementAndGet();
+    }
+
+    /** Runs the benchmark in-process and returns after JMH has finished. */
+    public static void run() throws RunnerException {
+        Lifecycle.reset();
+        Options opt = new OptionsBuilder()
+                .include(LifecycleDemo.class.getName() + ".work")
+                .forks(0)
+                .warmupIterations(1)
+                .warmupTime(TimeValue.milliseconds(50))
+                .measurementIterations(2)
+                .measurementTime(TimeValue.milliseconds(50))
+                .verbosity(org.openjdk.jmh.runner.options.VerboseMode.SILENT)
+                .build();
+        new Runner(opt).run();
+    }
+
+    public static void main(String[] args) throws RunnerException {
+        run();
+        System.out.println("Setup/TearDown counts for 1 warmup + 2 measurement iterations:");
+        System.out.println();
+        System.out.printf("@Setup(Level.Trial)          : %d%n", Lifecycle.trialSetup.get());
+        System.out.printf("@Setup(Level.Iteration)      : %d%n", Lifecycle.iterationSetup.get());
+        System.out.printf("@Setup(Level.Invocation)     : %d  (one per benchmark call)%n", Lifecycle.invocationSetup.get());
+        System.out.printf("benchmark method calls       : %d%n", Lifecycle.benchmarkCalls.get());
+        System.out.printf("@TearDown(Level.Invocation)  : %d%n", Lifecycle.invocationTearDown.get());
+        System.out.printf("@TearDown(Level.Iteration)   : %d%n", Lifecycle.iterationTearDown.get());
+        System.out.printf("@TearDown(Level.Trial)       : %d%n", Lifecycle.trialTearDown.get());
+    }
+}
@@ -0,0 +1,52 @@
+package com.ankurm.jmhbasics;
+
+/**
+ * The benchmark most people write first: a loop around System.nanoTime().
+ *
+ * It "works" in the sense that it prints numbers. The numbers are wrong in two independent ways,
+ * and this program shows both on one run:
+ *
+ *  1. The loop that discards its result can be removed by the JIT, so it reports a cost close to zero.
+ *  2. The same code reports a different cost every round, because the JIT is still compiling it.
+ *
+ * Run: java -cp target/classes com.ankurm.jmhbasics.NaiveTimingDemo
+ */
+public class NaiveTimingDemo {
+
+    static final int N = 20_000_000;
+
+    /** Result thrown away - nothing observable depends on the loop. */
+    static void discard() {
+        for (int i = 1; i <= N; i++) {
+            Math.log(i);
+        }
+    }
+
+    /** Result kept and printed - the loop must really run. */
+    static double keep() {
+        double sum = 0;
+        for (int i = 1; i <= N; i++) {
+            sum += Math.log(i);
+        }
+        return sum;
+    }
+
+    public static void main(String[] args) {
+        System.out.println("Naive System.nanoTime() timing, " + N + " iterations per round");
+        System.out.println();
+        System.out.printf("%-7s %-22s %-22s%n", "round", "discarded (ns/op)", "kept (ns/op)");
+        double sink = 0;
+        for (int round = 1; round <= 6; round++) {
+            long t0 = System.nanoTime();
+            discard();
+            long t1 = System.nanoTime();
+            sink += keep();
+            long t2 = System.nanoTime();
+            System.out.printf("%-7d %-22.3f %-22.3f%n", round,
+                    (t1 - t0) / (double) N, (t2 - t1) / (double) N);
+        }
+        System.out.println();
+        // Printing the sink is what stops the JIT removing keep() as well.
+        System.out.println("checksum (ignore): " + sink);
+    }
+}
@@ -0,0 +1,50 @@
+package com.ankurm.jmhbasics;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.Arrays;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * One benchmark, used for two demonstrations - neither fixes the iteration counts here:
+ *
+ *  1. Warmup trace:  java -jar target/benchmarks.jar SortBenchmark -f 1 -wi 10 -w 200ms -i 3 -r 1s
+ *     JMH prints every warmup iteration, so you can watch the time per call fall as the JIT works.
+ *
+ *  2. Benchmark modes in one run: the method carries three @BenchmarkMode values, so one run reports
+ *     throughput, average time and a sampled latency distribution with percentiles.
+ */
+@BenchmarkMode({Mode.Throughput, Mode.AverageTime, Mode.SampleTime})
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 5, time = 1)
+@State(Scope.Thread)
+public class SortBenchmark {
+
+    int[] original;
+
+    @Setup
+    public void fill() {
+        Random random = new Random(7);
+        original = new int[1000];
+        for (int i = 0; i < original.length; i++) {
+            original[i] = random.nextInt();
+        }
+    }
+
+    @Benchmark
+    public int[] sortCopy() {
+        int[] copy = original.clone();   // sorting in place would make every call after the first trivial
+        Arrays.sort(copy);
+        return copy;
+    }
+}
@@ -0,0 +1,55 @@
+package com.ankurm.jmhbasics;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Same operation, two state scopes, two benchmark threads.
+ *
+ *   Scope.Thread     every benchmark thread gets its own PerThread instance: no sharing, no contention
+ *   Scope.Benchmark  all benchmark threads share ONE Shared instance: they fight over one cache line
+ *
+ * The scope you pick decides whether you are measuring "increment a counter" or "increment a counter
+ * while another core is also incrementing it".
+ *
+ * Run: java -jar target/benchmarks.jar StateScopeBenchmark
+ */
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 5, time = 1)
+@Fork(2)
+@Threads(2)
+public class StateScopeBenchmark {
+
+    @State(Scope.Benchmark)
+    public static class Shared {
+        final AtomicLong counter = new AtomicLong();
+    }
+
+    @State(Scope.Thread)
+    public static class PerThread {
+        final AtomicLong counter = new AtomicLong();
+    }
+
+    @Benchmark
+    public long sharedCounter(Shared state) {
+        return state.counter.incrementAndGet();
+    }
+
+    @Benchmark
+    public long perThreadCounter(PerThread state) {
+        return state.counter.incrementAndGet();
+    }
+}
@@ -0,0 +1,54 @@
+package com.ankurm.jmhbasics;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * {@code @Param} runs the same benchmark at several input sizes, and {@code @Setup} builds the data
+ * outside the measured region. Without {@code @Param} you would copy-paste three benchmark methods.
+ *
+ * Run: java -jar target/benchmarks.jar SumBenchmark
+ */
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 3, time = 1)
+@Measurement(iterations = 5, time = 1)
+@Fork(1)
+@State(Scope.Thread)
+public class SumBenchmark {
+
+    @Param({"10", "1000", "100000"})
+    public int size;
+
+    int[] data;
+
+    @Setup
+    public void fill() {
+        Random random = new Random(42);   // fixed seed: the same data on every run
+        data = new int[size];
+        for (int i = 0; i < size; i++) {
+            data[i] = random.nextInt(100);
+        }
+    }
+
+    @Benchmark
+    public long sum() {
+        long total = 0;
+        for (int v : data) {
+            total += v;
+        }
+        return total;
+    }
+}
@@ -0,0 +1,64 @@
+package com.ankurm.jmhbasics;
+
+import org.junit.jupiter.api.Test;
+
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Pins the claims that are deterministic. Timing ratios are NOT asserted here: on a shared 2-vCPU
+ * host they are not stable enough to fail a build on. They live in output/ as captured runs.
+ */
+class JmhBasicsTest {
+
+    @Test
+    void annotationProcessorGeneratedBenchmarkList() throws Exception {
+        try (java.io.InputStream in = getClass().getResourceAsStream("/META-INF/BenchmarkList")) {
+            assertTrue(in != null, "annotation processor did not generate META-INF/BenchmarkList");
+            String list = new String(in.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8);
+            assertTrue(list.contains("com.ankurm.jmhbasics.jmh_generated.DeadCodeBenchmark_deadCode_jmhTest"));
+            assertTrue(list.contains("com.ankurm.jmhbasics.jmh_generated.SumBenchmark_sum_jmhTest"));
+            assertTrue(list.contains("com.ankurm.jmhbasics.jmh_generated.ForkingBenchmark_measure_2_c2_jmhTest"));
+            // 8 DeadCode + 1 Lifecycle + 3 Forking + 2 StateScope + 3 Allocation + 1 Sum
+            // + 3 Sort: one entry per @BenchmarkMode value, so SortBenchmark.sortCopy counts three times
+            assertEquals(21, list.lines().filter(l -> l.startsWith("JMH S")).count());
+        }
+    }
+
+    @Test
+    void lifecycleLevelsFireExactlyAsDocumented() throws Exception {
+        LifecycleDemo.run();
+        assertEquals(1, LifecycleDemo.Lifecycle.trialSetup.get(), "Trial setup: once per fork");
+        assertEquals(1, LifecycleDemo.Lifecycle.trialTearDown.get());
+        assertEquals(3, LifecycleDemo.Lifecycle.iterationSetup.get(), "Iteration setup: 1 warmup + 2 measurement");
+        assertEquals(3, LifecycleDemo.Lifecycle.iterationTearDown.get());
+        long calls = LifecycleDemo.Lifecycle.benchmarkCalls.get();
+        assertTrue(calls > 0);
+        assertEquals(calls, LifecycleDemo.Lifecycle.invocationSetup.get(), "Invocation setup: once per call");
+        assertEquals(calls, LifecycleDemo.Lifecycle.invocationTearDown.get());
+    }
+
+    @Test
+    void benchmarkBodiesComputeTheRightAnswers() {
+        SumBenchmark sum = new SumBenchmark();
+        sum.size = 1000;
+        sum.fill();
+        long expected = 0;
+        for (int v : sum.data) expected += v;
+        assertEquals(expected, sum.sum());
+
+        SortBenchmark sort = new SortBenchmark();
+        sort.fill();
+        int[] sorted = sort.sortCopy();
+        for (int i = 1; i < sorted.length; i++) assertTrue(sorted[i - 1] <= sorted[i]);
+        // The original must be untouched, otherwise later iterations would sort an already-sorted array.
+        boolean originalUnsorted = false;
+        for (int i = 1; i < sort.original.length; i++) if (sort.original[i - 1] > sort.original[i]) originalUnsorted = true;
+        assertTrue(originalUnsorted);
+
+        ForkingBenchmark f = new ForkingBenchmark();
+        assertEquals(10, f.measure_1_c1());
+        assertEquals(20, f.measure_2_c2());
+    }
+}