Add kafka4-migration module: Spring Kafka 3.3 to 4.1 migration on Boot 3.5 to 4.1

Co-Authored-By: Claude Sonnet 5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01Gth99spB1CzQmBFofXmU4Z

kafka4-migration/.gitignore

@@ -0,0 +1 @@
+target/

kafka4-migration/README.md

@@ -0,0 +1,13 @@
+# kafka4-migration: Spring Kafka 3 to 4 on Spring Boot 3.5 to 4.1
+
+Companion code for the ankurm.com post "Spring Kafka 3 to 4 Migration Guide".
+
+| Folder | What it is |
+|---|---|
+| `legacy/` | Producer, `@KafkaListener` and an `@EmbeddedKafka` test on Boot 3.5.16 / Spring Kafka 3.3.16 / kafka-clients 3.9.2 (JDK 21), using the Jackson 2 `JsonSerializer` |
+| `migrated/` | The same app on Boot 4.1.1 / Spring Kafka 4.1.1 / kafka-clients 4.2.1 (JDK 25), using `spring-boot-starter-kafka` and `JacksonJsonSerializer` |
+| `scripts/run-all.sh` | Builds the intermediate stages from `legacy/` and regenerates every file in `output/` |
+| `output/` | Transcripts for each stage |
+
+Spring Kafka 4.1.1 is the latest GA; 4.2.0-M1 and 4.2.0-M2 are milestones and are not tested here.
+No external broker is needed: the tests use `@EmbeddedKafka`. Tested on 28 September 2026.

kafka4-migration/legacy/pom.xml

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.springframework.boot</groupId>
+    <artifactId>spring-boot-starter-parent</artifactId>
+    <version>3.5.16</version>
+    <relativePath/>
+  </parent>
+  <groupId>com.ankurm</groupId>
+  <artifactId>kafka-legacy</artifactId>
+  <version>1.0.0</version>
+  <properties><java.version>21</java.version></properties>
+  <dependencies>
+    <dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter</artifactId></dependency>
+    <dependency><groupId>org.springframework.kafka</groupId><artifactId>spring-kafka</artifactId></dependency>
+    <dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-json</artifactId></dependency>
+    <dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-test</artifactId><scope>test</scope></dependency>
+    <dependency><groupId>org.springframework.kafka</groupId><artifactId>spring-kafka-test</artifactId><scope>test</scope></dependency>
+  </dependencies>
+</project>

kafka4-migration/legacy/src/main/java/com/ankurm/kafka/App.java

@@ -0,0 +1,9 @@
+package com.ankurm.kafka;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class App {
+    public static void main(String[] args) { SpringApplication.run(App.class, args); }
+}

kafka4-migration/legacy/src/main/java/com/ankurm/kafka/OrderEvent.java

@@ -0,0 +1,3 @@
+package com.ankurm.kafka;
+
+public record OrderEvent(String id, int qty) { }

kafka4-migration/legacy/src/main/java/com/ankurm/kafka/OrderListener.java

@@ -0,0 +1,18 @@
+package com.ankurm.kafka;
+
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.List;
+import org.springframework.kafka.annotation.KafkaListener;
+import org.springframework.kafka.support.KafkaHeaders;
+import org.springframework.messaging.handler.annotation.Header;
+import org.springframework.stereotype.Component;
+
+@Component
+public class OrderListener {
+    public final List<String> received = new CopyOnWriteArrayList<>();
+
+    @KafkaListener(topics = "orders")
+    void on(OrderEvent event, @Header(KafkaHeaders.RECEIVED_KEY) String key) {
+        received.add("key=" + key + " id=" + event.id() + " qty=" + event.qty());
+    }
+}

kafka4-migration/legacy/src/main/java/com/ankurm/kafka/SerdeProbe.java

@@ -0,0 +1,14 @@
+package com.ankurm.kafka;
+
+import java.nio.charset.StandardCharsets;
+import org.springframework.kafka.support.serializer.JsonSerializer;
+
+/** Serialises one event with the JSON serializer, using only the runtime classpath (no test jars). */
+public class SerdeProbe {
+    public static void main(String[] args) {
+        try (JsonSerializer<OrderEvent> s = new JsonSerializer<>()) {
+            byte[] bytes = s.serialize("orders", new OrderEvent("o-1", 3));
+            System.out.println("SERIALIZED " + new String(bytes, StandardCharsets.UTF_8));
+        }
+    }
+}

kafka4-migration/legacy/src/main/resources/application.properties

@@ -0,0 +1,8 @@
+spring.kafka.consumer.group-id=orders-group
+spring.kafka.consumer.auto-offset-reset=earliest
+spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer
+spring.kafka.producer.value-serializer=org.springframework.kafka.support.serializer.JsonSerializer
+spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer
+spring.kafka.consumer.value-deserializer=org.springframework.kafka.support.serializer.JsonDeserializer
+spring.kafka.consumer.properties.spring.json.trusted.packages=com.ankurm.kafka
+spring.kafka.consumer.properties.spring.json.value.default.type=com.ankurm.kafka.OrderEvent

kafka4-migration/legacy/src/test/java/com/ankurm/kafka/OrderFlowTest.java

@@ -0,0 +1,27 @@
+package com.ankurm.kafka;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.kafka.test.EmbeddedKafkaBroker;
+import org.springframework.kafka.test.context.EmbeddedKafka;
+
+@SpringBootTest(properties = "spring.kafka.bootstrap-servers=${spring.embedded.kafka.brokers}")
+@EmbeddedKafka(topics = "orders", partitions = 1)
+class OrderFlowTest {
+    @Autowired KafkaTemplate<String, OrderEvent> template;
+    @Autowired OrderListener listener;
+    @Autowired EmbeddedKafkaBroker broker;
+
+    @Test
+    void sendAndReceive() {
+        template.send("orders", "k1", new OrderEvent("o-1", 3));
+        await().untilAsserted(() -> assertThat(listener.received).hasSize(1));
+        System.out.println("RESULT " + listener.received.get(0));
+        System.out.println("BROKERS " + broker.getBrokersAsString().replaceAll(":\\d+", ":PORT"));
+    }
+}

kafka4-migration/migrated/pom.xml

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.springframework.boot</groupId>
+    <artifactId>spring-boot-starter-parent</artifactId>
+    <version>4.1.1</version>
+    <relativePath/>
+  </parent>
+  <groupId>com.ankurm</groupId>
+  <artifactId>kafka-migrated</artifactId>
+  <version>1.0.0</version>
+  <properties><java.version>25</java.version></properties>
+  <dependencies>
+    <dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter</artifactId></dependency>
+    <dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-kafka</artifactId></dependency>
+    <dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-jackson</artifactId></dependency>
+    <dependency><groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-test</artifactId><scope>test</scope></dependency>
+    <dependency><groupId>org.springframework.kafka</groupId><artifactId>spring-kafka-test</artifactId><scope>test</scope></dependency>
+  </dependencies>
+</project>

kafka4-migration/migrated/src/main/java/com/ankurm/kafka/App.java

@@ -0,0 +1,9 @@
+package com.ankurm.kafka;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class App {
+    public static void main(String[] args) { SpringApplication.run(App.class, args); }
+}

kafka4-migration/migrated/src/main/java/com/ankurm/kafka/OrderEvent.java

@@ -0,0 +1,3 @@
+package com.ankurm.kafka;
+
+public record OrderEvent(String id, int qty) { }

kafka4-migration/migrated/src/main/java/com/ankurm/kafka/OrderListener.java

@@ -0,0 +1,18 @@
+package com.ankurm.kafka;
+
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.List;
+import org.springframework.kafka.annotation.KafkaListener;
+import org.springframework.kafka.support.KafkaHeaders;
+import org.springframework.messaging.handler.annotation.Header;
+import org.springframework.stereotype.Component;
+
+@Component
+public class OrderListener {
+    public final List<String> received = new CopyOnWriteArrayList<>();
+
+    @KafkaListener(topics = "orders")
+    void on(OrderEvent event, @Header(KafkaHeaders.RECEIVED_KEY) String key) {
+        received.add("key=" + key + " id=" + event.id() + " qty=" + event.qty());
+    }
+}

kafka4-migration/migrated/src/main/java/com/ankurm/kafka/SerdeProbe.java

@@ -0,0 +1,14 @@
+package com.ankurm.kafka;
+
+import java.nio.charset.StandardCharsets;
+import org.springframework.kafka.support.serializer.JacksonJsonSerializer;
+
+/** Serialises one event with the JSON serializer, using only the runtime classpath (no test jars). */
+public class SerdeProbe {
+    public static void main(String[] args) {
+        try (JacksonJsonSerializer<OrderEvent> s = new JacksonJsonSerializer<>()) {
+            byte[] bytes = s.serialize("orders", new OrderEvent("o-1", 3));
+            System.out.println("SERIALIZED " + new String(bytes, StandardCharsets.UTF_8));
+        }
+    }
+}

kafka4-migration/migrated/src/main/resources/application.properties

@@ -0,0 +1,8 @@
+spring.kafka.consumer.group-id=orders-group
+spring.kafka.consumer.auto-offset-reset=earliest
+spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer
+spring.kafka.producer.value-serializer=org.springframework.kafka.support.serializer.JacksonJsonSerializer
+spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer
+spring.kafka.consumer.value-deserializer=org.springframework.kafka.support.serializer.JacksonJsonDeserializer
+spring.kafka.consumer.properties.spring.json.trusted.packages=com.ankurm.kafka
+spring.kafka.consumer.properties.spring.json.value.default.type=com.ankurm.kafka.OrderEvent

kafka4-migration/migrated/src/test/java/com/ankurm/kafka/OrderFlowTest.java

@@ -0,0 +1,27 @@
+package com.ankurm.kafka;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.kafka.core.KafkaTemplate;
+import org.springframework.kafka.test.EmbeddedKafkaBroker;
+import org.springframework.kafka.test.context.EmbeddedKafka;
+
+@SpringBootTest(properties = "spring.kafka.bootstrap-servers=${spring.embedded.kafka.brokers}")
+@EmbeddedKafka(topics = "orders", partitions = 1)
+class OrderFlowTest {
+    @Autowired KafkaTemplate<String, OrderEvent> template;
+    @Autowired OrderListener listener;
+    @Autowired EmbeddedKafkaBroker broker;
+
+    @Test
+    void sendAndReceive() {
+        template.send("orders", "k1", new OrderEvent("o-1", 3));
+        await().untilAsserted(() -> assertThat(listener.received).hasSize(1));
+        System.out.println("RESULT " + listener.received.get(0));
+        System.out.println("BROKERS " + broker.getBrokersAsString().replaceAll(":\\d+", ":PORT"));
+    }
+}

kafka4-migration/output/01-legacy-kafka3.txt

@@ -0,0 +1,12 @@
+$ mvn test   # Boot 3.5.16, JDK 21
+RESULT key=k1 id=o-1 qty=3
+BROKERS 127.0.0.1:PORT
+[INFO] Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 7.853 s -- in com.ankurm.kafka.OrderFlowTest
+[INFO] Tests run: 1, Failures: 0, Errors: 0, Skipped: 0
+[INFO] BUILD SUCCESS
+
+org.apache.kafka:kafka-clients:jar:3.9.2:compile
+org.springframework.kafka:spring-kafka:jar:3.3.16:compile
+
+$ java -cp target/classes:<runtime classpath> SerdeProbe
+SERIALIZED {"id":"o-1","qty":3}

kafka4-migration/output/02-boot4-no-starter.txt

@@ -0,0 +1,5 @@
+$ mvn test   # only the Boot version changed to 4.1.1
+[ERROR] Tests run: 1, Failures: 0, Errors: 1, Skipped: 0, Time elapsed: 5.728 s <<< FAILURE! -- in com.ankurm.kafka.OrderFlowTest
+[ERROR]   OrderFlowTest.sendAndReceive ? UnsatisfiedDependency Error creating bean with name 'com.ankurm.kafka.OrderFlowTest': Unsatisfied dependency expressed through field 'template': No qualifying bean of type 'org.springframework.kafka.core.KafkaTemplate<java.lang.String, com.ankurm.kafka.OrderEvent>' available: expected at 
+[ERROR] Tests run: 1, Failures: 0, Errors: 1, Skipped: 0
+[INFO] BUILD FAILURE

kafka4-migration/output/03-starter-added-tests-pass.txt

@@ -0,0 +1,15 @@
+$ mvn test   # spring-kafka replaced by spring-boot-starter-kafka; nothing else changed
+RESULT key=k1 id=o-1 qty=3
+BROKERS localhost:PORT
+[INFO] Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 9.853 s -- in com.ankurm.kafka.OrderFlowTest
+[INFO] Tests run: 1, Failures: 0, Errors: 0, Skipped: 0
+[INFO] BUILD SUCCESS
+
+org.apache.kafka:kafka-clients:jar:4.2.1:compile
+org.springframework.boot:spring-boot-kafka:jar:4.1.1:compile
+org.springframework.kafka:spring-kafka:jar:4.1.1:compile
+
+$ mvn dependency:list | grep jackson   # runtime scope, then test scope
+runtime: tools.jackson.core:jackson-core:jar:3.1.5:compile
+runtime: tools.jackson.core:jackson-databind:jar:3.1.5:compile
+test-only: com.fasterxml.jackson.core:jackson-databind:jar:2.21.5:test

kafka4-migration/output/04-runtime-classpath-has-no-jackson2.txt

@@ -0,0 +1,3 @@
+$ compile and run SerdeProbe on the runtime classpath only (JsonSerializer from Spring Kafka 4.1.1)
+[ERROR]   class file for com.fasterxml.jackson.core.type.TypeReference not found
+[ERROR] s2/src/main/java/com/ankurm/kafka/SerdeProbe.java:[9,45] cannot access com.fasterxml.jackson.core.type.TypeReference

kafka4-migration/output/05-migrated.txt

@@ -0,0 +1,13 @@
+$ mvn test   # final migrated project (JacksonJsonSerializer, spring-boot-starter-jackson)
+RESULT key=k1 id=o-1 qty=3
+BROKERS localhost:PORT
+[INFO] Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 8.478 s -- in com.ankurm.kafka.OrderFlowTest
+[INFO] Tests run: 1, Failures: 0, Errors: 0, Skipped: 0
+[INFO] BUILD SUCCESS
+
+org.apache.kafka:kafka-clients:jar:4.2.1:compile
+org.springframework.boot:spring-boot-kafka:jar:4.1.1:compile
+org.springframework.kafka:spring-kafka:jar:4.1.1:compile
+
+$ java -cp target/classes:<runtime classpath> SerdeProbe
+SERIALIZED {"id":"o-1","qty":3}

kafka4-migration/output/06-serializer-classes.txt

@@ -0,0 +1,2 @@
+$ jar tf spring-kafka-4.1.1.jar | grep support/serializer/(Json|Jackson)
+JacksonJsonDeserializer.class JacksonJsonSerde.class JacksonJsonSerializer.class JacksonJsonTypeResolver.class JsonDeserializer.class JsonSerde.class JsonSerializer.class JsonTypeResolver.class 

kafka4-migration/scripts/run-all.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Regenerates output/*.txt. Needs Maven, JDK 21 (J21), JDK 25 (J25) and Maven Central access.
+set -u
+HERE=$(cd "$(dirname "$0")/.." && pwd); OUT=$HERE/output; W=${WORK:-/tmp/k4-work}
+J21=${J21:-/usr/lib/jvm/java-21-openjdk-amd64}; J25=${J25:-/tmp/tools/jdk-25.0.4.1+1}; MVN=${MVN:-mvn}
+mkdir -p "$OUT"; rm -rf "$W"; mkdir -p "$W"
+mv_() { local jh=$1 d=$2; shift 2; (cd "$d" && JAVA_HOME=$jh $MVN -B "$@" 2>&1 | grep -v JAVA_TOOL_OPTIONS | sed "s#$W/##g"); }
+keep='^(RESULT|BROKERS)|Tests run:.*Fail|BUILD|ERROR.*(cannot access|No qualifying)'
+stage() { rm -rf "$W/$1"; cp -r "$HERE/$2" "$W/$1"; rm -rf "$W/$1/target"; }
+probe() { # probe <javahome> <dir>: compile and run SerdeProbe on the RUNTIME classpath only
+  local jh=$1 d=$2
+  (cd "$d" && JAVA_HOME=$jh $MVN -B -q compile dependency:build-classpath -Dmdep.includeScope=runtime -Dmdep.outputFile="$W/cp.txt" 2>&1 | grep -v JAVA_TOOL_OPTIONS | grep -E 'ERROR.*(cannot access|class file)' | sed "s#$W/##g" | sort -u
+   [ -f "$W/cp.txt" ] && JAVA_HOME=$jh $jh/bin/java -cp target/classes:$(cat "$W/cp.txt") com.ankurm.kafka.SerdeProbe 2>&1 | grep -v JAVA_TOOL_OPTIONS | head -3); rm -f "$W/cp.txt"; }
+versions() { (cd "$2" && JAVA_HOME=$1 $MVN -B dependency:list 2>&1 | grep -E 'org.springframework.kafka:spring-kafka:|org.springframework.boot:spring-boot-kafka:|org.apache.kafka:kafka-clients:jar:[0-9]' | sed 's/^\[INFO\] *//; s/ -- module.*//' | sort -u); }
+
+stage s0 legacy
+{ echo '$ mvn test   # Boot 3.5.16, JDK 21'; mv_ "$J21" "$W/s0" test | grep -E "$keep"
+  echo; versions "$J21" "$W/s0"
+  echo; echo '$ java -cp target/classes:<runtime classpath> SerdeProbe'; probe "$J21" "$W/s0"; } > "$OUT/01-legacy-kafka3.txt"
+
+stage s1 legacy; rm "$W/s1/src/main/java/com/ankurm/kafka/SerdeProbe.java"; sed -i 's/3.5.16/4.1.1/; s/<java.version>21/<java.version>25/' "$W/s1/pom.xml"
+{ echo '$ mvn test   # only the Boot version changed to 4.1.1'; mv_ "$J25" "$W/s1" test | grep -E "$keep" | cut -c1-330; } > "$OUT/02-boot4-no-starter.txt"
+
+cp -r "$W/s1" "$W/s2"; rm -rf "$W/s2/target"
+sed -i 's#<groupId>org.springframework.kafka</groupId><artifactId>spring-kafka</artifactId>#<groupId>org.springframework.boot</groupId><artifactId>spring-boot-starter-kafka</artifactId>#' "$W/s2/pom.xml"
+{ echo '$ mvn test   # spring-kafka replaced by spring-boot-starter-kafka; nothing else changed'; mv_ "$J25" "$W/s2" test | grep -E "$keep"
+  echo; versions "$J25" "$W/s2"
+  echo; echo '$ mvn dependency:list | grep jackson   # runtime scope, then test scope'
+  (cd "$W/s2" && JAVA_HOME=$J25 $MVN -B dependency:list -DincludeScope=runtime 2>&1 | grep -E 'jackson-(core|databind)' | sed 's/^\[INFO\] *//; s/ -- module.*//' | sort -u | sed 's/^/runtime: /'
+   JAVA_HOME=$J25 $MVN -B dependency:list -DexcludeScope=runtime 2>&1 | grep -E 'com.fasterxml.jackson.core:jackson-databind' | sed 's/^\[INFO\] *//; s/ -- module.*//' | sort -u | sed 's/^/test-only: /'); } > "$OUT/03-starter-added-tests-pass.txt"
+
+cp "$HERE/legacy/src/main/java/com/ankurm/kafka/SerdeProbe.java" "$W/s2/src/main/java/com/ankurm/kafka/"
+{ echo '$ compile and run SerdeProbe on the runtime classpath only (JsonSerializer from Spring Kafka 4.1.1)'; probe "$J25" "$W/s2"; } > "$OUT/04-runtime-classpath-has-no-jackson2.txt"
+
+stage m migrated
+{ echo '$ mvn test   # final migrated project (JacksonJsonSerializer, spring-boot-starter-jackson)'; mv_ "$J25" "$W/m" test | grep -E "$keep"
+  echo; versions "$J25" "$W/m"
+  echo; echo '$ java -cp target/classes:<runtime classpath> SerdeProbe'; probe "$J25" "$W/m"; } > "$OUT/05-migrated.txt"
+
+# 06 API facts from the Spring Kafka 4.1.1 jar
+J=$HOME/.m2/repository/org/springframework/kafka/spring-kafka/4.1.1/spring-kafka-4.1.1.jar
+{ echo '$ jar tf spring-kafka-4.1.1.jar | grep support/serializer/(Json|Jackson)'
+  env -u JAVA_TOOL_OPTIONS $J25/bin/jar tf "$J" | grep -E 'support/serializer/(Json|Jackson)' | sed 's#.*/##' | sort | tr '\n' ' '; echo; } > "$OUT/06-serializer-classes.txt"
+echo done