diff --git a/README.md b/README.md index 0e30b50..f7a6ada 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ facts, so the build fails when a claim stops being true. |---|---|---| | `embabel` | Embabel: Goal-Oriented AI Agents on the JVM | actions, goals, conditions and cost-based planning with GOAP, compared with plain Spring AI | | `a2a` | A2A Protocol in Java: Agents That Talk to Each Other | agent card, tasks, streaming, input-required and agent-to-agent calls with the A2A Java SDK | +| `jlama` | Local LLM Inference in Pure Java with Jlama | load and run a 4-bit TinyLlama inside the JVM on the Vector API, tokens per second, streaming, and the same measurement against Ollama | ## Versions (verified on Maven Central, 2026-10-11) @@ -15,6 +16,8 @@ facts, so the build fails when a claim stops being true. |---|---|---| | Embabel `embabel-agent-api` | 1.5.3 | brings Spring AI 2.0.1 and Spring Boot 4.1.1 transitively | | A2A Java SDK `io.github.a2asdk` | 1.0.0.Alpha3 | alpha; targets A2A protocol 1.0. The newest stable release, 0.3.3.Final, targets the 0.3 protocol and its API differs | +| Jlama `jlama-core` (`com.github.tjake`) | 0.8.4 | needs `--add-modules jdk.incubator.vector` at compile and run time | +| Model | `tjake/TinyLlama-1.1B-Chat-v1.0-Jlama-Q4` | 1.1 GB, downloaded by Jlama into `jlama/models/` on first run (git-ignored) | | JDK | 25 LTS | | | JUnit | 6.1.3 | | @@ -29,3 +32,17 @@ mvn test -pl a2a -am # one module No API key is needed. The A2A agents contain no model call, because the protocol is the subject. The Embabel tests use `ScriptedLlmOperations`, the scripted stand-in shipped inside `embabel-agent-api`, and the Spring AI comparison uses a small scripted `ChatModel`. Each module writes its transcripts to `/output/` when you run the tests. + +## The `jlama` module needs a model and takes minutes + +`mvn test -pl jlama` downloads the 1.1 GB model on the first run and then runs a few minutes of inference on the CPU. +The Ollama comparison is skipped unless you pass the address of a running Ollama that has a model named `tl`: + +```bash +# Modelfile: FROM ./tinyllama-1.1b-chat-v1.0.Q4_0.gguf (TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF) +# plus the TinyLlama chat TEMPLATE and PARAMETER stop "" +ollama create tl -f Modelfile +mvn test -pl jlama -Dollama.url=http://127.0.0.1:11434 +``` + +The committed numbers in `jlama/output/` come from one 2-vCPU cloud machine, not a laptop. Run it on yours and expect different figures. diff --git a/jlama/.gitignore b/jlama/.gitignore new file mode 100644 index 0000000..2bcdfd9 --- /dev/null +++ b/jlama/.gitignore @@ -0,0 +1 @@ +models/ diff --git a/jlama/output/01-load-and-ask.txt b/jlama/output/01-load-and-ask.txt new file mode 100644 index 0000000..2b77fe6 --- /dev/null +++ b/jlama/output/01-load-and-ask.txt @@ -0,0 +1,9 @@ +Machine: 2 CPUs, JDK 25.0.4.1+1-LTS, Vector API preferred float width 512 bits +Model: tjake/TinyLlama-1.1B-Chat-v1.0-Jlama-Q4, 1132 MB on disk +Load time: 665 ms; process resident memory 119 MB before, 175 MB after + +Question: In one sentence, what is a Java record? +Answer: A Java record is a data structure that allows for the creation of immutable, named, and indexed collections of objects. + +Prompt tokens: 26, generated tokens: 24 +Prompt time: 4838 ms, generation time: 2762 ms diff --git a/jlama/output/02-tokens-per-second.txt b/jlama/output/02-tokens-per-second.txt new file mode 100644 index 0000000..77c1c14 --- /dev/null +++ b/jlama/output/02-tokens-per-second.txt @@ -0,0 +1,11 @@ +Same question, 5 measured runs after 1 warm-up run, temperature 0, limit 64 tokens. + +run 1: prompt 26 tokens in 4902 ms (5.3 tok/s), generated 24 tokens in 3040 ms (7.9 tok/s) +run 2: prompt 26 tokens in 4762 ms (5.5 tok/s), generated 24 tokens in 3061 ms (7.8 tok/s) +run 3: prompt 26 tokens in 4527 ms (5.7 tok/s), generated 24 tokens in 2856 ms (8.4 tok/s) +run 4: prompt 26 tokens in 4892 ms (5.3 tok/s), generated 24 tokens in 3007 ms (8.0 tok/s) +run 5: prompt 26 tokens in 5203 ms (5.0 tok/s), generated 24 tokens in 2894 ms (8.3 tok/s) + +Generation tokens/s: min 7.8, median 8.0, max 8.4 +Prompt tokens/s: min 5.0, median 5.3, max 5.7 +Prompt + generated tokens in the longest run: 50 (limit was 64) diff --git a/jlama/output/03-streaming.txt b/jlama/output/03-streaming.txt new file mode 100644 index 0000000..5ecee4d --- /dev/null +++ b/jlama/output/03-streaming.txt @@ -0,0 +1,8 @@ +Callback receives one piece of text per generated token. + +Pieces received: 24, generated tokens reported: 24 +Time to first piece: 5230 ms (includes processing the 26-token prompt) +Time to last piece: 7997 ms; whole call 8114 ms + +First pieces: [[A], [ Java], [ record], [ is], [ a], [ data], [ structure], [ that]] +Joined: A Java record is a data structure that allows for the creation of immutable, named, and indexed collections of objects. diff --git a/jlama/output/04-temperature.txt b/jlama/output/04-temperature.txt new file mode 100644 index 0000000..5ad7832 --- /dev/null +++ b/jlama/output/04-temperature.txt @@ -0,0 +1,13 @@ +Temperature 0.0, asked twice: + 1: A Java record is a data structure that allows for the creation of immutable, named, and indexed collections of + 2: A Java record is a data structure that allows for the creation of immutable, named, and indexed collections of + identical: true + +Temperature 0.9, asked three times: + 1: In Java, a record is a type of data structure that stores and retrieves data in a logical, indexed + 2: A Java record is a type of nested class (or interface) that represents a collection of related data objects in + 3: A Java record is a data structure that contains multiple values and provides a way to represent multiple instances of the same + distinct answers: 3 + +Temperature 0.0, limit 64, asked six more times: 1 distinct answer(s) + 6x: A Java record is a data structure that allows for the creation of immutable, named, and indexed collections of objects. diff --git a/jlama/output/05-without-vector-module.txt b/jlama/output/05-without-vector-module.txt new file mode 100644 index 0000000..625f4ac --- /dev/null +++ b/jlama/output/05-without-vector-module.txt @@ -0,0 +1,5 @@ +Second JVM, same classpath, started WITHOUT --add-modules jdk.incubator.vector. +Exit code: 1 +Relevant lines: + Exception in thread "main" java.lang.RuntimeException: java.lang.reflect.InvocationTargetException + Caused by: java.lang.ClassNotFoundException: jdk.incubator.vector.FloatVector diff --git a/jlama/output/06-ollama-baseline.txt b/jlama/output/06-ollama-baseline.txt new file mode 100644 index 0000000..cddb654 --- /dev/null +++ b/jlama/output/06-ollama-baseline.txt @@ -0,0 +1,11 @@ +Ollama 0.40.3, model "tl" created from tinyllama-1.1b-chat-v1.0.Q4_0.gguf (TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF). +Same machine, same question, 1 warm-up run then 5 measured runs, temperature 0, limit 64 tokens. + +run 1: prompt 36 tokens (16 served from cache) in 190 ms, generated 40 tokens in 1196 ms (33.4 tok/s) +run 2: prompt 36 tokens (35 served from cache) in 38 ms, generated 40 tokens in 1172 ms (34.1 tok/s) +run 3: prompt 36 tokens (35 served from cache) in 28 ms, generated 40 tokens in 1187 ms (33.7 tok/s) +run 4: prompt 36 tokens (35 served from cache) in 28 ms, generated 40 tokens in 1169 ms (34.2 tok/s) +run 5: prompt 36 tokens (35 served from cache) in 29 ms, generated 40 tokens in 1215 ms (32.9 tok/s) + +Generation tokens/s: min 32.9, median 33.7, max 34.2 +Answer of the last run: A Java record is a data structure that combines the properties of a class and a struct. It is a type of data structure that allows for the creation of immutable objects with specific properties. diff --git a/jlama/output/07-token-limit.txt b/jlama/output/07-token-limit.txt new file mode 100644 index 0000000..04d2e75 --- /dev/null +++ b/jlama/output/07-token-limit.txt @@ -0,0 +1,4 @@ +The question is 26 prompt tokens long. + +Limit 30: generated 3 tokens (prompt + generated = 29) +Limit 20: IllegalArgumentException: Prompt exceeds max tokens diff --git a/jlama/output/08-dependencies.txt b/jlama/output/08-dependencies.txt new file mode 100644 index 0000000..c9a9148 --- /dev/null +++ b/jlama/output/08-dependencies.txt @@ -0,0 +1,39 @@ +mvn dependency:tree for the jlama module: +com.ankurm.agents:jlama:jar:1.0.0 +com.github.tjake:jlama-core:jar:0.8.4:compile +com.fasterxml.jackson.core:jackson-databind:jar:2.15.2:compile +com.fasterxml.jackson.core:jackson-annotations:jar:2.15.2:compile +com.fasterxml.jackson.core:jackson-core:jar:2.15.2:compile +com.google.guava:guava:jar:32.0.1-jre:compile +com.google.guava:failureaccess:jar:1.0.1:compile +com.google.guava:listenablefuture:jar:9999.0-empty-to-avoid-conflict-with-guava:compile +com.google.code.findbugs:jsr305:jar:3.0.2:compile +org.checkerframework:checker-qual:jar:3.33.0:compile +com.google.errorprone:error_prone_annotations:jar:2.18.0:compile +com.google.j2objc:j2objc-annotations:jar:2.8:compile +org.jctools:jctools-core:jar:4.0.1:compile +com.hubspot.jinjava:jinjava:jar:2.7.2:compile +org.javassist:javassist:jar:3.24.1-GA:compile +com.google.re2j:re2j:jar:1.2:compile +org.apache.commons:commons-lang3:jar:3.9:compile +commons-net:commons-net:jar:3.9.0:compile +com.googlecode.java-ipv6:java-ipv6:jar:0.17:compile +com.google.code.findbugs:annotations:jar:3.0.1:compile +com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:jar:2.14.0:compile +org.yaml:snakeyaml:jar:1.33:compile +ch.obermuhlner:big-math:jar:2.0.0:compile +net.fellbaum:jemoji:jar:1.4.1:compile +org.jspecify:jspecify:jar:0.3.0:compile +net.jafama:jafama:jar:2.3.2:compile +org.slf4j:slf4j-api:jar:2.0.7:compile +org.slf4j:slf4j-simple:jar:2.0.17:compile +org.junit.jupiter:junit-jupiter:jar:6.1.3:test +org.junit.jupiter:junit-jupiter-api:jar:6.1.3:test +org.opentest4j:opentest4j:jar:1.3.0:test +org.junit.platform:junit-platform-commons:jar:6.1.3:test +org.apiguardian:apiguardian-api:jar:1.1.2:test +org.junit.jupiter:junit-jupiter-params:jar:6.1.3:test +org.junit.jupiter:junit-jupiter-engine:jar:6.1.3:test +org.junit.platform:junit-platform-engine:jar:6.1.3:test +org.assertj:assertj-core:jar:3.27.6:test +net.bytebuddy:byte-buddy:jar:1.17.7:test diff --git a/jlama/pom.xml b/jlama/pom.xml new file mode 100644 index 0000000..1a01f7c --- /dev/null +++ b/jlama/pom.xml @@ -0,0 +1,68 @@ + + + 4.0.0 + + + com.ankurm.agents + java-ai-agents + 1.0.0 + + + jlama + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + --add-modules + jdk.incubator.vector + + + + + org.apache.maven.plugins + maven-surefire-plugin + + --add-modules jdk.incubator.vector + + ${project.basedir}/models + ${ollama.url} + + + + + + + + + + + + + com.github.tjake + jlama-core + ${jlama.version} + + + org.slf4j + slf4j-simple + ${slf4j.version} + + + org.junit.jupiter + junit-jupiter + test + + + org.assertj + assertj-core + ${assertj.version} + test + + + diff --git a/jlama/src/main/java/com/ankurm/agents/jlama/LoadOnly.java b/jlama/src/main/java/com/ankurm/agents/jlama/LoadOnly.java new file mode 100644 index 0000000..b174d36 --- /dev/null +++ b/jlama/src/main/java/com/ankurm/agents/jlama/LoadOnly.java @@ -0,0 +1,10 @@ +package com.ankurm.agents.jlama; + +/** Loads the model and nothing else. A test runs it in a second JVM that was started without the Vector API module. */ +public final class LoadOnly { + public static void main(String[] args) throws Exception { + try (LocalModel m = LocalModel.load(args[0])) { + System.out.println("loaded"); + } + } +} diff --git a/jlama/src/main/java/com/ankurm/agents/jlama/LocalModel.java b/jlama/src/main/java/com/ankurm/agents/jlama/LocalModel.java new file mode 100644 index 0000000..cd11673 --- /dev/null +++ b/jlama/src/main/java/com/ankurm/agents/jlama/LocalModel.java @@ -0,0 +1,66 @@ +package com.ankurm.agents.jlama; + +import com.github.tjake.jlama.model.AbstractModel; +import com.github.tjake.jlama.model.ModelSupport; +import com.github.tjake.jlama.safetensors.DType; +import com.github.tjake.jlama.safetensors.SafeTensorSupport; +import com.github.tjake.jlama.safetensors.prompt.PromptContext; + +import java.io.File; +import java.io.IOException; +import java.util.UUID; +import java.util.function.BiConsumer; + +/** + * A thin wrapper over Jlama's loader and generator. It exists so the tests read as "load, ask, measure" + * instead of as Jlama API calls. + */ +public final class LocalModel implements AutoCloseable { + + /** The model used throughout: TinyLlama 1.1B Chat, quantised to 4 bits by the Jlama author. */ + public static final String MODEL = "tjake/TinyLlama-1.1B-Chat-v1.0-Jlama-Q4"; + + /** What one call produced, with the timings Jlama reports. */ + public record Answer(String text, int promptTokens, int generatedTokens, long promptMs, long generateMs) { + public double generateTokensPerSecond() { + return generatedTokens * 1000.0 / generateMs; + } + + public double promptTokensPerSecond() { + return promptTokens * 1000.0 / promptMs; + } + } + + private final AbstractModel model; + + private LocalModel(AbstractModel model) { + this.model = model; + } + + /** Finds the model under modelsDir, downloading it from Hugging Face first if it is not there. */ + public static LocalModel load(String modelsDir) throws IOException { + File dir = SafeTensorSupport.maybeDownloadModel(modelsDir, MODEL); + return new LocalModel(ModelSupport.loadModel(dir, DType.F32, DType.I8)); + } + + public Answer ask(String question, int maxTokens, float temperature) { + return stream(question, maxTokens, temperature, (token, ms) -> { + }); + } + + /** onToken receives each piece of text as the model produces it. */ + public Answer stream(String question, int maxTokens, float temperature, BiConsumer onToken) { + PromptContext prompt = model.promptSupport().orElseThrow().builder().addUserMessage(question).build(); + var r = model.generate(UUID.randomUUID(), prompt, temperature, maxTokens, onToken); + return new Answer(r.responseText, r.promptTokens, r.generatedTokens, r.promptTimeMs, r.generateTimeMs); + } + + public AbstractModel raw() { + return model; + } + + @Override + public void close() { + model.close(); + } +} diff --git a/jlama/src/test/java/com/ankurm/agents/jlama/JlamaTest.java b/jlama/src/test/java/com/ankurm/agents/jlama/JlamaTest.java new file mode 100644 index 0000000..23dee5c --- /dev/null +++ b/jlama/src/test/java/com/ankurm/agents/jlama/JlamaTest.java @@ -0,0 +1,190 @@ +package com.ankurm.agents.jlama; + +import jdk.incubator.vector.FloatVector; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class JlamaTest { + + static final String MODELS = System.getProperty("jlama.models", "models"); + static final String QUESTION = "In one sentence, what is a Java record?"; + + LocalModel model; + long loadMs; + long rssBeforeKb; + long rssAfterKb; + + @BeforeAll + void load() throws Exception { + rssBeforeKb = rssKb(); + long t0 = System.nanoTime(); + model = LocalModel.load(MODELS); + loadMs = (System.nanoTime() - t0) / 1_000_000; + rssAfterKb = rssKb(); + } + + @AfterAll + void close() { + model.close(); + } + + private static long rssKb() throws Exception { + for (String l : Files.readAllLines(Path.of("/proc/self/status"))) { + if (l.startsWith("VmRSS:")) return Long.parseLong(l.replaceAll("\\D+", "")); + } + return -1; + } + + private static String f1(double d) { + return String.format(Locale.ROOT, "%.1f", d); + } + + // ------------------------------------------------------------------ 1. load and ask + + @Test + void loadsAModelInsideTheJvmAndAnswers() throws Exception { + LocalModel.Answer a = model.ask(QUESTION, 64, 0.0f); + long fileBytes = 0; + File dir = new File(MODELS, "tjake_TinyLlama-1.1B-Chat-v1.0-Jlama-Q4"); + for (File f : dir.listFiles()) fileBytes += f.length(); + + StringBuilder sb = new StringBuilder(); + sb.append("Machine: ").append(Runtime.getRuntime().availableProcessors()).append(" CPUs, JDK ").append(Runtime.version()) + .append(", Vector API preferred float width ").append(FloatVector.SPECIES_PREFERRED.vectorBitSize()).append(" bits\n"); + sb.append("Model: ").append(LocalModel.MODEL).append(", ").append(fileBytes / 1_000_000).append(" MB on disk\n"); + sb.append("Load time: ").append(loadMs).append(" ms; process resident memory ").append(rssBeforeKb / 1024).append(" MB before, ") + .append(rssAfterKb / 1024).append(" MB after\n\n"); + sb.append("Question: ").append(QUESTION).append('\n'); + sb.append("Answer: ").append(a.text()).append("\n\n"); + sb.append("Prompt tokens: ").append(a.promptTokens()).append(", generated tokens: ").append(a.generatedTokens()).append('\n'); + sb.append("Prompt time: ").append(a.promptMs()).append(" ms, generation time: ").append(a.generateMs()).append(" ms\n"); + Transcript.write("01-load-and-ask.txt", sb.toString()); + + assertThat(a.text()).isNotBlank(); + assertThat(a.generatedTokens()).isPositive(); + assertThat(FloatVector.SPECIES_PREFERRED.vectorBitSize()).isGreaterThanOrEqualTo(128); + } + + // ------------------------------------------------------------------ 2. tokens per second + + @Test + void measuresTokensPerSecond() { + model.ask(QUESTION, 64, 0.0f); // warm-up: the JIT and the weight pages are cold on the first call + List runs = new ArrayList<>(); + for (int i = 0; i < 5; i++) runs.add(model.ask(QUESTION, 64, 0.0f)); + + StringBuilder sb = new StringBuilder("Same question, 5 measured runs after 1 warm-up run, temperature 0, limit 64 tokens.\n\n"); + for (int i = 0; i < runs.size(); i++) { + LocalModel.Answer a = runs.get(i); + sb.append("run ").append(i + 1).append(": prompt ").append(a.promptTokens()).append(" tokens in ").append(a.promptMs()) + .append(" ms (").append(f1(a.promptTokensPerSecond())).append(" tok/s), generated ").append(a.generatedTokens()) + .append(" tokens in ").append(a.generateMs()).append(" ms (").append(f1(a.generateTokensPerSecond())).append(" tok/s)\n"); + } + double[] gen = runs.stream().mapToDouble(LocalModel.Answer::generateTokensPerSecond).sorted().toArray(); + double[] prm = runs.stream().mapToDouble(LocalModel.Answer::promptTokensPerSecond).sorted().toArray(); + sb.append("\nGeneration tokens/s: min ").append(f1(gen[0])).append(", median ").append(f1(gen[gen.length / 2])) + .append(", max ").append(f1(gen[gen.length - 1])).append('\n'); + sb.append("Prompt tokens/s: min ").append(f1(prm[0])).append(", median ").append(f1(prm[prm.length / 2])) + .append(", max ").append(f1(prm[prm.length - 1])).append('\n'); + sb.append("Prompt + generated tokens in the longest run: ").append(runs.stream().mapToInt(a -> a.promptTokens() + a.generatedTokens()).max().getAsInt()) + .append(" (limit was 64)\n"); + Transcript.write("02-tokens-per-second.txt", sb.toString()); + + assertThat(gen[0]).isPositive(); + assertThat(runs).allMatch(a -> a.promptTokens() + a.generatedTokens() <= 64); + } + + // ------------------------------------------------------------------ 3. streaming + + @Test + void streamsTokensAsTheyAreProduced() { + model.ask(QUESTION, 40, 0.0f); // warm-up, so the time to first piece is not dominated by a cold JIT + List pieces = new ArrayList<>(); + List at = new ArrayList<>(); + long t0 = System.nanoTime(); + LocalModel.Answer a = model.stream(QUESTION, 64, 0.0f, (tok, ms) -> { + pieces.add(tok); + at.add((System.nanoTime() - t0) / 1_000_000); + }); + long total = (System.nanoTime() - t0) / 1_000_000; + + StringBuilder sb = new StringBuilder("Callback receives one piece of text per generated token.\n\n"); + sb.append("Pieces received: ").append(pieces.size()).append(", generated tokens reported: ").append(a.generatedTokens()).append('\n'); + sb.append("Time to first piece: ").append(at.get(0)).append(" ms (includes processing the ").append(a.promptTokens()).append("-token prompt)\n"); + sb.append("Time to last piece: ").append(at.get(at.size() - 1)).append(" ms; whole call ").append(total).append(" ms\n\n"); + sb.append("First pieces: ").append(pieces.subList(0, Math.min(8, pieces.size())).stream().map(s -> "[" + s + "]").toList()).append('\n'); + sb.append("Joined: ").append(String.join("", pieces).strip()).append('\n'); + Transcript.write("03-streaming.txt", sb.toString()); + + assertThat(pieces).isNotEmpty(); + assertThat(String.join("", pieces).strip()).isEqualTo(a.text().strip()); + } + + // ------------------------------------------------------------------ 4. temperature + + @Test + void temperatureZeroIsRepeatable() { + String first = model.ask(QUESTION, 48, 0.0f).text(); + String second = model.ask(QUESTION, 48, 0.0f).text(); + List warm = new ArrayList<>(); + for (int i = 0; i < 3; i++) warm.add(model.ask(QUESTION, 48, 0.9f).text()); + + StringBuilder sb = new StringBuilder("Temperature 0.0, asked twice:\n 1: ").append(first).append("\n 2: ").append(second) + .append("\n identical: ").append(first.equals(second)).append("\n\nTemperature 0.9, asked three times:\n"); + for (int i = 0; i < warm.size(); i++) sb.append(" ").append(i + 1).append(": ").append(warm.get(i)).append('\n'); + sb.append(" distinct answers: ").append(warm.stream().distinct().count()).append('\n'); + List again = new ArrayList<>(); + for (int i = 0; i < 6; i++) again.add(model.ask(QUESTION, 64, 0.0f).text()); + sb.append("\nTemperature 0.0, limit 64, asked six more times: ").append(again.stream().distinct().count()).append(" distinct answer(s)\n"); + again.stream().distinct().forEach(t -> sb.append(" ").append(again.stream().filter(t::equals).count()).append("x: ").append(t).append('\n')); + Transcript.write("04-temperature.txt", sb.toString()); + + assertThat(second).isEqualTo(first); + } + + // ------------------------------------------------------------------ 5. forgetting the module + + @Test + void withoutTheVectorModuleTheModelDoesNotLoad() throws Exception { + String java = Path.of(System.getProperty("java.home"), "bin", "java").toString(); + Process p = new ProcessBuilder(java, "-cp", System.getProperty("java.class.path"), LoadOnly.class.getName(), MODELS) + .redirectErrorStream(true).start(); + String out = new String(p.getInputStream().readAllBytes()); + assertThat(p.waitFor(120, TimeUnit.SECONDS)).isTrue(); + List keep = out.lines().filter(l -> l.contains("ClassNotFoundException") || l.contains("Exception in thread")).toList(); + + Transcript.write("05-without-vector-module.txt", "Second JVM, same classpath, started WITHOUT --add-modules jdk.incubator.vector.\n" + + "Exit code: " + p.exitValue() + "\nRelevant lines:\n" + String.join("\n", keep.stream().map(l -> " " + l).toList()) + "\n"); + + assertThat(p.exitValue()).isNotZero(); + assertThat(out).contains("jdk.incubator.vector.FloatVector"); + } + + // ------------------------------------------------------------------ 6. the token limit counts the prompt + + @Test + void theTokenLimitIncludesThePrompt() { + LocalModel.Answer fits = model.ask(QUESTION, 30, 0.0f); + Throwable tooSmall = org.assertj.core.api.Assertions.catchThrowable(() -> model.ask(QUESTION, 20, 0.0f)); + + Transcript.write("07-token-limit.txt", "The question is " + fits.promptTokens() + " prompt tokens long.\n\n" + + "Limit 30: generated " + fits.generatedTokens() + " tokens (prompt + generated = " + (fits.promptTokens() + fits.generatedTokens()) + ")\n" + + "Limit 20: " + tooSmall.getClass().getSimpleName() + ": " + tooSmall.getMessage() + "\n"); + + assertThat(fits.promptTokens() + fits.generatedTokens()).isLessThanOrEqualTo(30); + assertThat(tooSmall).isInstanceOf(IllegalArgumentException.class).hasMessage("Prompt exceeds max tokens"); + } +} diff --git a/jlama/src/test/java/com/ankurm/agents/jlama/OllamaBaselineTest.java b/jlama/src/test/java/com/ankurm/agents/jlama/OllamaBaselineTest.java new file mode 100644 index 0000000..d6aa805 --- /dev/null +++ b/jlama/src/test/java/com/ankurm/agents/jlama/OllamaBaselineTest.java @@ -0,0 +1,71 @@ +package com.ankurm.agents.jlama; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * The same question, on the same machine, against a running Ollama. It only runs when you pass -Dollama.url=..., + * because it needs an Ollama server with a model called "tl" (see the README for how it was created). + */ +class OllamaBaselineTest { + + static final String URL = System.getProperty("ollama.url", ""); + static final ObjectMapper JSON = new ObjectMapper(); + + private static JsonNode call(HttpClient http, String path, String body) throws Exception { + HttpRequest.Builder b = HttpRequest.newBuilder(URI.create(URL + path)); + HttpRequest req = body == null ? b.GET().build() + : b.header("Content-Type", "application/json").POST(HttpRequest.BodyPublishers.ofString(body)).build(); + return JSON.readTree(http.send(req, HttpResponse.BodyHandlers.ofString()).body()); + } + + private static String f1(double d) { + return String.format(Locale.ROOT, "%.1f", d); + } + + @Test + void sameQuestionAgainstOllama() throws Exception { + assumeTrue(!URL.isBlank(), "pass -Dollama.url=http://127.0.0.1:11555 to run the Ollama baseline"); + HttpClient http = HttpClient.newHttpClient(); + String body = "{\"model\":\"tl\",\"prompt\":\"" + JlamaTest.QUESTION + "\",\"stream\":false," + + "\"options\":{\"temperature\":0,\"num_predict\":64}}"; + // Warm-up with a different prompt, so the first measured run has to process its prompt (apart from the shared template start). + call(http, "/api/generate", body.replace(JlamaTest.QUESTION, "Say hello.")); + List runs = new ArrayList<>(); + for (int i = 0; i < 5; i++) runs.add(call(http, "/api/generate", body)); + + StringBuilder sb = new StringBuilder("Ollama ").append(call(http, "/api/version", null).get("version").asText()) + .append(", model \"tl\" created from tinyllama-1.1b-chat-v1.0.Q4_0.gguf (TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF).\n"); + sb.append("Same machine, same question, 1 warm-up run then 5 measured runs, temperature 0, limit 64 tokens.\n\n"); + List gen = new ArrayList<>(); + for (int i = 0; i < runs.size(); i++) { + JsonNode r = runs.get(i); + double g = r.get("eval_count").asInt() * 1e9 / r.get("eval_duration").asLong(); + gen.add(g); + sb.append("run ").append(i + 1).append(": prompt ").append(r.get("prompt_eval_count").asInt()).append(" tokens (") + .append(r.get("prompt_eval_cached_count").asInt()).append(" served from cache) in ") + .append(r.get("prompt_eval_duration").asLong() / 1_000_000).append(" ms, generated ") + .append(r.get("eval_count").asInt()).append(" tokens in ").append(r.get("eval_duration").asLong() / 1_000_000) + .append(" ms (").append(f1(g)).append(" tok/s)\n"); + } + gen.sort(Double::compare); + sb.append("\nGeneration tokens/s: min ").append(f1(gen.get(0))).append(", median ").append(f1(gen.get(gen.size() / 2))) + .append(", max ").append(f1(gen.get(gen.size() - 1))).append('\n'); + sb.append("Answer of the last run: ").append(runs.get(runs.size() - 1).get("response").asText()).append('\n'); + Transcript.write("06-ollama-baseline.txt", sb.toString()); + + assertThat(gen.get(0)).isPositive(); + } +} diff --git a/jlama/src/test/java/com/ankurm/agents/jlama/Transcript.java b/jlama/src/test/java/com/ankurm/agents/jlama/Transcript.java new file mode 100644 index 0000000..05ee526 --- /dev/null +++ b/jlama/src/test/java/com/ankurm/agents/jlama/Transcript.java @@ -0,0 +1,39 @@ +package com.ankurm.agents.jlama; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.regex.Pattern; + +/** Writes what a test observed to output/NN-name.txt so every figure in the post comes from a file. */ +final class Transcript { + + private static final Path DIR = Path.of("output"); + private static final Pattern UUID = Pattern.compile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"); + private static final Pattern PORT = Pattern.compile("127\\.0\\.0\\.1:\\d+"); + private static final Pattern TS = Pattern.compile("\\d{4}-\\d\\d-\\d\\dT[\\d:.]+Z"); + + private Transcript() { + } + + /** Ports, ids and timestamps change on every run, so they are replaced by stable placeholders. */ + static String scrub(String s) { + var ids = new java.util.LinkedHashMap(); + var m = UUID.matcher(s); + StringBuilder out = new StringBuilder(); + while (m.find()) { + m.appendReplacement(out, ids.computeIfAbsent(m.group(), k -> "")); + } + m.appendTail(out); + return TS.matcher(PORT.matcher(out.toString()).replaceAll("127.0.0.1:PORT")).replaceAll("