Add docker-model-runner module: Spring AI OpenAI starter against Docker Model Runner and Ollama with a benchmark runner (time to first token, decode tokens/second, container memory, tool-call success); real docker/model-runner v1.2.9 CPU image and Ollama 0.40.13 on a shared 2-vCPU sandbox, no GPU

Co-Authored-By: Claude Sonnet 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_017ic4b4VQVEJWetTT1dYzyJ
2026-10-29 06:24:00 +00:00
parent 6d792f00e0
commit c97bcf9adf
18 changed files with 668 additions and 0 deletions
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# bench.sh <label> <base-url> <model> <container> [extra bench args]
+# Runs the Spring AI benchmark against one OpenAI-compatible server and samples the container's
+# memory with `docker stats` once a second while it runs. Writes output/<label>.txt.
+#   Docker Model Runner : bench.sh dmr    http://localhost:12434/engines/v1 hf.co/qwen/qwen2.5-0.5b-instruct-gguf:Q4_K_M dmr
+#   Ollama              : bench.sh ollama http://localhost:11434/v1          qwen2.5:0.5b ollama
+set -euo pipefail
+cd "$(dirname "$0")/.."
+LABEL=$1; URL=$2; MODEL=$3; CTR=$4; shift 4
+[ -f target/docker-model-runner-1.0.0.jar ] || mvn -B -q -T1 package -DskipTests
+mkdir -p output
+MEM=$(mktemp)
+( while true; do docker stats --no-stream --format '{{.MemUsage}}' "$CTR" | cut -d/ -f1 >> "$MEM"; sleep 1; done ) &
+SAMPLER=$!
+trap 'kill $SAMPLER 2>/dev/null || true' EXIT
+echo "idle container memory before: $(docker stats --no-stream --format '{{.MemUsage}}' "$CTR" | cut -d/ -f1)" > "output/$LABEL.mem.tmp"
+java -jar target/docker-model-runner-1.0.0.jar --bench.run=true --bench.label="$LABEL" \
+  --bench.out-file="output/$LABEL.txt" --spring.ai.openai.base-url="$URL" \
+  --spring.ai.openai.chat.options.model="$MODEL" "$@" > "output/$LABEL.log" 2>&1 || { tail -20 "output/$LABEL.log"; exit 1; }
+kill $SAMPLER 2>/dev/null || true
+python3 - "$MEM" "output/$LABEL.mem.tmp" "output/$LABEL.txt" <<'P'
+import sys,re
+def mb(s):
+    s=s.strip(); m=re.match(r'([\d.]+)(\w+)',s)
+    if not m: return None
+    v=float(m[1]); u=m[2]
+    return v*{'B':1/1048576,'KiB':1/1024,'MiB':1,'GiB':1024}[u]
+vals=[x for x in (mb(l) for l in open(sys.argv[1])) if x is not None]
+idle=open(sys.argv[2]).read().strip()
+with open(sys.argv[3],'a') as f:
+    f.write(f"\nmemory ({idle})\ncontainer memory during run: peak {max(vals):.0f} MiB, median {sorted(vals)[len(vals)//2]:.0f} MiB, {len(vals)} samples\n")
+P
+rm -f "output/$LABEL.mem.tmp" "output/$LABEL.log"
+cat "output/$LABEL.txt"
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Regenerates output/dmr.txt and output/ollama.txt. Needs both servers up (scripts/start-dmr.sh,
+# scripts/start-ollama.sh, or your own Docker Desktop Model Runner on :12434). Each engine is
+# restarted first so its memory figure is a fresh container with only this one model loaded.
+# Run it on an otherwise idle machine; the figures are only comparable within one run.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+ARGS="${BENCH_ARGS:---bench.repeats=3 --bench.max-tokens=48 --bench.tool-runs=10}"
+docker restart ollama >/dev/null; sleep 5
+scripts/bench.sh ollama http://localhost:11434/v1 qwen2.5:0.5b ollama $ARGS
+docker restart dmr >/dev/null; sleep 5
+scripts/bench.sh dmr http://localhost:12434/engines/v1 huggingface.co/qwen/qwen2.5-0.5b-instruct-gguf:Q4_K_M dmr $ARGS
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Starts the real Docker Model Runner server image (docker/model-runner, llama.cpp backend, CPU
+# build here) on host port 12434 and pulls the model from Hugging Face into it.
+#
+# On a machine with Docker Desktop (4.40+) or Docker Engine plus the docker-model-plugin you do
+# NOT need this script: `docker model pull hf.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M` and
+# enable host-side TCP support (Docker Desktop: Settings > Features in development > Enable
+# host-side TCP support, port 12434). This script is the same server in a container, which is
+# what lets the benchmark run on a Linux box with no Desktop.
+#
+# EXTRA_CA=/path/to/ca.crt  (optional) bakes a corporate/proxy CA into the image, needed when
+# outbound HTTPS is intercepted: the server downloads the model itself, from inside the container.
+set -euo pipefail
+NAME="${NAME:-dmr}"
+IMAGE=docker/model-runner:latest
+if [ -n "${EXTRA_CA:-}" ]; then
+	D=$(mktemp -d); cp "$EXTRA_CA" "$D/extra-ca.crt"
+	printf 'FROM %s\nUSER root\nCOPY extra-ca.crt /usr/local/share/ca-certificates/extra-ca.crt\nRUN update-ca-certificates\nUSER modelrunner\n' "$IMAGE" > "$D/Dockerfile"
+	docker build -q -t dmr-ca "$D" >/dev/null; IMAGE=dmr-ca
+fi
+docker rm -f "$NAME" >/dev/null 2>&1 || true
+docker run -d --name "$NAME" -p 12434:12434 "$IMAGE" >/dev/null
+until curl -sf localhost:12434/engines/v1/models >/dev/null; do sleep 1; done
+curl -s -X POST localhost:12434/models/create -H 'Content-Type: application/json' \
+	-d '{"from":"hf.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M"}' | tail -c 120; echo
+curl -s localhost:12434/engines/v1/models; echo
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+# Starts ollama/ollama on host port 11434 and pulls qwen2.5:1.5b (Ollama's default tag is Q4_K_M,
+# the same quantisation as the GGUF the Docker Model Runner script pulls).
+# EXTRA_CA=/path/to/ca.crt  (optional) as in start-dmr.sh.
+set -euo pipefail
+NAME="${NAME:-ollama}"
+IMAGE=ollama/ollama:latest
+if [ -n "${EXTRA_CA:-}" ]; then
+	D=$(mktemp -d); cp "$EXTRA_CA" "$D/extra-ca.crt"
+	printf 'FROM %s\nCOPY extra-ca.crt /usr/local/share/ca-certificates/extra-ca.crt\nRUN update-ca-certificates\n' "$IMAGE" > "$D/Dockerfile"
+	docker build -q -t ollama-ca "$D" >/dev/null; IMAGE=ollama-ca
+fi
+docker rm -f "$NAME" >/dev/null 2>&1 || true
+docker run -d --name "$NAME" -p 11434:11434 "$IMAGE" >/dev/null
+until docker exec "$NAME" ollama list >/dev/null 2>&1; do sleep 1; done
+docker exec "$NAME" ollama pull qwen2.5:1.5b
+docker exec "$NAME" ollama show qwen2.5:1.5b | head -12