asmhatreandClaude Sonnet 5.5
c97bcf9adf
Co-Authored-By: Claude Sonnet 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_017ic4b4VQVEJWetTT1dYzyJ
27 lines
1.6 KiB
Bash
Executable File
27 lines
1.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Starts the real Docker Model Runner server image (docker/model-runner, llama.cpp backend, CPU
|
|
# build here) on host port 12434 and pulls the model from Hugging Face into it.
|
|
#
|
|
# On a machine with Docker Desktop (4.40+) or Docker Engine plus the docker-model-plugin you do
|
|
# NOT need this script: `docker model pull hf.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M` and
|
|
# enable host-side TCP support (Docker Desktop: Settings > Features in development > Enable
|
|
# host-side TCP support, port 12434). This script is the same server in a container, which is
|
|
# what lets the benchmark run on a Linux box with no Desktop.
|
|
#
|
|
# EXTRA_CA=/path/to/ca.crt (optional) bakes a corporate/proxy CA into the image, needed when
|
|
# outbound HTTPS is intercepted: the server downloads the model itself, from inside the container.
|
|
set -euo pipefail
|
|
NAME="${NAME:-dmr}"
|
|
IMAGE=docker/model-runner:latest
|
|
if [ -n "${EXTRA_CA:-}" ]; then
|
|
D=$(mktemp -d); cp "$EXTRA_CA" "$D/extra-ca.crt"
|
|
printf 'FROM %s\nUSER root\nCOPY extra-ca.crt /usr/local/share/ca-certificates/extra-ca.crt\nRUN update-ca-certificates\nUSER modelrunner\n' "$IMAGE" > "$D/Dockerfile"
|
|
docker build -q -t dmr-ca "$D" >/dev/null; IMAGE=dmr-ca
|
|
fi
|
|
docker rm -f "$NAME" >/dev/null 2>&1 || true
|
|
docker run -d --name "$NAME" -p 12434:12434 "$IMAGE" >/dev/null
|
|
until curl -sf localhost:12434/engines/v1/models >/dev/null; do sleep 1; done
|
|
curl -s -X POST localhost:12434/models/create -H 'Content-Type: application/json' \
|
|
-d '{"from":"hf.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M"}' | tail -c 120; echo
|
|
curl -s localhost:12434/engines/v1/models; echo
|