From aa8b1604eda462e758bc058607db64434c7d1b34 Mon Sep 17 00:00:00 2001 From: asmhatre Date: Wed, 28 Oct 2026 18:36:40 +0000 Subject: [PATCH] Add image-gen module: Spring AI ImageModel options on the wire, a moderation gate, and safe storage of generated images in MinIO (sniffed content type, byte and pixel caps, server-chosen keys, capped URL fetcher); per-provider cost table dated 2026-10-28. ImageModel and ModerationModel are hand-written fakes; MinIO is real via Testcontainers; no image provider called. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_017ic4b7rnC7jppGGdHGESvy --- README.md | 1 + image-gen/.gitignore | 1 + image-gen/README.md | 45 ++++++ image-gen/output/01-api-shape.txt | 25 +++ image-gen/output/02-wire-requests.txt | 21 +++ image-gen/output/03-wire-response.txt | 12 ++ image-gen/output/04-wire-refusal.txt | 5 + image-gen/output/05-store-minio.txt | 13 ++ image-gen/output/05b-presigned.txt | 7 + image-gen/output/06-verifier.txt | 15 ++ image-gen/output/07-trailing-bytes.txt | 7 + image-gen/output/08-base64-cap.txt | 4 + image-gen/output/09-fetcher.txt | 10 ++ image-gen/output/10-keys.txt | 15 ++ image-gen/output/11-naive-vs-pinned.txt | 6 + image-gen/output/12-service-happy.txt | 9 ++ image-gen/output/13-refusals.txt | 11 ++ image-gen/output/14-batch.txt | 4 + image-gen/output/15-url-results.txt | 5 + image-gen/output/16-cost.txt | 38 +++++ image-gen/pom.xml | 83 ++++++++++ image-gen/scripts/run-all.sh | 9 ++ .../java/com/ankurm/imagegen/CostModel.java | 71 +++++++++ .../java/com/ankurm/imagegen/ImagePolicy.java | 30 ++++ .../com/ankurm/imagegen/ImageRequest.java | 12 ++ .../com/ankurm/imagegen/ImageService.java | 83 ++++++++++ .../com/ankurm/imagegen/ImageSniffer.java | 80 ++++++++++ .../java/com/ankurm/imagegen/ImageStore.java | 93 ++++++++++++ .../com/ankurm/imagegen/ImageVerifier.java | 46 ++++++ .../ankurm/imagegen/OpenAiOptionsFactory.java | 40 +++++ .../java/com/ankurm/imagegen/PromptGate.java | 65 ++++++++ .../ankurm/imagegen/RejectedException.java | 16 ++ .../com/ankurm/imagegen/RemoteFetcher.java | 78 ++++++++++ .../java/com/ankurm/imagegen/Sniffed.java | 12 ++ .../java/com/ankurm/imagegen/StoredImage.java | 5 + .../com/ankurm/imagegen/ApiShapeTest.java | 72 +++++++++ .../java/com/ankurm/imagegen/CostTest.java | 33 ++++ .../com/ankurm/imagegen/OpenAiWireTest.java | 135 +++++++++++++++++ .../java/com/ankurm/imagegen/ServiceTest.java | 140 +++++++++++++++++ .../ankurm/imagegen/StorageChecksTest.java | 135 +++++++++++++++++ .../com/ankurm/imagegen/StoreMinioTest.java | 142 ++++++++++++++++++ .../imagegen/support/FakeImagesServer.java | 120 +++++++++++++++ .../com/ankurm/imagegen/support/Fixtures.java | 31 ++++ .../com/ankurm/imagegen/support/Images.java | 72 +++++++++ .../com/ankurm/imagegen/support/Minio.java | 77 ++++++++++ .../imagegen/support/ScriptedImageModel.java | 76 ++++++++++ .../support/ScriptedModerationModel.java | 56 +++++++ .../ankurm/imagegen/support/Transcript.java | 46 ++++++ 48 files changed, 2112 insertions(+) create mode 100644 image-gen/.gitignore create mode 100644 image-gen/README.md create mode 100644 image-gen/output/01-api-shape.txt create mode 100644 image-gen/output/02-wire-requests.txt create mode 100644 image-gen/output/03-wire-response.txt create mode 100644 image-gen/output/04-wire-refusal.txt create mode 100644 image-gen/output/05-store-minio.txt create mode 100644 image-gen/output/05b-presigned.txt create mode 100644 image-gen/output/06-verifier.txt create mode 100644 image-gen/output/07-trailing-bytes.txt create mode 100644 image-gen/output/08-base64-cap.txt create mode 100644 image-gen/output/09-fetcher.txt create mode 100644 image-gen/output/10-keys.txt create mode 100644 image-gen/output/11-naive-vs-pinned.txt create mode 100644 image-gen/output/12-service-happy.txt create mode 100644 image-gen/output/13-refusals.txt create mode 100644 image-gen/output/14-batch.txt create mode 100644 image-gen/output/15-url-results.txt create mode 100644 image-gen/output/16-cost.txt create mode 100644 image-gen/pom.xml create mode 100755 image-gen/scripts/run-all.sh create mode 100644 image-gen/src/main/java/com/ankurm/imagegen/CostModel.java create mode 100644 image-gen/src/main/java/com/ankurm/imagegen/ImagePolicy.java create mode 100644 image-gen/src/main/java/com/ankurm/imagegen/ImageRequest.java create mode 100644 image-gen/src/main/java/com/ankurm/imagegen/ImageService.java create mode 100644 image-gen/src/main/java/com/ankurm/imagegen/ImageSniffer.java create mode 100644 image-gen/src/main/java/com/ankurm/imagegen/ImageStore.java create mode 100644 image-gen/src/main/java/com/ankurm/imagegen/ImageVerifier.java create mode 100644 image-gen/src/main/java/com/ankurm/imagegen/OpenAiOptionsFactory.java create mode 100644 image-gen/src/main/java/com/ankurm/imagegen/PromptGate.java create mode 100644 image-gen/src/main/java/com/ankurm/imagegen/RejectedException.java create mode 100644 image-gen/src/main/java/com/ankurm/imagegen/RemoteFetcher.java create mode 100644 image-gen/src/main/java/com/ankurm/imagegen/Sniffed.java create mode 100644 image-gen/src/main/java/com/ankurm/imagegen/StoredImage.java create mode 100644 image-gen/src/test/java/com/ankurm/imagegen/ApiShapeTest.java create mode 100644 image-gen/src/test/java/com/ankurm/imagegen/CostTest.java create mode 100644 image-gen/src/test/java/com/ankurm/imagegen/OpenAiWireTest.java create mode 100644 image-gen/src/test/java/com/ankurm/imagegen/ServiceTest.java create mode 100644 image-gen/src/test/java/com/ankurm/imagegen/StorageChecksTest.java create mode 100644 image-gen/src/test/java/com/ankurm/imagegen/StoreMinioTest.java create mode 100644 image-gen/src/test/java/com/ankurm/imagegen/support/FakeImagesServer.java create mode 100644 image-gen/src/test/java/com/ankurm/imagegen/support/Fixtures.java create mode 100644 image-gen/src/test/java/com/ankurm/imagegen/support/Images.java create mode 100644 image-gen/src/test/java/com/ankurm/imagegen/support/Minio.java create mode 100644 image-gen/src/test/java/com/ankurm/imagegen/support/ScriptedImageModel.java create mode 100644 image-gen/src/test/java/com/ankurm/imagegen/support/ScriptedModerationModel.java create mode 100644 image-gen/src/test/java/com/ankurm/imagegen/support/Transcript.java diff --git a/README.md b/README.md index 88bc99e..3da83b9 100644 --- a/README.md +++ b/README.md @@ -30,5 +30,6 @@ Runnable companion code for the Spring AI articles on [ankurm.com](https://ankur | [`rerank/`](rerank) | A bi-encoder (all-MiniLM-L6-v2) retrieves candidates, a cross-encoder (ms-marco-MiniLM-L-6-v2) rescores them, and a `DocumentPostProcessor` keeps the top 5 for `RetrievalAugmentationAdvisor`. Both models run in-process through ONNX Runtime 1.31.0 with DJL tokenizers 0.39.0 (no database, no API key). 60 questions over 94 passages: hit@1 41 to 49 and MRR 0.78 to 0.87, almost all of it on exact error codes, with paraphrases slightly worse; pool size, batch size, threads and int8 against latency on 2 vCPUs; and five traps reproduced (missing `token_type_ids`, swapped pair order, logits are not probabilities, truncation, a pool that is too small). Spring Boot 4.1.2, Spring AI 2.0.2, Java 25. | [Reranking RAG Results in Java: Cross-Encoder Reranker with ONNX Runtime Before the LLM](https://ankurm.com/reranking-rag-results-java-cross-encoder-onnx-runtime-spring-ai/) | | [`audio/`](audio) | A meeting-minutes generator: `OpenAiAudioTranscriptionModel` -> `ChatClient.entity(Minutes.class)` -> `OpenAiAudioSpeechModel`. Splitting a 30 MB recording at a silence under the 25 MB upload limit (and what a blind split does), SDK retries that re-upload the file, the 4096-character speech limit, three replies that fail to parse, and a cost model with formulas. The audio endpoints are a local mock and the chat model a stub; no real model was called. Spring Boot 4.1.2, Spring AI 2.0.2, Java 25. | [Speech-to-Text and Text-to-Speech with Spring AI: Build a Meeting-Minutes Generator](https://ankurm.com/spring-ai-speech-to-text-text-to-speech-meeting-minutes/) | | [`graphrag-neo4j/`](graphrag-neo4j) | Entities and relations extracted with `ChatClient.entity(...)` into a typed Neo4j graph with a vector index on entity names (Testcontainers, `neo4j:2026.10.1-community`; also run on 5.26), then multi-hop retrieval by span linking, variable-length Cypher paths and path ranking, against plain vector RAG over the same 35 chunks and 9 questions: complete evidence 7/9 for the graph, 5/9 for vector k=8, 2/9 for k=4, with the traps reproduced (duplicate entity under an alias, an embedding merge that joins the wrong incidents, a reply with a sentence before the JSON fence, a hostile relation type, hop explosion). **The extraction step is a rule-based stand-in, not a language model**; Neo4j, embeddings and Cypher are real. Spring Boot 4.1.2, Spring AI 2.0.2, Java 25. | [GraphRAG in Java with Neo4j and Spring AI](https://ankurm.com/graphrag-java-neo4j-spring-ai-knowledge-graph/) | +| [`image-gen/`](image-gen) | `ImageModel` and `ImagePrompt` with `OpenAiImageOptions` (what the real `OpenAiImageModel` puts on the wire, a per-call options object that silently resets the model, a `List` prompt that throws), a prompt gate with `ModerationModel` that fails closed, and safe storage in MinIO: sniffed content type, byte and pixel caps, server-chosen keys, a fetcher for provider URLs, presigned reads. A per-image price table from the vendors' pages dated 2026-10-28. No image provider called: the `ImageModel` and `ModerationModel` are hand-written fakes and a mock server; the MinIO tests use a real server through Testcontainers. Spring Boot 4.1.2, Spring AI 2.0.2, Java 25. | [Image Generation and Vision in Spring AI: ImageModel, Prompt Options and Safe Storage](https://ankurm.com/spring-ai-image-generation-imagemodel-s3-minio-safe-storage/) | Upgrading from Spring AI 1.x: [migration guide](https://ankurm.com/spring-ai-1-to-2-migration-guide/). diff --git a/image-gen/.gitignore b/image-gen/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/image-gen/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/image-gen/README.md b/image-gen/README.md new file mode 100644 index 0000000..2c4364b --- /dev/null +++ b/image-gen/README.md @@ -0,0 +1,45 @@ +# image-gen + +Companion code for [Image Generation and Vision in Spring AI: ImageModel, Prompt Options and Safe Storage](https://ankurm.com/spring-ai-image-generation-imagemodel-s3-minio-safe-storage/), part of the [Spring AI series](../README.md) on ankurm.com. It is the generation half of [Multimodal Spring AI: Extract Structured Data from Images](https://ankurm.com/multimodal-spring-ai-extract-structured-data-from-images-receipts-java-records/). + +A prompt goes through our own checks, into `ImageModel`, and the bytes that come back are verified and written to an S3-compatible bucket under a key we choose. + +**No image provider was called.** The tests use two hand-written fakes -- `ScriptedImageModel` (an `ImageModel` that returns what it was told to) and `ScriptedModerationModel` -- and, for the wire tests, the real `OpenAiImageModel` against a local mock of `/v1/images/generations` (`FakeImagesServer`) that returns a fixed 64x64 PNG whatever the prompt says. Nothing here measures image quality, latency, or whether a real model would refuse a prompt. **The storage is real:** the `ImageStore` tests run against a MinIO server in Docker through Testcontainers (the image is `chainguard/minio`, an upstream MinIO build, because `minio/minio` and `quay.io/minio/minio` could not be pulled from the machine these were captured on; the exact version is in `output/05-store-minio.txt`). Prices in `CostModel` were copied from the providers' pages on 2026-10-28 and are not checked by any test. + +| | | +|---|---| +| Spring Boot | 4.1.2 (parent; GA on Maven Central 2026-10-22) | +| Spring AI | 2.0.2 (`spring-ai-model`, `spring-ai-openai`, `spring-ai-stability-ai`, `spring-ai-google-genai-image`) | +| AWS SDK for Java | 2.57.5 (`s3`) | +| Testcontainers | 2.0.5 | +| Java | 25 | +| Run | `scripts/run-all.sh` (about 20 s, needs Docker for the MinIO tests) | + +## Source + +| File | What it is | +|---|---| +| [`ImageService`](src/main/java/com/ankurm/imagegen/ImageService.java) | prompt gate -> `ImageModel` -> bytes -> verify -> store; one bad image in a batch stores nothing | +| [`PromptGate`](src/main/java/com/ankurm/imagegen/PromptGate.java) | length and control-character checks, then `ModerationModel`; fails closed when moderation is down | +| [`OpenAiOptionsFactory`](src/main/java/com/ankurm/imagegen/OpenAiOptionsFactory.java) | size / quality / n checked against allow-lists, then `OpenAiImageOptions` with the model set explicitly | +| [`ImageSniffer`](src/main/java/com/ankurm/imagegen/ImageSniffer.java), [`ImageVerifier`](src/main/java/com/ankurm/imagegen/ImageVerifier.java), [`ImagePolicy`](src/main/java/com/ankurm/imagegen/ImagePolicy.java) | magic-number sniffing, size cap, pixel cap read from the header, a real decode, allow-list of types | +| [`RemoteFetcher`](src/main/java/com/ankurm/imagegen/RemoteFetcher.java) | downloads a provider-returned URL: https and listed hosts only, no redirects, hard byte cap | +| [`ImageStore`](src/main/java/com/ankurm/imagegen/ImageStore.java) | S3/MinIO writer: server-chosen key, pinned `Content-Type`, `Content-Disposition: attachment`, presigned GET | +| [`CostModel`](src/main/java/com/ankurm/imagegen/CostModel.java) | per-image prices with the date they were read and the page they came from | + +## Captured output (`output/`) + +| File | Shows | +|---|---| +| `01-api-shape.txt` | class names, portable vs provider options, what is *not* there (moderation option, image moderation) | +| `02-wire-requests.txt` | JSON body sent by the real `OpenAiImageModel` for six option shapes, including the model reset and the `ImagePrompt(List)` failure | +| `03-wire-response.txt` | what `ImageResponse` carries back (b64 or URL, revised prompt, no usage) | +| `04-wire-refusal.txt` | a provider 400 `content_policy_violation` arrives as an SDK exception | +| `05-store-minio.txt`, `05b-presigned.txt` | the object as MinIO holds it; presigned URL behaviour | +| `06-verifier.txt`, `07-trailing-bytes.txt`, `08-base64-cap.txt` | twelve inputs against the verifier; the limit of sniffing; base64 refused before decoding | +| `09-fetcher.txt` | redirect, scheme, host list, endless body | +| `10-keys.txt`, `11-naive-vs-pinned.txt` | owner and id validation; trusting the sender's `Content-Type` | +| `12-service-happy.txt` .. `15-url-results.txt` | the service end to end; refusals before any paid call; a batch with one bad image; URL results | +| `16-cost.txt` | the per-image price table | + +Keys, ports and timestamps differ between runs; everything else is stable. diff --git a/image-gen/output/01-api-shape.txt b/image-gen/output/01-api-shape.txt new file mode 100644 index 0000000..6eb9e05 --- /dev/null +++ b/image-gen/output/01-api-shape.txt @@ -0,0 +1,25 @@ +# Spring AI 2.0.2 image classes (read from the jars) + +interface : org.springframework.ai.image.ImageModel +single method : [call] +request : org.springframework.ai.image.ImagePrompt +OpenAI : org.springframework.ai.openai.OpenAiImageModel +Stability AI : org.springframework.ai.stabilityai.StabilityAiImageModel +moderation interface : org.springframework.ai.moderation.ModerationModel +moderation OpenAI : org.springframework.ai.openai.OpenAiModerationModel + +portable ImageOptions getters : [getHeight, getModel, getN, getResponseFormat, getStyle, getWidth] +OpenAiImageOptions getters : [getApiKey, getBaseUrl, getCredential, getCustomHeaders, getDeploymentName, getHeight, getMaxRetries, getMicrosoftDeploymentName, getMicrosoftFoundryServiceVersion, getModel, getN, getOrganizationId, getProxy, getQuality, getResponseFormat, getSize, getStyle, getTimeout, getUser, getWidth, isGitHubModels, isMicrosoftFoundry] +StabilityAiImageOptions : [getCfgScale, getClipGuidancePreset, getHeight, getModel, getN, getResponseFormat, getSampler, getSeed, getSteps, getStyle, getStylePreset, getWidth] +GoogleGenAiImageOptions : [getAspectRatio, getHeight, getImageSize, getLabels, getMaxOutputTokens, getModel, getN, getOutputCompressionQuality, getOutputMimeType, getPersonGeneration, getResponseFormat, getSafetyFilterLevel, getSeed, getStyle, getTemperature, getTopK, getTopP, getWidth] + +OpenAiImageOptions.DEFAULT_IMAGE_MODEL = gpt-image-1-mini +OpenAiModerationOptions.DEFAULT_MODERATION_MODEL = omni-moderation-latest +OpenAiImageOptions has a getModeration() : false +ModerationMessage getters : [getText] (text only, no media) +Stability default engine / base URL : stable-diffusion-v1-6 @ https://api.stability.ai/v1 +GoogleGenAiImageModelName : [gemini-2.5-flash-image, gemini-3-pro-image, gemini-3.1-flash-image] +Google SafetyFilterLevel : [BLOCK_LOW_AND_ABOVE, BLOCK_MEDIUM_AND_ABOVE, BLOCK_ONLY_HIGH, BLOCK_NONE, SAFETY_FILTER_LEVEL_UNSPECIFIED] +Google PersonGeneration : [DONT_ALLOW, ALLOW_ADULT, ALLOW_ALL, PERSON_GENERATION_UNSPECIFIED] +AbstractOpenAiOptions.DEFAULT_MAX_RETRIES (image options extend it) = 3 +portable builder gives : DefaultImageModelOptions diff --git a/image-gen/output/02-wire-requests.txt b/image-gen/output/02-wire-requests.txt new file mode 100644 index 0000000..2394495 --- /dev/null +++ b/image-gen/output/02-wire-requests.txt @@ -0,0 +1,21 @@ +# What OpenAiImageModel sends (mock server; bodies recorded as received) + +1. ImagePrompt("a red bicycle"), model default only + POST /v1/images/generations + {"prompt":"a red bicycle","model":"gpt-image-test"} + +2. OpenAiImageOptions size=1536x1024 quality=high n=2 + {"prompt":"a red bicycle","model":"gpt-image-1-mini","n":2,"quality":"high","size":"1536x1024"} + +3. portable ImageOptionsBuilder width=1024 height=1536 + {"prompt":"a red bicycle","model":"gpt-image-test","size":"1024x1536"} + +4. OpenAiImageOptions user=user-8f2c + {"prompt":"a red bicycle","model":"gpt-image-1-mini","user":"user-8f2c"} + +5. OpenAiImageOptions responseFormat=b64_json + {"prompt":"a red bicycle","model":"gpt-image-1-mini","response_format":"b64_json"} + +6a. ImagePrompt(List) with no options: IllegalArgumentException: image options cannot be null +6b. same two messages (weights 1.0 and -1.0) with empty OpenAiImageOptions + {"prompt":"a red bicycle","model":"gpt-image-1-mini"} diff --git a/image-gen/output/03-wire-response.txt b/image-gen/output/03-wire-response.txt new file mode 100644 index 0000000..b9d70c5 --- /dev/null +++ b/image-gen/output/03-wire-response.txt @@ -0,0 +1,12 @@ +# What comes back from OpenAiImageModel (mock server) + +results : 1 +Image.getUrl() : null +Image.getB64Json() len : 10500 chars +metadata type : OpenAiImageGenerationMetadata +revised prompt : a rewritten prompt +response created : 1760000000 +response usage : EmptyUsage, total tokens 0 (the mock body carried usage.total_tokens=1068) + +URL mode: Image.getUrl() = http://127.0.0.1:43039/files/ok.png +URL mode: Image.getB64Json() = null diff --git a/image-gen/output/04-wire-refusal.txt b/image-gen/output/04-wire-refusal.txt new file mode 100644 index 0000000..f36e0e4 --- /dev/null +++ b/image-gen/output/04-wire-refusal.txt @@ -0,0 +1,5 @@ +# A provider-side refusal (400 content_policy_violation from the mock) + +exception class : com.openai.errors.BadRequestException +message (first 200): 400: Your request was rejected as a result of our safety system. +requests received : 1 (maxRetries=0) diff --git a/image-gen/output/05-store-minio.txt b/image-gen/output/05-store-minio.txt new file mode 100644 index 0000000..3e3d860 --- /dev/null +++ b/image-gen/output/05-store-minio.txt @@ -0,0 +1,13 @@ +# ImageStore on a real MinIO server + +image : chainguard/minio:latest +minio --version : minio version RELEASE.2026-10-23T20-06-51Z (commit-id=76c06ff1d2cc844050fdeb1d54bf8fa70db72278) +Server header : MinIO +stored key : generated/user-42/4cf346e4b7008b1874935d7969e8ccb1.png +content type : image/png, 6363 bytes, 64x48 +sha256 : 7254899daa04ebf862c59d32d2d3280bb179e6c61899505c43aed169c6197bcb +HEAD Content-Type : image/png +HEAD Content-Disposition : attachment +HEAD Cache-Control : private, max-age=3600 +HEAD metadata : {owner=user-42, sha256=7254899daa04ebf862c59d32d2d3280bb179e6c61899505c43aed169c6197bcb} +read back equal : true diff --git a/image-gen/output/05b-presigned.txt b/image-gen/output/05b-presigned.txt new file mode 100644 index 0000000..43aad02 --- /dev/null +++ b/image-gen/output/05b-presigned.txt @@ -0,0 +1,7 @@ +# Presigned GET (5 minutes) against MinIO + +signed URL, parameters : X-Amz-Algorithm=... & X-Amz-Date=... & X-Amz-SignedHeaders=host & X-Amz-Credential=... & X-Amz-Expires=300 & X-Amz-Signature=... +GET signed URL : 200, 742 bytes, Content-Type image/jpeg +Content-Disposition : attachment +same URL, other owner : 403 +same path, no signature : 403 diff --git a/image-gen/output/06-verifier.txt b/image-gen/output/06-verifier.txt new file mode 100644 index 0000000..fac82ce --- /dev/null +++ b/image-gen/output/06-verifier.txt @@ -0,0 +1,15 @@ +# ImageVerifier decisions (cap 8 MiB, 16,000,000 pixels, png/jpeg/webp) + +input verdict detail +real PNG 64x48 ACCEPT image/png 64x48 +real JPEG 64x48 ACCEPT image/jpeg 64x48 +WebP header only (sniffed, not decoded) ACCEPT image/webp +SVG with a script REJECT not-an-image: no known image signature in the first bytes +HTML page REJECT not-an-image: no known image signature in the first bytes +PDF REJECT not-an-image: no known image signature in the first bytes +PNG signature + garbage REJECT undecodable: IIOException +valid PNG + 40 bytes of HTML appended ACCEPT image/png 16x16 +PNG header claiming 40000x40000 REJECT too-many-pixels: 40000x40000 = 1600000000 > cap 16000000 +GIF (sniffed but not on the allow list) REJECT type-not-allowed: image/gif +empty REJECT not-an-image: no known image signature in the first bytes +9 MB of PNG-looking zeros REJECT too-large: 9437184 bytes > cap 8388608 diff --git a/image-gen/output/07-trailing-bytes.txt b/image-gen/output/07-trailing-bytes.txt new file mode 100644 index 0000000..4a1ce12 --- /dev/null +++ b/image-gen/output/07-trailing-bytes.txt @@ -0,0 +1,7 @@ +# Sniff + decode does not catch bytes after the image + +input : valid 16x16 PNG followed by 25 bytes of ".getBytes(StandardCharsets.UTF_8)); + cases.put("HTML page", "".getBytes(StandardCharsets.UTF_8)); + cases.put("PDF", "%PDF-1.7\n1 0 obj<<>>endobj\n".getBytes(StandardCharsets.UTF_8)); + cases.put("PNG signature + garbage", Images.concat(new byte[] {(byte) 0x89, 'P', 'N', 'G', 0x0D, 0x0A, 0x1A, 0x0A}, new byte[40])); + cases.put("valid PNG + 40 bytes of HTML appended", Images.concat(Images.png(16, 16), "".getBytes(StandardCharsets.UTF_8))); + cases.put("PNG header claiming 40000x40000", Images.pngClaiming(40000, 40000)); + cases.put("GIF (sniffed but not on the allow list)", "GIF89a\u0001\u0000\u0001\u0000\u0000\u0000\u0000;".getBytes(StandardCharsets.ISO_8859_1)); + cases.put("empty", new byte[0]); + cases.put("9 MB of PNG-looking zeros", withPngSignature(9 * 1024 * 1024)); + + try (Transcript t = new Transcript("06-verifier.txt", "ImageVerifier decisions (cap 8 MiB, 16,000,000 pixels, png/jpeg/webp)")) { + t.line("%-45s %-9s %s", "input", "verdict", "detail"); + for (var e : cases.entrySet()) { + String verdict; + String detail; + try { + Sniffed s = verifier.verify(e.getValue()); + verdict = "ACCEPT"; + detail = s.contentType() + (s.width() > 0 ? " " + s.width() + "x" + s.height() : ""); + } catch (RejectedException r) { + verdict = "REJECT"; + detail = r.getMessage(); + } + t.line("%-45s %-9s %s", e.getKey(), verdict, detail); + } + } + assertThat(verifier.verify(Images.png(64, 48)).contentType()).isEqualTo("image/png"); + assertReason(() -> verifier.verify(cases.get("HTML page")), "not-an-image"); + assertReason(() -> verifier.verify(cases.get("PNG header claiming 40000x40000")), "too-many-pixels"); + assertReason(() -> verifier.verify(cases.get("9 MB of PNG-looking zeros")), "too-large"); + assertReason(() -> verifier.verify(cases.get("PNG signature + garbage")), "undecodable"); + } + + /** Honest limit, pinned by a test: a valid image with junk appended is still accepted. */ + @Test + void trailingBytesSurvive() { + byte[] polyglot = Images.concat(Images.png(16, 16), "".getBytes(StandardCharsets.UTF_8)); + assertThat(verifier.verify(polyglot).contentType()).isEqualTo("image/png"); + try (Transcript t = new Transcript("07-trailing-bytes.txt", "Sniff + decode does not catch bytes after the image")) { + t.line("input : valid 16x16 PNG followed by 25 bytes of ".getBytes(); + s3.putObject(PutObjectRequest.builder().bucket(BUCKET).key("naive/upload.png").contentType("text/html").build(), RequestBody.fromBytes(html)); + HeadObjectResponse naive = s3.headObject(HeadObjectRequest.builder().bucket(BUCKET).key("naive/upload.png").build()); + try (Transcript t = new Transcript("11-naive-vs-pinned.txt", "Trusting the sender's Content-Type vs pinning it")) { + t.line("naive writer : key naive/upload.png Content-Type = %s (sender said so; the .png suffix did not matter)", naive.contentType()); + t.line("ImageStore : the verifier refuses these bytes first: "); + try { new ImageVerifier(ImagePolicy.defaults()).verify(html); } catch (RejectedException e) { t.line(" %s", e.getMessage()); } + t.line("objects under naive/ : %d", s3.listObjectsV2(ListObjectsV2Request.builder().bucket(BUCKET).prefix("naive/").build()).keyCount()); + } + assertThat(naive.contentType()).isEqualTo("text/html"); + } +} diff --git a/image-gen/src/test/java/com/ankurm/imagegen/support/FakeImagesServer.java b/image-gen/src/test/java/com/ankurm/imagegen/support/FakeImagesServer.java new file mode 100644 index 0000000..c6d2bb6 --- /dev/null +++ b/image-gen/src/test/java/com/ankurm/imagegen/support/FakeImagesServer.java @@ -0,0 +1,120 @@ +package com.ankurm.imagegen.support; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; + +/** + * A MOCK of OpenAI's POST /v1/images/generations and /v1/images/edits on a local port, plus a tiny + * "image host" for the URL-returning case. The real {@code OpenAiImageModel} (and the OpenAI Java + * SDK under it) talk to it over HTTP, so the recorded request bodies are what Spring AI really sends. + * + *

NOT real: the pictures. The server returns a fixed 64x64 gradient PNG whatever the prompt says. + * It checks nothing about size, quality or model names, so it cannot tell you whether a provider + * would accept the values. The 400 "content_policy_violation" body copies the shape OpenAI's error + * format uses; whether and when a real model refuses a prompt is not tested here. + */ +public final class FakeImagesServer implements AutoCloseable { + + public record Seen(String method, String path, String contentType, String body) {} + + public enum Mode { B64, URL, REFUSE_400 } + + private final HttpServer server; + private final List seen = new CopyOnWriteArrayList<>(); + private volatile Mode mode = Mode.B64; + private final byte[] png = Images.png(64, 64); + + public FakeImagesServer() throws IOException { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/v1/images/generations", this::generate); + server.createContext("/v1/images/edits", this::generate); + server.createContext("/files/ok.png", ex -> bytes(ex, 200, "image/png", png)); + server.createContext("/files/redirect", ex -> { + ex.getResponseHeaders().add("Location", "/files/ok.png"); + ex.sendResponseHeaders(302, -1); + ex.close(); + }); + server.createContext("/files/endless", this::endless); + server.start(); + } + + public String baseUrl() { return "http://127.0.0.1:" + server.getAddress().getPort() + "/v1"; } + + public String host() { return "127.0.0.1"; } + + public String fileUrl(String name) { return "http://127.0.0.1:" + server.getAddress().getPort() + "/files/" + name; } + + public List seen() { return seen; } + + public void clear() { seen.clear(); } + + public void mode(Mode m) { this.mode = m; } + + @Override + public void close() { server.stop(0); } + + private void generate(HttpExchange ex) throws IOException { + String body = new String(ex.getRequestBody().readAllBytes(), StandardCharsets.ISO_8859_1); + String ct = ex.getRequestHeaders().getFirst("Content-Type"); + boolean multipart = ct != null && ct.startsWith("multipart/"); + seen.add(new Seen(ex.getRequestMethod(), ex.getRequestURI().getPath(), ct, multipart ? summarise(body) : body)); + String json; + int status = 200; + switch (mode) { + case REFUSE_400 -> { + status = 400; + json = "{\"error\":{\"message\":\"Your request was rejected as a result of our safety system.\"," + + "\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"content_policy_violation\"}}"; + } + case URL -> json = "{\"created\":1760000000,\"data\":[{\"url\":\"" + fileUrl("ok.png") + "\",\"revised_prompt\":\"a rewritten prompt\"}]}"; + default -> json = "{\"created\":1760000000,\"data\":[{\"b64_json\":\"" + Images.b64(png) + + "\",\"revised_prompt\":\"a rewritten prompt\"}]," + + "\"usage\":{\"input_tokens\":12,\"output_tokens\":1056,\"total_tokens\":1068," + + "\"input_tokens_details\":{\"text_tokens\":12,\"image_tokens\":0}}}"; + } + bytes(ex, status, "application/json", json.getBytes(StandardCharsets.UTF_8)); + } + + /** Multipart bodies carry binary parts; keep only the part names and text values. */ + private static String summarise(String body) { + StringBuilder sb = new StringBuilder(); + for (String part : body.split("--[A-Za-z0-9'()+_,./:=?-]{10,}")) { + int nameAt = part.indexOf("name=\""); + if (nameAt < 0) continue; + String name = part.substring(nameAt + 6, part.indexOf('"', nameAt + 6)); + boolean file = part.contains("filename="); + int bodyAt = part.indexOf("\r\n\r\n"); + String value = bodyAt < 0 ? "" : part.substring(bodyAt + 4).trim(); + sb.append(name).append('=').append(file ? "" : value).append("; "); + } + return sb.toString(); + } + + private void endless(HttpExchange ex) throws IOException { + ex.getResponseHeaders().add("Content-Type", "image/png"); + ex.sendResponseHeaders(200, 0); + byte[] chunk = new byte[64 * 1024]; + try (OutputStream out = ex.getResponseBody()) { + for (int i = 0; i < 4096; i++) { // up to 256 MB; the client should hang up long before + out.write(chunk); + } + } catch (IOException ignored) { + // the client stopped reading, which is the point + } + } + + private static void bytes(HttpExchange ex, int status, String type, byte[] data) throws IOException { + ex.getResponseHeaders().add("Content-Type", type); + ex.sendResponseHeaders(status, data.length); + try (OutputStream out = ex.getResponseBody()) { + out.write(data); + } + } +} diff --git a/image-gen/src/test/java/com/ankurm/imagegen/support/Fixtures.java b/image-gen/src/test/java/com/ankurm/imagegen/support/Fixtures.java new file mode 100644 index 0000000..50b7374 --- /dev/null +++ b/image-gen/src/test/java/com/ankurm/imagegen/support/Fixtures.java @@ -0,0 +1,31 @@ +package com.ankurm.imagegen.support; + +import java.nio.charset.StandardCharsets; +import java.util.Set; + +import com.ankurm.imagegen.ImagePolicy; +import com.ankurm.imagegen.ImageService; +import com.ankurm.imagegen.ImageStore; +import com.ankurm.imagegen.OpenAiOptionsFactory; +import com.ankurm.imagegen.PromptGate; +import com.ankurm.imagegen.RemoteFetcher; +import org.springframework.ai.image.ImageModel; +import org.springframework.ai.moderation.ModerationModel; + +public final class Fixtures { + + private Fixtures() {} + + public static byte[] utf8(String s) { + return s.getBytes(StandardCharsets.UTF_8); + } + + public static ImageService service(ImageModel model, ModerationModel moderation, ImagePolicy policy, ImageStore store) { + return new ImageService(model, new OpenAiOptionsFactory("gpt-image-test"), new PromptGate(moderation, policy), + policy, new RemoteFetcher(policy), store); + } + + public static ImagePolicy withHost(ImagePolicy p, String host) { + return p.withFetch(Set.of(host), true); + } +} diff --git a/image-gen/src/test/java/com/ankurm/imagegen/support/Images.java b/image-gen/src/test/java/com/ankurm/imagegen/support/Images.java new file mode 100644 index 0000000..ab51212 --- /dev/null +++ b/image-gen/src/test/java/com/ankurm/imagegen/support/Images.java @@ -0,0 +1,72 @@ +package com.ankurm.imagegen.support; + +import java.awt.Color; +import java.awt.image.BufferedImage; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import javax.imageio.ImageIO; + +/** Real, decodable image bytes made in memory - the "generated" pictures in every test. */ +public final class Images { + + private Images() {} + + public static byte[] png(int w, int h) { + return encode(w, h, "png"); + } + + public static byte[] jpeg(int w, int h) { + return encode(w, h, "jpg"); + } + + private static byte[] encode(int w, int h, String fmt) { + BufferedImage img = new BufferedImage(w, h, BufferedImage.TYPE_INT_RGB); + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) { + img.setRGB(x, y, new Color((x * 255) / Math.max(1, w - 1), (y * 255) / Math.max(1, h - 1), 128).getRGB()); + } + } + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + ImageIO.write(img, fmt, out); + return out.toByteArray(); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + /** Smallest RIFF/WEBP header the sniffer recognises (not a decodable WebP; the verifier does not decode WebP). */ + public static byte[] webpHeaderOnly() { + byte[] b = new byte[32]; + System.arraycopy("RIFF".getBytes(StandardCharsets.US_ASCII), 0, b, 0, 4); + System.arraycopy("WEBP".getBytes(StandardCharsets.US_ASCII), 0, b, 8, 4); + return b; + } + + /** A valid PNG whose IHDR has been rewritten to claim w x h. The pixel data no longer matches. */ + public static byte[] pngClaiming(int w, int h) { + byte[] b = png(8, 8); + put32(b, 16, w); + put32(b, 20, h); + return b; + } + + public static byte[] concat(byte[] a, byte[] b) { + byte[] out = new byte[a.length + b.length]; + System.arraycopy(a, 0, out, 0, a.length); + System.arraycopy(b, 0, out, a.length, b.length); + return out; + } + + public static String b64(byte[] b) { + return Base64.getEncoder().encodeToString(b); + } + + private static void put32(byte[] b, int i, int v) { + b[i] = (byte) (v >>> 24); + b[i + 1] = (byte) (v >>> 16); + b[i + 2] = (byte) (v >>> 8); + b[i + 3] = (byte) v; + } +} diff --git a/image-gen/src/test/java/com/ankurm/imagegen/support/Minio.java b/image-gen/src/test/java/com/ankurm/imagegen/support/Minio.java new file mode 100644 index 0000000..28ef1e2 --- /dev/null +++ b/image-gen/src/test/java/com/ankurm/imagegen/support/Minio.java @@ -0,0 +1,77 @@ +package com.ankurm.imagegen.support; + +import java.net.URI; +import java.time.Duration; + +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Configuration; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; + +/** + * A real MinIO server in Docker (Testcontainers). The image is Chainguard's build of upstream MinIO + * because minio/minio and quay.io/minio/minio were not pullable from the build sandbox; the version + * is printed in output/05-store-minio.txt. Override with -Dminio.image=... to use another. + */ +public final class Minio implements AutoCloseable { + + public static final String USER = "minio-test-user"; + public static final String PASSWORD = "minio-test-password-not-real"; + + private final GenericContainer container = new GenericContainer<>( + DockerImageName.parse(System.getProperty("minio.image", "chainguard/minio:latest"))) + .withEnv("MINIO_ROOT_USER", USER) + .withEnv("MINIO_ROOT_PASSWORD", PASSWORD) + .withCommand("server", "/tmp/data") + .withExposedPorts(9000) + .waitingFor(Wait.forHttp("/minio/health/live").forPort(9000).withStartupTimeout(Duration.ofSeconds(90))); + + public Minio start() { + container.start(); + return this; + } + + public String endpoint() { + return "http://" + container.getHost() + ":" + container.getMappedPort(9000); + } + + public String imageName() { + return container.getDockerImageName(); + } + + public String version() { + try { + return container.execInContainer("/usr/bin/minio", "--version").getStdout().lines().findFirst().orElse("?"); + } catch (Exception e) { + return "unknown (" + e.getClass().getSimpleName() + ")"; + } + } + + public S3Client client() { + return S3Client.builder() + .endpointOverride(URI.create(endpoint())) + .region(Region.US_EAST_1) + .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(USER, PASSWORD))) + .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()) + .build(); + } + + public S3Presigner presigner() { + return S3Presigner.builder() + .endpointOverride(URI.create(endpoint())) + .region(Region.US_EAST_1) + .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(USER, PASSWORD))) + .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()) + .build(); + } + + @Override + public void close() { + container.stop(); + } +} diff --git a/image-gen/src/test/java/com/ankurm/imagegen/support/ScriptedImageModel.java b/image-gen/src/test/java/com/ankurm/imagegen/support/ScriptedImageModel.java new file mode 100644 index 0000000..23c2faf --- /dev/null +++ b/image-gen/src/test/java/com/ankurm/imagegen/support/ScriptedImageModel.java @@ -0,0 +1,76 @@ +package com.ankurm.imagegen.support; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; + +import org.springframework.ai.image.Image; +import org.springframework.ai.image.ImageGeneration; +import org.springframework.ai.image.ImageModel; +import org.springframework.ai.image.ImagePrompt; +import org.springframework.ai.image.ImageResponse; +import org.springframework.ai.openai.metadata.OpenAiImageGenerationMetadata; + +/** + * A hand-written {@link ImageModel}: it returns the responses it was given, in order, and records + * every {@link ImagePrompt} it receives. No network, no provider, no randomness. + */ +public final class ScriptedImageModel implements ImageModel { + + private final java.util.LinkedList script = new java.util.LinkedList<>(); + private final List seen = new ArrayList<>(); + + public ScriptedImageModel thenB64(byte[]... images) { + return thenB64Revised(null, images); + } + + public ScriptedImageModel thenB64Revised(String revisedPrompt, byte[]... images) { + List gens = new ArrayList<>(); + for (byte[] b : images) { + gens.add(new ImageGeneration(new Image(null, Images.b64(b)), + revisedPrompt == null ? null : new OpenAiImageGenerationMetadata(revisedPrompt))); + } + script.add(new ImageResponse(gens)); + return this; + } + + public ScriptedImageModel thenRawB64(String b64) { + script.add(new ImageResponse(List.of(new ImageGeneration(new Image(null, b64))))); + return this; + } + + public ScriptedImageModel thenUrl(String url) { + script.add(new ImageResponse(List.of(new ImageGeneration(new Image(url, null))))); + return this; + } + + public ScriptedImageModel thenThrow(RuntimeException e) { + script.add(null); + throwables.add(e); + return this; + } + + private final Queue throwables = new java.util.LinkedList<>(); + + @Override + public ImageResponse call(ImagePrompt prompt) { + seen.add(prompt); + if (script.isEmpty()) { + throw new IllegalStateException("ScriptedImageModel has no response left for call #" + seen.size()); + } + ImageResponse next = script.poll(); + if (next == null) { + throw throwables.poll(); + } + return next; + } + + public List seen() { + return seen; + } + + public int calls() { + return seen.size(); + } +} diff --git a/image-gen/src/test/java/com/ankurm/imagegen/support/ScriptedModerationModel.java b/image-gen/src/test/java/com/ankurm/imagegen/support/ScriptedModerationModel.java new file mode 100644 index 0000000..287c208 --- /dev/null +++ b/image-gen/src/test/java/com/ankurm/imagegen/support/ScriptedModerationModel.java @@ -0,0 +1,56 @@ +package com.ankurm.imagegen.support; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; + +import org.springframework.ai.moderation.Categories; +import org.springframework.ai.moderation.Generation; +import org.springframework.ai.moderation.Moderation; +import org.springframework.ai.moderation.ModerationModel; +import org.springframework.ai.moderation.ModerationPrompt; +import org.springframework.ai.moderation.ModerationResponse; +import org.springframework.ai.moderation.ModerationResult; + +/** + * A hand-written {@link ModerationModel}. A rule maps the prompt text to a flagged category name + * (or null for "clean"). It is a stand-in for OpenAiModerationModel; it judges nothing. + */ +public final class ScriptedModerationModel implements ModerationModel { + + private final Function rule; + private final List seen = new ArrayList<>(); + private RuntimeException failure; + + public ScriptedModerationModel(Function rule) { + this.rule = rule; + } + + public static ScriptedModerationModel flagging(String word, String category) { + return new ScriptedModerationModel(p -> p.toLowerCase().contains(word) ? category : null); + } + + public ScriptedModerationModel failingWith(RuntimeException e) { + this.failure = e; + return this; + } + + @Override + public ModerationResponse call(ModerationPrompt prompt) { + String text = prompt.getInstructions().getText(); + seen.add(text); + if (failure != null) throw failure; + String category = rule.apply(text); + Categories.Builder c = Categories.builder(); + if ("violence".equals(category)) c.violence(true); + if ("sexual".equals(category)) c.sexual(true); + if ("hate".equals(category)) c.hate(true); + ModerationResult r = ModerationResult.builder().flagged(category != null).categories(c.build()).build(); + Moderation m = Moderation.builder().id("scripted").model("scripted").results(List.of(r)).build(); + return new ModerationResponse(new Generation(m)); + } + + public List seen() { + return seen; + } +} diff --git a/image-gen/src/test/java/com/ankurm/imagegen/support/Transcript.java b/image-gen/src/test/java/com/ankurm/imagegen/support/Transcript.java new file mode 100644 index 0000000..3fdea7a --- /dev/null +++ b/image-gen/src/test/java/com/ankurm/imagegen/support/Transcript.java @@ -0,0 +1,46 @@ +package com.ankurm.imagegen.support; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Writes a numbered transcript under {@code output/} (module root, not {@code docs/}) and echoes + * it to the console. Every console block quoted in the article comes out of one of these files. + */ +public final class Transcript implements AutoCloseable { + + private final Path path; + private final StringWriter buffer = new StringWriter(); + private final PrintWriter out = new PrintWriter(buffer); + + public Transcript(String fileName, String title) { + this.path = Path.of("output", fileName); + out.println("# " + title); + out.println(); + } + + public Transcript line(String format, Object... args) { + out.println(args.length == 0 ? format : String.format(format, args)); + return this; + } + + public Transcript blank() { + out.println(); + return this; + } + + @Override + public void close() { + out.flush(); + try { + Files.createDirectories(path.getParent()); + Files.writeString(path, buffer.toString()); + } catch (IOException e) { + throw new IllegalStateException("could not write " + path, e); + } + System.out.print(buffer); + } +}