From 008211f2fa8fbfecb0c52cd6c49edc6dedddc326 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 17 Oct 2026 15:41:24 +0000 Subject: [PATCH] Add file-transfer module: multipart limits, filename handling, Range downloads, streaming under -Xmx64m, S3 presigned URLs Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VRLbHYdXVEBzGVhYq3pz4q --- README.md | 1 + file-transfer/.gitignore | 2 + file-transfer/README.md | 25 +++ file-transfer/cap/01-limits.txt | 6 + file-transfer/cap/02-filename.txt | 4 + file-transfer/cap/03-download.txt | 12 ++ file-transfer/cap/04-presign.txt | 18 ++ file-transfer/cap/05-streaming.txt | 4 + file-transfer/cap/06-big-upload.txt | 3 + file-transfer/pom.xml | 67 +++++++ file-transfer/scripts/run-all.sh | 36 ++++ .../filetransfer/DownloadController.java | 42 +++++ .../com/ankurm/filetransfer/FileStore.java | 72 ++++++++ .../filetransfer/FileTransferApplication.java | 12 ++ .../filetransfer/PresignController.java | 42 +++++ .../ankurm/filetransfer/S3PresignService.java | 68 ++++++++ .../ankurm/filetransfer/StreamController.java | 39 +++++ .../ankurm/filetransfer/UploadController.java | 49 ++++++ .../com/ankurm/filetransfer/UploadErrors.java | 20 +++ .../src/main/resources/application.properties | 13 ++ .../java/com/ankurm/filetransfer/Cap.java | 89 ++++++++++ .../java/com/ankurm/filetransfer/FakeS3.java | 164 ++++++++++++++++++ .../filetransfer/FilenameAndDownloadTest.java | 92 ++++++++++ .../filetransfer/MultipartLimitsTest.java | 74 ++++++++ .../ankurm/filetransfer/PresignedUrlTest.java | 118 +++++++++++++ 25 files changed, 1072 insertions(+) create mode 100644 file-transfer/.gitignore create mode 100644 file-transfer/README.md create mode 100644 file-transfer/cap/01-limits.txt create mode 100644 file-transfer/cap/02-filename.txt create mode 100644 file-transfer/cap/03-download.txt create mode 100644 file-transfer/cap/04-presign.txt create mode 100644 file-transfer/cap/05-streaming.txt create mode 100644 file-transfer/cap/06-big-upload.txt create mode 100644 file-transfer/pom.xml create mode 100755 file-transfer/scripts/run-all.sh create mode 100644 file-transfer/src/main/java/com/ankurm/filetransfer/DownloadController.java create mode 100644 file-transfer/src/main/java/com/ankurm/filetransfer/FileStore.java create mode 100644 file-transfer/src/main/java/com/ankurm/filetransfer/FileTransferApplication.java create mode 100644 file-transfer/src/main/java/com/ankurm/filetransfer/PresignController.java create mode 100644 file-transfer/src/main/java/com/ankurm/filetransfer/S3PresignService.java create mode 100644 file-transfer/src/main/java/com/ankurm/filetransfer/StreamController.java create mode 100644 file-transfer/src/main/java/com/ankurm/filetransfer/UploadController.java create mode 100644 file-transfer/src/main/java/com/ankurm/filetransfer/UploadErrors.java create mode 100644 file-transfer/src/main/resources/application.properties create mode 100644 file-transfer/src/test/java/com/ankurm/filetransfer/Cap.java create mode 100644 file-transfer/src/test/java/com/ankurm/filetransfer/FakeS3.java create mode 100644 file-transfer/src/test/java/com/ankurm/filetransfer/FilenameAndDownloadTest.java create mode 100644 file-transfer/src/test/java/com/ankurm/filetransfer/MultipartLimitsTest.java create mode 100644 file-transfer/src/test/java/com/ankurm/filetransfer/PresignedUrlTest.java diff --git a/README.md b/README.md index c90dd02..779e59a 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ files. | [`kotlin/`](kotlin) | [Kotlin with Spring Boot 4.1: Null Safety, Coroutines, and What Java Developers Get Wrong](https://ankurm.com/) | Kotlin 2.3.21 (Boot 4.1.1's own managed version, not the latest GA), the real `@Configuration class may not be final` and `@Bean method must not be private or final` failures without the allopen/noarg `spring`/`jpa` compiler plugins, `suspend fun`/`Flow` controller methods proven to still resume off a Tomcat thread under plain `spring-boot-starter-webmvc`, a JSpecify `@NullMarked` Java method that still NPEs at runtime because the contract isn't runtime-checked, and `TestRestTemplate` confirmed absent from Boot 4.1's `spring-boot-test` jar | | [`testing-no-containers/`](testing-no-containers) | [Testing Spring Boot 4 Without Containers: WireMock, MockWebServer and @MockitoBean](https://ankurm.com/) | `@MockBean`'s total removal proven with a real `javac` failure, `@MockitoBean`/`@MockitoSpyBean` context-cache behavior proven with real `org.springframework.test.context.cache` DEBUG log lines, WireMock fault injection against a real `RestClient`, a real `NoClassDefFoundError` from mixing `wiremock` with `wiremock-jetty12` (fixed with `wiremock-standalone`), and MockWebServer's `SocketEffect.ShutdownConnection` for genuinely severing a connection mid-response | | [`rest-test-client/`](rest-test-client) | [RestTestClient in Spring Framework 7: Testing REST APIs End-to-End (vs MockMvcTester)](https://ankurm.com/resttestclient-spring-framework-7-end-to-end-testing/) | one fluent `RestTestClient` API proven across `bindToController`, `bindTo(mockMvc)`, `bindToApplicationContext` and `bindToServer`; `RestTestClient`, `TestRestTemplate` and `@WebMvcTest` confirmed relocated into new Spring Boot 4 modules by unzipping the real jars; two distinct real failures for the two now-separate `@AutoConfigureRestTestClient`/`@AutoConfigureTestRestTemplate` annotations; and the JDK `HttpClient` Boot picks for `bindToServer` caught transparently decoding gzip and hiding `Content-Encoding`, verified against `curl` and against the older `SimpleClientHttpRequestFactory` | +| [`file-transfer/`](file-transfer) | [File Upload and Download in Spring Boot 4: Multipart, Streaming and S3 Presigned URLs](https://ankurm.com/file-upload-download-spring-boot-4-multipart-streaming-s3-presigned-urls/) | multipart limits enforced by Tomcat (and ignored by `MockMvc`), the client-controlled filename, `Range` downloads, a 200 MB download and 150 MB upload under `-Xmx64m`, and S3 presigned URLs checked against a local SigV4 verifier (not a real S3) | `core-di`, `core-beans`, `custom-starter`, `core-events`, `multi-datasource`, `i18n`, `aot-cache`, `kotlin`, `testing-no-containers`, `rest-test-client` and `hexagonal` are the exception: they have **no `docs/NN-topic.md` chapter folder**. Their deeper material lives in collapsible sections inside the articles themselves. Where their captured output sits is not diff --git a/file-transfer/.gitignore b/file-transfer/.gitignore new file mode 100644 index 0000000..1abd88b --- /dev/null +++ b/file-transfer/.gitignore @@ -0,0 +1,2 @@ +target/ +cap/app.log diff --git a/file-transfer/README.md b/file-transfer/README.md new file mode 100644 index 0000000..4a1ef1e --- /dev/null +++ b/file-transfer/README.md @@ -0,0 +1,25 @@ +# file-transfer + +Companion code for **[File Upload and Download in Spring Boot 4: Multipart, Streaming and S3 Presigned URLs](https://ankurm.com/file-upload-download-spring-boot-4-multipart-streaming-s3-presigned-urls/)**. + +Spring Boot 4.1.1, Spring Framework 7.0.9, Tomcat 11.0.24, JDK 25, AWS SDK for Java 2.56.0. No Docker needed. + +| Area | Code | What the run shows | +|---|---|---| +| Multipart limits | `UploadController`, `UploadErrors`, `MultipartLimitsTest` | the 1 MB / 2 MB limits are enforced by Tomcat, so `MockMvc` accepts a file the real server rejects; the rejected-upload race (`cap/01-limits.txt`) | +| Filenames | `FileStore`, `FilenameAndDownloadTest` | `getOriginalFilename()` is whatever the client sent, including `../../`; the file is stored under a UUID (`cap/02-filename.txt`) | +| Download | `DownloadController` | `Resource` gives `Range`/206/416 for free; `Content-Disposition` encoding (`cap/03-download.txt`) | +| Presigned URLs | `S3PresignService`, `PresignController`, `FakeS3`, `PresignedUrlTest` | what is and is not covered by the signature; expiry; reuse (`cap/04-presign.txt`) | +| Streaming | `StreamController`, `scripts/run-all.sh` | 200 MB download and a 150 MB upload under `-Xmx64m` (`cap/05-streaming.txt`, `cap/06-big-upload.txt`) | + +```bash +./scripts/run-all.sh # tests, then the -Xmx64m experiments; rewrites cap/*.txt +``` + +## What was and was not run + +- The presigned URLs are produced by the real AWS SDK presigner and checked by `FakeS3`, a small in-process server + that recomputes the AWS Signature V4 query signature from the published algorithm. That proves the URLs are consistent + with the algorithm as documented. It does **not** prove a real S3 or MinIO server accepts them: this environment has + no Docker and no S3-compatible server, so nothing here was run against one. +- `app.s3.access-key` and `app.s3.secret-key` are the example keys from the AWS documentation. They are not credentials. diff --git a/file-transfer/cap/01-limits.txt b/file-transfer/cap/01-limits.txt new file mode 100644 index 0000000..95b88e6 --- /dev/null +++ b/file-transfer/cap/01-limits.txt @@ -0,0 +1,6 @@ +limits: max-file-size=1MB, max-request-size=2MB +real HTTP, one 500 KB file -> status=201 body={"id":"d8d0b4b8-d530-42aa-b005-9e1b747449a7","originalFilename":"a.txt","displayName":"a.txt","size":512000,"declaredContentType":"text/plai... +real HTTP, one 1.5 MB file -> status=413 body={"detail":"The upload is larger than the configured limit (spring.servlet.multipart.max-file-size / max-request-size).","instance":"/files",... +real HTTP, 3 x 900 KB in one request -> status=413 body={"detail":"The upload is larger than the configured limit (spring.servlet.multipart.max-file-size / max-request-size).","instance":"/files",... +real HTTP, one 10 MB file, 10 attempts -> 7 x status=413, 3 x connection error +MockMvc, the same 1.5 MB file -> status=201 diff --git a/file-transfer/cap/02-filename.txt b/file-transfer/cap/02-filename.txt new file mode 100644 index 0000000..0d3c120 --- /dev/null +++ b/file-transfer/cap/02-filename.txt @@ -0,0 +1,4 @@ +client sent filename=../../etc/cron.d/evil.txt Content-Type=image/png +server response: {"id":"4980219c-b92a-46d4-a5ab-b5d7636fb7f0","originalFilename":"../../etc/cron.d/evil.txt","displayName":"evil.txt","size":5,"declaredContentType":"image/png"} +root.resolve(originalFilename).normalize() stays inside the storage dir: false +client sent a Windows-style path; server response: {"id":"d436e57e-6855-4fff-b5ca-b9dd0fec25c2","originalFilename":"..\\..\\windows\\system32\\evil.dll","displayName":"evil.dll","size":1,"declaredContentType":"text/plain"} diff --git a/file-transfer/cap/03-download.txt b/file-transfer/cap/03-download.txt new file mode 100644 index 0000000..747b2b0 --- /dev/null +++ b/file-transfer/cap/03-download.txt @@ -0,0 +1,12 @@ +GET /files/{id} + status=200 body=ABCDEFGHIJKLMNOPQRSTUVWXYZ + content-type: application/octet-stream + content-length: 26 + content-disposition: attachment; filename="resume 2026.txt"; filename*=UTF-8''r%C3%A9sum%C3%A9%202026.txt + accept-ranges: bytes + x-content-type-options: nosniff +GET /files/{id} with Range: bytes=10-19 + status=206 body=KLMNOPQRST + content-range: bytes 10-19/26 +GET /files/{id} with Range: bytes=500- + status=416 content-range: bytes */26 diff --git a/file-transfer/cap/04-presign.txt b/file-transfer/cap/04-presign.txt new file mode 100644 index 0000000..7e87b35 --- /dev/null +++ b/file-transfer/cap/04-presign.txt @@ -0,0 +1,18 @@ +PUT a URL signed for 1 second, sent after 2.2 seconds -> status=403 AccessDenied +POST /presign/upload {"contentType":"image/png"} -> status=200 + method=PUT expiresInSeconds=300 headers={"content-type":"image/png"} + url path: http:///uploads/4dfdf0af-141a-4d1b-9d35-3609a9385791 + query: X-Amz-Algorithm=AWS4-HMAC-SHA256 + query: X-Amz-Date=20261017T154013Z + query: X-Amz-SignedHeaders=content-type%3Bhost + query: X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20261017%2Fus-east-1%2Fs3%2Faws4_request + query: X-Amz-Expires=300 + query: X-Amz-Signature=4e5c56c29980cf521e477119788a0616c392deac4686df7259957af9f1445203 +PUT with the signed Content-Type (image/png) -> status=200 +PUT the same URL again (presigned URLs are reusable) -> status=200 +PUT with a different Content-Type (text/html) -> status=403 SignatureDoesNotMatch +PUT with no Content-Type header -> status=403 SignatureDoesNotMatch +PUT after changing the last character of the key -> status=403 SignatureDoesNotMatch +PUT after raising X-Amz-Expires to 7 days -> status=403 SignatureDoesNotMatch +GET presigned URL -> status=200 bytes match upload=true content-disposition=attachment; filename="holiday.png" +disallowed type: POST /presign/upload text/html -> status=400 diff --git a/file-transfer/cap/05-streaming.txt b/file-transfer/cap/05-streaming.txt new file mode 100644 index 0000000..b6a5636 --- /dev/null +++ b/file-transfer/cap/05-streaming.txt @@ -0,0 +1,4 @@ +JVM started with -Xmx64m +GET /buffered/200 (200 MB) -> status=500 downloaded=108 bytes in 0.273337s +GET /stream/200 (200 MB) -> status=200 downloaded=209715200 bytes in 0.337059s +GET /stream/1 afterwards -> status=200 diff --git a/file-transfer/cap/06-big-upload.txt b/file-transfer/cap/06-big-upload.txt new file mode 100644 index 0000000..8396f47 --- /dev/null +++ b/file-transfer/cap/06-big-upload.txt @@ -0,0 +1,3 @@ +JVM started with -Xmx64m, multipart limits removed, uploading a 150 MB file +multipart POST /files -> status=201 sent=157286616 bytes in 1.518657s +raw POST /files/raw -> status=201 sent=157286400 bytes in 1.850810s diff --git a/file-transfer/pom.xml b/file-transfer/pom.xml new file mode 100644 index 0000000..b59b078 --- /dev/null +++ b/file-transfer/pom.xml @@ -0,0 +1,67 @@ + + + 4.0.0 + + + org.springframework.boot + spring-boot-starter-parent + 4.1.1 + + + + com.ankurm + file-transfer + 1.0.0 + file-transfer + File upload and download in Spring Boot 4.1: multipart limits, filename handling, Range + downloads, StreamingResponseBody under a 64 MB heap, and S3 presigned URLs checked against a local + SigV4 verifier. + jar + + + 25 + 2.56.0 + + + + + + software.amazon.awssdk + bom + ${awssdk.version} + pom + import + + + + + + + org.springframework.boot + spring-boot-starter-webmvc + + + + + software.amazon.awssdk + s3 + + + + org.springframework.boot + spring-boot-starter-webmvc-test + test + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + diff --git a/file-transfer/scripts/run-all.sh b/file-transfer/scripts/run-all.sh new file mode 100755 index 0000000..5eb9fd6 --- /dev/null +++ b/file-transfer/scripts/run-all.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Runs the whole module: the tests (which write cap/01..04), then the -Xmx64m streaming experiment (cap/05..06). +# No Docker needed. Takes a couple of minutes. +set -euo pipefail +cd "$(dirname "$0")/.." +mkdir -p cap +mvn -q -B test +mvn -q -B package -DskipTests +JAR=$(ls target/file-transfer-*.jar | grep -v original | head -1) + +run_app() { # $1 = extra args; sets APP_PID + java -Xmx64m -jar "$JAR" --server.port=18080 $1 > cap/app.log 2>&1 & + APP_PID=$! + for _ in $(seq 1 60); do curl -s -o /dev/null http://localhost:18080/stream/1 && break; sleep 0.5; done +} + +OUT=cap/05-streaming.txt; : > "$OUT" +run_app "" +echo "JVM started with -Xmx64m" >> "$OUT" +for kind in buffered stream; do + R=$(curl -s -o /dev/null -w "status=%{http_code} downloaded=%{size_download} bytes in %{time_total}s" http://localhost:18080/$kind/200 || true) + echo "GET /$kind/200 (200 MB) -> $R" >> "$OUT" +done +echo "GET /stream/1 afterwards -> $(curl -s -o /dev/null -w 'status=%{http_code}' http://localhost:18080/stream/1)" >> "$OUT" +kill $APP_PID; wait $APP_PID 2>/dev/null || true + +OUT=cap/06-big-upload.txt; : > "$OUT" +head -c 157286400 /dev/zero > /tmp/ft-150mb.bin +run_app "--spring.servlet.multipart.max-file-size=-1 --spring.servlet.multipart.max-request-size=-1" +echo "JVM started with -Xmx64m, multipart limits removed, uploading a 150 MB file" >> "$OUT" +R=$(curl -s -o /dev/null -w "status=%{http_code} sent=%{size_upload} bytes in %{time_total}s" -F "file=@/tmp/ft-150mb.bin;type=application/octet-stream" http://localhost:18080/files || true) +echo "multipart POST /files -> $R" >> "$OUT" +R=$(curl -s -o /dev/null -w "status=%{http_code} sent=%{size_upload} bytes in %{time_total}s" -H "X-Filename: big.bin" --data-binary @/tmp/ft-150mb.bin -H "Content-Type: application/octet-stream" http://localhost:18080/files/raw || true) +echo "raw POST /files/raw -> $R" >> "$OUT" +kill $APP_PID; wait $APP_PID 2>/dev/null || true +rm -f /tmp/ft-150mb.bin diff --git a/file-transfer/src/main/java/com/ankurm/filetransfer/DownloadController.java b/file-transfer/src/main/java/com/ankurm/filetransfer/DownloadController.java new file mode 100644 index 0000000..f415a0e --- /dev/null +++ b/file-transfer/src/main/java/com/ankurm/filetransfer/DownloadController.java @@ -0,0 +1,42 @@ +package com.ankurm.filetransfer; + +import java.nio.charset.StandardCharsets; + +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.Resource; +import org.springframework.http.ContentDisposition; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.server.ResponseStatusException; + +@RestController +public class DownloadController { + + private final FileStore store; + + public DownloadController(FileStore store) { + this.store = store; + } + + /** + * Returning a Resource gets Range support for free: Spring answers "Range: bytes=..." with 206 Partial Content. + * The content type is fixed to octet-stream on purpose: echoing back the type the client declared at upload + * would let an uploader make the browser render their file as HTML. + */ + @GetMapping("/files/{id}") + public ResponseEntity download(@PathVariable String id) { + FileStore.StoredFile f = store.find(id) + .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "no such file")); + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_DISPOSITION, + ContentDisposition.attachment().filename(f.displayName(), StandardCharsets.UTF_8).build().toString()) + .header("X-Content-Type-Options", "nosniff") + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .body(new FileSystemResource(f.path())); + } +} diff --git a/file-transfer/src/main/java/com/ankurm/filetransfer/FileStore.java b/file-transfer/src/main/java/com/ankurm/filetransfer/FileStore.java new file mode 100644 index 0000000..79a8def --- /dev/null +++ b/file-transfer/src/main/java/com/ankurm/filetransfer/FileStore.java @@ -0,0 +1,72 @@ +package com.ankurm.filetransfer; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +/** + * Stores uploads on local disk under a server-generated id. The client's filename is never used as a path: + * it is kept only as display metadata, after stripping any directory part. + */ +@Service +public class FileStore { + + public record StoredFile(String id, String originalFilename, String displayName, long size, + String declaredContentType, Path path) { } + + private final Path root; + private final Map files = new ConcurrentHashMap<>(); + + public FileStore(@Value("${app.storage-dir:}") String dir) throws IOException { + this.root = (dir.isBlank() ? Files.createTempDirectory("file-transfer") : Path.of(dir)) + .toAbsolutePath().normalize(); + Files.createDirectories(root); + } + + public Path root() { + return root; + } + + public StoredFile save(MultipartFile file) throws IOException { + String id = UUID.randomUUID().toString(); + Path target = root.resolve(id); + file.transferTo(target); + return remember(id, file.getOriginalFilename(), Files.size(target), file.getContentType(), target); + } + + public StoredFile saveStream(InputStream in, String name) throws IOException { + String id = UUID.randomUUID().toString(); + Path target = root.resolve(id); + long size = Files.copy(in, target); + return remember(id, name, size, null, target); + } + + public Optional find(String id) { + return Optional.ofNullable(files.get(id)); + } + + private StoredFile remember(String id, String original, long size, String contentType, Path path) { + StoredFile stored = new StoredFile(id, original, displayName(original), size, contentType, path); + files.put(id, stored); + return stored; + } + + /** Last path segment only, control characters removed. Handles both / and \ separators. */ + static String displayName(String raw) { + if (raw == null || raw.isBlank()) { + return "unnamed"; + } + String name = raw.replace('\\', '/'); + name = name.substring(name.lastIndexOf('/') + 1).replaceAll("\\p{Cntrl}", ""); + return name.isBlank() ? "unnamed" : name; + } +} diff --git a/file-transfer/src/main/java/com/ankurm/filetransfer/FileTransferApplication.java b/file-transfer/src/main/java/com/ankurm/filetransfer/FileTransferApplication.java new file mode 100644 index 0000000..354818d --- /dev/null +++ b/file-transfer/src/main/java/com/ankurm/filetransfer/FileTransferApplication.java @@ -0,0 +1,12 @@ +package com.ankurm.filetransfer; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class FileTransferApplication { + + public static void main(String[] args) { + SpringApplication.run(FileTransferApplication.class, args); + } +} diff --git a/file-transfer/src/main/java/com/ankurm/filetransfer/PresignController.java b/file-transfer/src/main/java/com/ankurm/filetransfer/PresignController.java new file mode 100644 index 0000000..eb398a5 --- /dev/null +++ b/file-transfer/src/main/java/com/ankurm/filetransfer/PresignController.java @@ -0,0 +1,42 @@ +package com.ankurm.filetransfer; + +import java.time.Duration; +import java.util.Map; +import java.util.Set; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.server.ResponseStatusException; + +@RestController +public class PresignController { + + public record UploadRequest(String contentType) { } + + private static final Set ALLOWED = Set.of("image/png", "image/jpeg", "application/pdf"); + private static final Duration TTL = Duration.ofMinutes(5); + + private final S3PresignService presign; + + public PresignController(S3PresignService presign) { + this.presign = presign; + } + + @PostMapping("/presign/upload") + public S3PresignService.Upload upload(@RequestBody UploadRequest request) { + if (request.contentType() == null || !ALLOWED.contains(request.contentType())) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "content type not allowed"); + } + return presign.forUpload(request.contentType(), TTL); + } + + @GetMapping("/presign/download/{key}") + public Map download(@PathVariable String key, @RequestParam(defaultValue = "download.bin") String filename) { + return Map.of("url", presign.forDownload(key, filename, TTL)); + } +} diff --git a/file-transfer/src/main/java/com/ankurm/filetransfer/S3PresignService.java b/file-transfer/src/main/java/com/ankurm/filetransfer/S3PresignService.java new file mode 100644 index 0000000..0371bb8 --- /dev/null +++ b/file-transfer/src/main/java/com/ankurm/filetransfer/S3PresignService.java @@ -0,0 +1,68 @@ +package com.ankurm.filetransfer; + +import java.net.URI; +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Configuration; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; +import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; +import software.amazon.awssdk.services.s3.presigner.model.PresignedPutObjectRequest; + +/** Signs S3 / MinIO URLs locally. Nothing here talks to the network. */ +@Service +public class S3PresignService implements AutoCloseable { + + public record Upload(String key, String url, String method, Map headers, long expiresInSeconds) { } + + private final S3Presigner presigner; + private final String bucket; + + public S3PresignService(@Value("${app.s3.endpoint}") String endpoint, @Value("${app.s3.bucket}") String bucket, + @Value("${app.s3.access-key}") String accessKey, @Value("${app.s3.secret-key}") String secretKey) { + this.bucket = bucket; + this.presigner = S3Presigner.builder() + .region(Region.US_EAST_1) + .endpointOverride(URI.create(endpoint)) + .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secretKey))) + .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()) + .build(); + } + + /** The object key is chosen here, never by the client. The content type is part of the signature. */ + public Upload forUpload(String contentType, Duration ttl) { + String key = UUID.randomUUID().toString(); + PutObjectRequest put = PutObjectRequest.builder().bucket(bucket).key(key).contentType(contentType).build(); + PresignedPutObjectRequest signed = presigner.presignPutObject(r -> r.signatureDuration(ttl).putObjectRequest(put)); + Map headers = new LinkedHashMap<>(); + for (Map.Entry> h : signed.signedHeaders().entrySet()) { + if (!h.getKey().equalsIgnoreCase("host")) { + headers.put(h.getKey(), String.join(",", h.getValue())); + } + } + return new Upload(key, signed.url().toString(), "PUT", headers, ttl.toSeconds()); + } + + public String forDownload(String key, String filename, Duration ttl) { + GetObjectRequest get = GetObjectRequest.builder().bucket(bucket).key(key) + .responseContentDisposition("attachment; filename=\"" + filename.replace("\"", "") + "\"").build(); + PresignedGetObjectRequest signed = presigner.presignGetObject(r -> r.signatureDuration(ttl).getObjectRequest(get)); + return signed.url().toString(); + } + + @Override + public void close() { + presigner.close(); + } +} diff --git a/file-transfer/src/main/java/com/ankurm/filetransfer/StreamController.java b/file-transfer/src/main/java/com/ankurm/filetransfer/StreamController.java new file mode 100644 index 0000000..0614643 --- /dev/null +++ b/file-transfer/src/main/java/com/ankurm/filetransfer/StreamController.java @@ -0,0 +1,39 @@ +package com.ankurm.filetransfer; + +import java.util.Arrays; + +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; + +/** The same N megabytes of data, sent two ways. Run both under a small heap and compare. */ +@RestController +public class StreamController { + + private static final int CHUNK = 64 * 1024; + + /** Streams in 64 KB chunks: memory use does not depend on the size of the file. */ + @GetMapping("/stream/{mb}") + public ResponseEntity stream(@PathVariable int mb) { + long total = (long) mb * 1024 * 1024; + StreamingResponseBody body = out -> { + byte[] chunk = new byte[CHUNK]; + Arrays.fill(chunk, (byte) 'x'); + for (long written = 0; written < total; written += CHUNK) { + out.write(chunk, 0, (int) Math.min(CHUNK, total - written)); + } + }; + return ResponseEntity.ok().contentType(MediaType.APPLICATION_OCTET_STREAM).contentLength(total).body(body); + } + + /** Builds the whole thing in memory first: the heap must hold the entire file. */ + @GetMapping(path = "/buffered/{mb}", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE) + public byte[] buffered(@PathVariable int mb) { + byte[] all = new byte[mb * 1024 * 1024]; + Arrays.fill(all, (byte) 'x'); + return all; + } +} diff --git a/file-transfer/src/main/java/com/ankurm/filetransfer/UploadController.java b/file-transfer/src/main/java/com/ankurm/filetransfer/UploadController.java new file mode 100644 index 0000000..b055d5d --- /dev/null +++ b/file-transfer/src/main/java/com/ankurm/filetransfer/UploadController.java @@ -0,0 +1,49 @@ +package com.ankurm.filetransfer; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; + +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.server.ResponseStatusException; + +@RestController +public class UploadController { + + public record UploadResult(String id, String originalFilename, String displayName, long size, + String declaredContentType) { } + + private final FileStore store; + + public UploadController(FileStore store) { + this.store = store; + } + + /** Multipart upload: the container parses the body and spills big parts to a temp file. */ + @PostMapping(path = "/files", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public ResponseEntity upload(@RequestParam("file") MultipartFile file) throws IOException { + if (file.isEmpty()) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "empty file"); + } + return created(store.save(file)); + } + + /** Raw upload: the request body is the file. No multipart parsing, so the multipart limits do not apply. */ + @PostMapping("/files/raw") + public ResponseEntity raw(InputStream body, + @RequestHeader(value = "X-Filename", defaultValue = "upload.bin") String name) throws IOException { + return created(store.saveStream(body, name)); + } + + private static ResponseEntity created(FileStore.StoredFile s) { + return ResponseEntity.created(URI.create("/files/" + s.id())) + .body(new UploadResult(s.id(), s.originalFilename(), s.displayName(), s.size(), s.declaredContentType())); + } +} diff --git a/file-transfer/src/main/java/com/ankurm/filetransfer/UploadErrors.java b/file-transfer/src/main/java/com/ankurm/filetransfer/UploadErrors.java new file mode 100644 index 0000000..620b7c6 --- /dev/null +++ b/file-transfer/src/main/java/com/ankurm/filetransfer/UploadErrors.java @@ -0,0 +1,20 @@ +package com.ankurm.filetransfer; + +import org.springframework.http.HttpStatus; +import org.springframework.http.ProblemDetail; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.multipart.MaxUploadSizeExceededException; + +/** Turns the container's size-limit failure into a JSON problem instead of an empty 413. */ +@RestControllerAdvice +public class UploadErrors { + + @ExceptionHandler(MaxUploadSizeExceededException.class) + public ProblemDetail tooLarge(MaxUploadSizeExceededException e) { + ProblemDetail pd = ProblemDetail.forStatusAndDetail(HttpStatus.CONTENT_TOO_LARGE, + "The upload is larger than the configured limit (spring.servlet.multipart.max-file-size / max-request-size)."); + pd.setTitle("Upload too large"); + return pd; + } +} diff --git a/file-transfer/src/main/resources/application.properties b/file-transfer/src/main/resources/application.properties new file mode 100644 index 0000000..c3ad3c8 --- /dev/null +++ b/file-transfer/src/main/resources/application.properties @@ -0,0 +1,13 @@ +# Per-file and per-request limits. Enforced by the servlet container (Tomcat), not by Spring MVC. +spring.servlet.multipart.max-file-size=1MB +spring.servlet.multipart.max-request-size=2MB + +# Where uploads land. Blank = a fresh temp directory per run. +app.storage-dir= + +# S3 / MinIO settings for the presigner. These are the example keys from the AWS documentation, +# not real credentials. Override all four for a real bucket. +app.s3.endpoint=http://localhost:9000 +app.s3.bucket=uploads +app.s3.access-key=AKIAIOSFODNN7EXAMPLE +app.s3.secret-key=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY diff --git a/file-transfer/src/test/java/com/ankurm/filetransfer/Cap.java b/file-transfer/src/test/java/com/ankurm/filetransfer/Cap.java new file mode 100644 index 0000000..fe9e055 --- /dev/null +++ b/file-transfer/src/test/java/com/ankurm/filetransfer/Cap.java @@ -0,0 +1,89 @@ +package com.ankurm.filetransfer; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; + +/** Test helpers: write the transcript to cap/, and build multipart bodies by hand so the test controls every byte. */ +final class Cap { + + static final Path DIR = Path.of("cap"); + static final HttpClient HTTP = HttpClient.newHttpClient(); + + private Cap() { } + + static void reset(String file) { + try { + Files.createDirectories(DIR); + Files.deleteIfExists(DIR.resolve(file)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + static void say(String file, String line) { + System.out.println(line); + try { + Files.writeString(DIR.resolve(file), line + System.lineSeparator(), + StandardOpenOption.CREATE, StandardOpenOption.APPEND); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + record Part(String field, String filename, String contentType, byte[] content) { } + + static byte[] multipart(String boundary, List parts) { + List chunks = new ArrayList<>(); + for (Part p : parts) { + String head = "--" + boundary + "\r\n" + + "Content-Disposition: form-data; name=\"" + p.field() + "\"; filename=\"" + p.filename() + "\"\r\n" + + "Content-Type: " + p.contentType() + "\r\n\r\n"; + chunks.add(head.getBytes(StandardCharsets.UTF_8)); + chunks.add(p.content()); + chunks.add("\r\n".getBytes(StandardCharsets.UTF_8)); + } + chunks.add(("--" + boundary + "--\r\n").getBytes(StandardCharsets.UTF_8)); + int len = chunks.stream().mapToInt(c -> c.length).sum(); + byte[] all = new byte[len]; + int pos = 0; + for (byte[] c : chunks) { + System.arraycopy(c, 0, all, pos, c.length); + pos += c.length; + } + return all; + } + + /** POST the parts to /files; returns "status=NNN body=..." or the exception if the connection broke. */ + static String postMultipart(String base, List parts) { + String boundary = "----ft" + System.nanoTime(); + HttpRequest req = HttpRequest.newBuilder(URI.create(base + "/files")) + .header("Content-Type", "multipart/form-data; boundary=" + boundary) + .POST(HttpRequest.BodyPublishers.ofByteArray(multipart(boundary, parts))).build(); + try { + HttpResponse r = HTTP.send(req, HttpResponse.BodyHandlers.ofString()); + String body = r.body().replaceAll("\\s+", " "); + return "status=" + r.statusCode() + " body=" + (body.length() > 140 ? body.substring(0, 140) + "..." : body); + } catch (IOException e) { + return "no response, " + e.getClass().getSimpleName() + ": " + e.getMessage(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return "interrupted"; + } + } + + static byte[] bytes(int n) { + byte[] b = new byte[n]; + java.util.Arrays.fill(b, (byte) 'a'); + return b; + } +} diff --git a/file-transfer/src/test/java/com/ankurm/filetransfer/FakeS3.java b/file-transfer/src/test/java/com/ankurm/filetransfer/FakeS3.java new file mode 100644 index 0000000..81df013 --- /dev/null +++ b/file-transfer/src/test/java/com/ankurm/filetransfer/FakeS3.java @@ -0,0 +1,164 @@ +package com.ankurm.filetransfer; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; + +/** + * A tiny stand-in for S3 / MinIO that accepts PUT and GET on /{bucket}/{key} and checks the AWS Signature V4 + * query-string signature, following the published algorithm. It exists because this build environment has no + * Docker. It proves the presigned URLs are self-consistent with the algorithm as documented; it does not prove + * that a real S3 or MinIO server would accept them. + */ +final class FakeS3 implements AutoCloseable { + + record Obj(byte[] data, String contentType) { } + + final Map objects = new ConcurrentHashMap<>(); + private final HttpServer server; + private final String secretKey; + + FakeS3(String secretKey) throws IOException { + this.secretKey = secretKey; + this.server = HttpServer.create(new InetSocketAddress("localhost", 0), 0); + this.server.createContext("/", this::handle); + this.server.start(); + } + + String endpoint() { + return "http://localhost:" + server.getAddress().getPort(); + } + + @Override + public void close() { + server.stop(0); + } + + private void handle(HttpExchange ex) throws IOException { + try { + String error = verify(ex); + if (error != null) { + String[] parts = error.split("\\|", 2); + reply(ex, 403, "" + parts[0] + "" + parts[1] + "", "application/xml", null); + return; + } + String path = ex.getRequestURI().getRawPath(); + if (ex.getRequestMethod().equals("PUT")) { + objects.put(path, new Obj(ex.getRequestBody().readAllBytes(), ex.getRequestHeaders().getFirst("Content-Type"))); + reply(ex, 200, "", "text/plain", null); + } else if (ex.getRequestMethod().equals("GET")) { + Obj o = objects.get(path); + if (o == null) { + reply(ex, 404, "NoSuchKey", "application/xml", null); + return; + } + String cd = queryParams(ex).get("response-content-disposition"); + reply(ex, 200, o.data(), o.contentType() == null ? "application/octet-stream" : o.contentType(), + cd == null ? null : URLDecoder.decode(cd, StandardCharsets.UTF_8)); + } else { + reply(ex, 405, "", "text/plain", null); + } + } catch (Exception e) { + reply(ex, 500, e.toString(), "text/plain", null); + } + } + + private static Map queryParams(HttpExchange ex) { + Map m = new TreeMap<>(); + String raw = ex.getRequestURI().getRawQuery(); + if (raw != null) { + for (String kv : raw.split("&")) { + int i = kv.indexOf('='); + m.put(i < 0 ? kv : kv.substring(0, i), i < 0 ? "" : kv.substring(i + 1)); + } + } + return m; + } + + /** Returns null if the signature is valid, otherwise "Code|Message". */ + private String verify(HttpExchange ex) throws Exception { + Map q = queryParams(ex); + String signature = q.get("X-Amz-Signature"); + if (signature == null || !"AWS4-HMAC-SHA256".equals(q.get("X-Amz-Algorithm"))) { + return "AccessDenied|Query-string authentication requires the X-Amz-Algorithm and X-Amz-Signature parameters"; + } + String amzDate = q.get("X-Amz-Date"); + long expires = Long.parseLong(q.get("X-Amz-Expires")); + Instant signedAt = LocalDateTime.parse(amzDate, DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'")).toInstant(ZoneOffset.UTC); + if (Instant.now().isAfter(signedAt.plusSeconds(expires))) { + return "AccessDenied|Request has expired"; + } + + String credential = URLDecoder.decode(q.get("X-Amz-Credential"), StandardCharsets.UTF_8); + String[] scope = credential.split("/"); // accessKey/date/region/service/aws4_request + String signedHeaders = URLDecoder.decode(q.get("X-Amz-SignedHeaders"), StandardCharsets.UTF_8); + + Map signedQuery = new TreeMap<>(q); + signedQuery.remove("X-Amz-Signature"); + List pairs = new ArrayList<>(); + signedQuery.forEach((k, v) -> pairs.add(k + "=" + v)); + + StringBuilder canonicalHeaders = new StringBuilder(); + for (String h : signedHeaders.split(";")) { + String value = h.equals("host") ? ex.getRequestHeaders().getFirst("Host") : ex.getRequestHeaders().getFirst(h); + canonicalHeaders.append(h).append(':').append(value == null ? "" : value.trim()).append('\n'); + } + + String canonicalRequest = ex.getRequestMethod() + "\n" + ex.getRequestURI().getRawPath() + "\n" + + String.join("&", pairs) + "\n" + canonicalHeaders + "\n" + signedHeaders + "\nUNSIGNED-PAYLOAD"; + String stringToSign = "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope[1] + "/" + scope[2] + "/" + scope[3] + "/aws4_request\n" + + hex(MessageDigest.getInstance("SHA-256").digest(canonicalRequest.getBytes(StandardCharsets.UTF_8))); + + byte[] k = hmac(("AWS4" + secretKey).getBytes(StandardCharsets.UTF_8), scope[1]); + k = hmac(k, scope[2]); + k = hmac(k, scope[3]); + k = hmac(k, "aws4_request"); + String expected = hex(hmac(k, stringToSign)); + return expected.equals(signature) ? null + : "SignatureDoesNotMatch|The request signature we calculated does not match the signature you provided"; + } + + private static byte[] hmac(byte[] key, String data) throws Exception { + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(key, "HmacSHA256")); + return mac.doFinal(data.getBytes(StandardCharsets.UTF_8)); + } + + private static String hex(byte[] b) { + return HexFormat.of().formatHex(b); + } + + private static void reply(HttpExchange ex, int status, String body, String type, String disposition) throws IOException { + reply(ex, status, body.getBytes(StandardCharsets.UTF_8), type, disposition); + } + + private static void reply(HttpExchange ex, int status, byte[] body, String type, String disposition) throws IOException { + ex.getResponseHeaders().set("Content-Type", type); + if (disposition != null) { + ex.getResponseHeaders().set("Content-Disposition", disposition); + } + ex.sendResponseHeaders(status, body.length == 0 ? -1 : body.length); + if (body.length > 0) { + ex.getResponseBody().write(body); + } + ex.close(); + } +} diff --git a/file-transfer/src/test/java/com/ankurm/filetransfer/FilenameAndDownloadTest.java b/file-transfer/src/test/java/com/ankurm/filetransfer/FilenameAndDownloadTest.java new file mode 100644 index 0000000..26b701c --- /dev/null +++ b/file-transfer/src/test/java/com/ankurm/filetransfer/FilenameAndDownloadTest.java @@ -0,0 +1,92 @@ +package com.ankurm.filetransfer; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.net.URI; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.server.LocalServerPort; + +import com.ankurm.filetransfer.Cap.Part; + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +class FilenameAndDownloadTest { + + static final String F = "02-filename.txt"; + static final String G = "03-download.txt"; + + @LocalServerPort int port; + @Autowired FileStore store; + + @BeforeAll + static void clean() { + Cap.reset(F); + Cap.reset(G); + } + + String base() { + return "http://localhost:" + port; + } + + String postRaw(String filename, String contentType, byte[] content) throws Exception { + String boundary = "----ft" + System.nanoTime(); + byte[] body = Cap.multipart(boundary, List.of(new Part("file", filename, contentType, content))); + HttpRequest req = HttpRequest.newBuilder(URI.create(base() + "/files")) + .header("Content-Type", "multipart/form-data; boundary=" + boundary) + .POST(HttpRequest.BodyPublishers.ofByteArray(body)).build(); + return Cap.HTTP.send(req, HttpResponse.BodyHandlers.ofString()).body(); + } + + @Test + void whatTheServerSeesIsWhatTheClientSent() throws Exception { + String evil = "../../etc/cron.d/evil.txt"; + String json = postRaw(evil, "image/png", "hello".getBytes(StandardCharsets.UTF_8)); + Cap.say(F, "client sent filename=" + evil + " Content-Type=image/png"); + Cap.say(F, "server response: " + json); + + Path root = store.root(); + boolean escapes = !root.resolve(evil).normalize().startsWith(root); + Cap.say(F, "root.resolve(originalFilename).normalize() stays inside the storage dir: " + !escapes); + assertThat(json).contains("\"displayName\":\"evil.txt\""); + assertThat(escapes).isTrue(); + + String win = postRaw("..\\..\\windows\\system32\\evil.dll", "text/plain", "x".getBytes(StandardCharsets.UTF_8)); + Cap.say(F, "client sent a Windows-style path; server response: " + win); + } + + @Test + void downloadSupportsRangeAndEncodesNames() throws Exception { + String json = postRaw("résumé 2026.txt", "text/plain", "ABCDEFGHIJKLMNOPQRSTUVWXYZ".getBytes(StandardCharsets.UTF_8)); + String id = json.replaceAll(".*\"id\":\"([^\"]+)\".*", "$1"); + + HttpResponse full = Cap.HTTP.send(HttpRequest.newBuilder(URI.create(base() + "/files/" + id)).build(), + HttpResponse.BodyHandlers.ofString()); + Cap.say(G, "GET /files/{id}"); + Cap.say(G, " status=" + full.statusCode() + " body=" + full.body()); + for (String h : List.of("content-type", "content-length", "content-disposition", "accept-ranges", "x-content-type-options")) { + Cap.say(G, " " + h + ": " + full.headers().firstValue(h).orElse("(absent)")); + } + + HttpResponse part = Cap.HTTP.send(HttpRequest.newBuilder(URI.create(base() + "/files/" + id)) + .header("Range", "bytes=10-19").build(), HttpResponse.BodyHandlers.ofString()); + Cap.say(G, "GET /files/{id} with Range: bytes=10-19"); + Cap.say(G, " status=" + part.statusCode() + " body=" + part.body()); + Cap.say(G, " content-range: " + part.headers().firstValue("content-range").orElse("(absent)")); + + HttpResponse past = Cap.HTTP.send(HttpRequest.newBuilder(URI.create(base() + "/files/" + id)) + .header("Range", "bytes=500-").build(), HttpResponse.BodyHandlers.ofString()); + Cap.say(G, "GET /files/{id} with Range: bytes=500-"); + Cap.say(G, " status=" + past.statusCode() + " content-range: " + past.headers().firstValue("content-range").orElse("(absent)")); + + assertThat(part.statusCode()).isEqualTo(206); + assertThat(part.body()).isEqualTo("KLMNOPQRST"); + } +} diff --git a/file-transfer/src/test/java/com/ankurm/filetransfer/MultipartLimitsTest.java b/file-transfer/src/test/java/com/ankurm/filetransfer/MultipartLimitsTest.java new file mode 100644 index 0000000..0c386af --- /dev/null +++ b/file-transfer/src/test/java/com/ankurm/filetransfer/MultipartLimitsTest.java @@ -0,0 +1,74 @@ +package com.ankurm.filetransfer; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; + +import com.ankurm.filetransfer.Cap.Part; + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@AutoConfigureMockMvc +class MultipartLimitsTest { + + static final String F = "01-limits.txt"; + static final int KB = 1024; + + @LocalServerPort int port; + @Autowired MockMvc mvc; + + @BeforeAll + static void clean() { + Cap.reset(F); + } + + String base() { + return "http://localhost:" + port; + } + + @Test + void limitsAreEnforcedByTheContainerNotByMockMvc() throws Exception { + Cap.say(F, "limits: max-file-size=1MB, max-request-size=2MB"); + + String small = Cap.postMultipart(base(), List.of(new Part("file", "a.txt", "text/plain", Cap.bytes(500 * KB)))); + Cap.say(F, "real HTTP, one 500 KB file -> " + small); + + String big = Cap.postMultipart(base(), List.of(new Part("file", "b.txt", "text/plain", Cap.bytes(1536 * KB)))); + Cap.say(F, "real HTTP, one 1.5 MB file -> " + big); + + List three = List.of( + new Part("file", "c1.txt", "text/plain", Cap.bytes(900 * KB)), + new Part("file", "c2.txt", "text/plain", Cap.bytes(900 * KB)), + new Part("file", "c3.txt", "text/plain", Cap.bytes(900 * KB))); + String request = Cap.postMultipart(base(), three); + Cap.say(F, "real HTTP, 3 x 900 KB in one request -> " + request); + + // Tomcat only swallows a limited amount (maxSwallowSize, 2 MB) of a rejected upload before it closes the + // connection, so a client sending far more than that may see a reset instead of the 413. Count outcomes. + int got413 = 0, reset = 0; + for (int i = 0; i < 10; i++) { + String huge = Cap.postMultipart(base(), List.of(new Part("file", "d.bin", "application/octet-stream", Cap.bytes(10 * 1024 * KB)))); + if (huge.startsWith("status=413")) got413++; + else reset++; + } + Cap.say(F, "real HTTP, one 10 MB file, 10 attempts -> " + got413 + " x status=413, " + reset + " x connection error"); + + int mockStatus = mvc.perform(MockMvcRequestBuilders.multipart("/files") + .file(new MockMultipartFile("file", "b.txt", "text/plain", Cap.bytes(1536 * KB)))) + .andReturn().getResponse().getStatus(); + Cap.say(F, "MockMvc, the same 1.5 MB file -> status=" + mockStatus); + + assertThat(small).startsWith("status=201"); + assertThat(big).startsWith("status=413"); + assertThat(mockStatus).isEqualTo(201); + } +} diff --git a/file-transfer/src/test/java/com/ankurm/filetransfer/PresignedUrlTest.java b/file-transfer/src/test/java/com/ankurm/filetransfer/PresignedUrlTest.java new file mode 100644 index 0000000..7765c1e --- /dev/null +++ b/file-transfer/src/test/java/com/ankurm/filetransfer/PresignedUrlTest.java @@ -0,0 +1,118 @@ +package com.ankurm.filetransfer; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Arrays; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; + +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.json.JsonMapper; + +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +class PresignedUrlTest { + + static final String F = "04-presign.txt"; + static final String SECRET = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; + static FakeS3 s3; + + @LocalServerPort int port; + @Autowired S3PresignService presign; + JsonMapper json = JsonMapper.builder().build(); + + @DynamicPropertySource + static void s3(DynamicPropertyRegistry r) throws IOException { + s3 = new FakeS3(SECRET); + r.add("app.s3.endpoint", s3::endpoint); + } + + @BeforeAll + static void clean() { + Cap.reset(F); + } + + @AfterAll + static void stop() { + s3.close(); + } + + HttpResponse put(String url, String contentType, byte[] body) throws Exception { + HttpRequest.Builder b = HttpRequest.newBuilder(URI.create(url)).PUT(HttpRequest.BodyPublishers.ofByteArray(body)); + if (contentType != null) { + b.header("Content-Type", contentType); + } + return Cap.HTTP.send(b.build(), HttpResponse.BodyHandlers.ofString()); + } + + static String brief(HttpResponse r) { + String body = r.body().replaceAll(".*", "").replaceAll("\\s+", " "); + return "status=" + r.statusCode() + (body.isBlank() ? "" : " " + body); + } + + @Test + void presignedUploadAndDownload() throws Exception { + HttpResponse res = Cap.HTTP.send(HttpRequest.newBuilder(URI.create("http://localhost:" + port + "/presign/upload")) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString("{\"contentType\":\"image/png\"}")).build(), HttpResponse.BodyHandlers.ofString()); + JsonNode node = json.readTree(res.body()); + String url = node.get("url").asString(); + String key = node.get("key").asString(); + + Cap.say(F, "POST /presign/upload {\"contentType\":\"image/png\"} -> status=" + res.statusCode()); + Cap.say(F, " method=" + node.get("method").asString() + " expiresInSeconds=" + node.get("expiresInSeconds").asLong() + + " headers=" + node.get("headers")); + String[] split = url.split("\\?"); + Cap.say(F, " url path: " + split[0].replaceFirst("http://localhost:\\d+", "http://")); + for (String p : split[1].split("&")) { + Cap.say(F, " query: " + p); + } + + byte[] png = {(byte) 0x89, 'P', 'N', 'G', 1, 2, 3, 4}; + Cap.say(F, "PUT with the signed Content-Type (image/png) -> " + brief(put(url, "image/png", png))); + assertThat(s3.objects.values().iterator().next().data()).isEqualTo(png); + + Cap.say(F, "PUT the same URL again (presigned URLs are reusable) -> " + brief(put(url, "image/png", png))); + Cap.say(F, "PUT with a different Content-Type (text/html) -> " + brief(put(url, "text/html", png))); + Cap.say(F, "PUT with no Content-Type header -> " + brief(put(url, null, png))); + + String tampered = url.replace(key, key.substring(0, key.length() - 1) + (key.endsWith("a") ? "b" : "a")); + Cap.say(F, "PUT after changing the last character of the key -> " + brief(put(tampered, "image/png", png))); + + String longer = url.replace("X-Amz-Expires=300", "X-Amz-Expires=604800"); + Cap.say(F, "PUT after raising X-Amz-Expires to 7 days -> " + brief(put(longer, "image/png", png))); + + // download with a Content-Disposition override + HttpResponse dl = Cap.HTTP.send(HttpRequest.newBuilder(URI.create( + "http://localhost:" + port + "/presign/download/" + key + "?filename=holiday.png")).build(), HttpResponse.BodyHandlers.ofString()); + String getUrl = json.readTree(dl.body()).get("url").asString(); + HttpResponse got = Cap.HTTP.send(HttpRequest.newBuilder(URI.create(getUrl)).build(), HttpResponse.BodyHandlers.ofByteArray()); + Cap.say(F, "GET presigned URL -> status=" + got.statusCode() + " bytes match upload=" + Arrays.equals(got.body(), png) + + " content-disposition=" + got.headers().firstValue("content-disposition").orElse("(absent)")); + + Cap.say(F, "disallowed type: POST /presign/upload text/html -> status=" + Cap.HTTP.send(HttpRequest.newBuilder( + URI.create("http://localhost:" + port + "/presign/upload")).header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString("{\"contentType\":\"text/html\"}")).build(), HttpResponse.BodyHandlers.ofString()).statusCode()); + } + + @Test + void expiredUrlIsRejected() throws Exception { + S3PresignService.Upload u = presign.forUpload("image/png", Duration.ofSeconds(1)); + Thread.sleep(2200); + HttpResponse r = put(u.url(), "image/png", "x".getBytes(StandardCharsets.UTF_8)); + Cap.say(F, "PUT a URL signed for 1 second, sent after 2.2 seconds -> " + brief(r)); + assertThat(r.statusCode()).isEqualTo(403); + } +}