diff --git a/README.md b/README.md index d11236a..35b8437 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ files. | [`docker-images/`](docker-images) | [Dockerizing Spring Boot 4: Layered Jars, Buildpacks, Distroless and Image Size Benchmarks](https://ankurm.com/dockerizing-spring-boot-4-layered-jars-buildpacks-distroless/) | one service packaged nine ways and measured: size on disk and pushed, rebuild delta, startup, PID 1 and signals, jlink, the JDK 25 AOT cache | | [`kubernetes-deployment/`](kubernetes-deployment) | [Deploying Spring Boot 4 on Kubernetes](https://ankurm.com/spring-boot-4-kubernetes-probes-graceful-shutdown-cpu-limits-hpa/) | probe groups under a dependency outage, graceful shutdown under load four ways, JVM ergonomics per pod shape, CPU limits throttling GC, HPA on a Micrometer metric | | [`problem-details/`](problem-details) | [Global Exception Handling with ProblemDetail (RFC 9457) in Spring Boot 4](https://ankurm.com/spring-boot-4-problemdetail-rfc-9457-global-exception-handling/) | thirteen failures under five handling setups, validation errors, i18n, content negotiation, errors outside MVC, silent 500s, decoding on the client | +| [`resilience/`](resilience) | [Spring Framework 7's Built-in Resilience: @Retryable, @ConcurrencyLimit, and What's Left for Resilience4j](https://ankurm.com/spring-framework-7-retryable-concurrencylimit-resilience4j/) | `@Retryable` and `@ConcurrencyLimit` counted invocation by invocation, retries inside transactions, where Resilience4j still earns its place, migrating from Spring Retry | Articles whose text is kept here rather than only on the blog have it under `/post/` — `post.md` for the body and `meta.md` for the title, excerpt and diff --git a/resilience/README.md b/resilience/README.md new file mode 100644 index 0000000..dbbfc80 --- /dev/null +++ b/resilience/README.md @@ -0,0 +1,65 @@ +# Spring Framework 7's built-in resilience - and what is left for Resilience4j + +Companion project for [**Spring Framework 7's Built-in Resilience: @Retryable, @ConcurrencyLimit, and What's Left for Resilience4j**](https://ankurm.com/spring-framework-7-retryable-concurrencylimit-resilience4j/) +on ankurm.com. + +Every behaviour the article states was counted here - by recording each real invocation of the +guarded method, not by trusting the retry machinery's own view. `./scripts/run-all.sh` +regenerates every transcript in [`docs/output/`](docs/output). + +## Versions + +| | | +|---|---| +| Spring Boot | 4.1.1 | +| Spring Framework | 7.0.9 (`org.springframework.resilience`, `org.springframework.core.retry`) | +| Resilience4j | 2.4.0 (`resilience4j-spring-boot4`, not managed by Boot - pin it) | +| JDK | Eclipse Temurin 25.0.4.1 (LTS) | + +## Quickstart + +```bash +export JAVA_HOME=/path/to/jdk-25 +mvn -DskipTests package +./scripts/run.sh # port 8081 +curl -s localhost:8081/demo/retry/exhausted | python3 -m json.tool +./scripts/run-all.sh # about 90 s: some scenarios sleep through 1 s delays +mvn test # 16 contract tests +``` + +## Scenarios + +| Endpoint | Shows | +|---|---| +| `/demo/retry/{defaults,exhausted,exponential,jitter,includes,cause,future,mono,timeout,hang,self-invocation}` | what `@Retryable` does, counted | +| `/demo/tx/{standalone,joined}` | `@Retryable` + `@Transactional`, alone and inside a caller's transaction | +| `/demo/limit/{none,block,reject,limit-and-retry,r4j-bulkhead}` | `@ConcurrencyLimit` policies, and Resilience4j's bulkhead for comparison | +| `/demo/r4j/{retry,breaker,combo,timelimiter}` | what only Resilience4j does, and both libraries on one method | +| `/demo/proxy/{payments,stock-writer,reports}` | the advisor chain on each proxy, outermost first | + +All of these are diagnostics for the article. Delete [`DemoController`](src/main/java/com/ankurm/resilience/web/DemoController.java) before shipping anything. + +## Documentation + +1. [What Spring Framework 7 ships, and in which 7.0.x release](docs/01-whats-in-framework-7.md) +2. [Switching it on - and the three ways it stays off](docs/02-enabling.md) +3. [`@Retryable`, measured](docs/03-retryable-measured.md) +4. [`@ConcurrencyLimit`: BLOCK, REJECT, and nesting with retry](docs/04-concurrency-limit.md) +5. [Retry and transactions](docs/05-retry-and-transactions.md) +6. [What is left for Resilience4j](docs/06-what-is-left-for-resilience4j.md) +7. [Migrating from spring-retry](docs/07-migrating-from-spring-retry.md) + +## Findings worth the trip + +- **`maxRetries = 3` is four invocations.** Resilience4j's `maxAttempts: 3` is three, and so was + spring-retry's `maxAttempts`. A mechanical migration adds one call. +- **`timeout` is a budget checked between attempts, not a call timeout.** A 1.5 s attempt under a + 500 ms timeout runs its full 1.5 s. +- **Jitter only ever adds delay** (for the first delay, and for every delay when `multiplier` is 1): + `delay=200, jitter=100` produced gaps of 211-296 ms, never below 200. +- **A `CompletableFuture` that completes exceptionally is not retried**; a `Mono` is. +- **`@Retryable` is placed outside `@Transactional` and outside Resilience4j** regardless of any + `order` - both post-processors call `setBeforeExistingAdvisors(true)`. Good for transactions, bad + for a circuit breaker: an open circuit is retried three times. +- **`timeout` and `ConcurrencyLimit.ThrottlePolicy.REJECT` did not exist in 7.0.0.** `timeout` + arrived in 7.0.2, `policy` in 7.0.3. diff --git a/resilience/docs/01-whats-in-framework-7.md b/resilience/docs/01-whats-in-framework-7.md new file mode 100644 index 0000000..4b82c1b --- /dev/null +++ b/resilience/docs/01-whats-in-framework-7.md @@ -0,0 +1,45 @@ +# 1. What Spring Framework 7 ships + +[Index](../README.md) · Next: [2. Enabling →](02-enabling.md) + +| Piece | Package | Jar | +|---|---|---| +| `@Retryable`, `@ConcurrencyLimit`, `@EnableResilientMethods` | `org.springframework.resilience.annotation` | spring-context | +| `MethodRetryEvent`, `MethodRetryPredicate`, interceptors | `org.springframework.resilience.retry` | spring-context | +| `InvocationRejectedException` | `org.springframework.resilience` | spring-context | +| `RetryTemplate`, `RetryPolicy`, `RetryListener`, `RetryException` | `org.springframework.core.retry` | spring-core | + +## Defaults, read from the annotations + +Reflection over `Retryable.class.getDeclaredMethods()` on 7.0.9 +([`ApiSurfaceTest`](../src/test/java/com/ankurm/resilience/ApiSurfaceTest.java)): + +| Attribute | Default | +|---|---| +| `maxRetries` | `3` - retries *after* the first call | +| `delay` | `1000` (ms) | +| `multiplier` | `1.0` - no growth | +| `maxDelay` | `Long.MAX_VALUE` | +| `jitter` | `0` | +| `timeout` | `0` - no budget | +| `timeUnit` | `MILLISECONDS` | +| `includes` / `excludes` | empty - every exception is retryable | + +`@ConcurrencyLimit` has no usable default limit (`Integer.MIN_VALUE` means "not set") and +`policy = BLOCK`. `@EnableResilientMethods` has `proxyTargetClass = false` and +`order = Integer.MAX_VALUE - 1`. + +## It moved during 7.0.x + +[`api-surface.txt`](output/api-surface.txt), per release: + +| Added in | What | +|---|---| +| 7.0.0 | `@Retryable` (no `timeout`), `@ConcurrencyLimit` (no `policy`), `RetryTemplate`, `RetryPolicy` | +| 7.0.2 | `@Retryable.timeout`/`timeoutString`, `RetryPolicy.Builder.timeout(..)`, `RetryListener.onRetryPolicyTimeout` and `onRetryableExecution` | +| 7.0.3 | `@ConcurrencyLimit.policy` with `ThrottlePolicy.BLOCK` / `REJECT` | + +Release dates from the Maven Central `Last-Modified` headers: 7.0.0 on 13 Nov 2025, 7.0.2 on +11 Dec 2025, 7.0.3 on 15 Jan 2026. Articles written against the 7.0 GA - including the Spring reference page at the time +this was written, which does not mention `timeout` or `policy` - describe a smaller API than the +one you have. diff --git a/resilience/docs/02-enabling.md b/resilience/docs/02-enabling.md new file mode 100644 index 0000000..e3bcb28 --- /dev/null +++ b/resilience/docs/02-enabling.md @@ -0,0 +1,33 @@ +# 2. Switching it on - and the three ways it stays off + +[← 1. What ships](01-whats-in-framework-7.md) · [Index](../README.md) · Next: [3. @Retryable measured →](03-retryable-measured.md) + +## Spring Boot 4.1.1 does not enable it + +No auto-configuration in Boot 4.1.1 registers `RetryAnnotationBeanPostProcessor` or +`ConcurrencyLimitBeanPostProcessor` (no Boot 4.1.1 jar in this project's dependency tree mentions +either class). You need [`@EnableResilientMethods`](../src/main/java/com/ankurm/resilience/ResilienceConfig.java). + +Without it the annotations are metadata. [`retry-disabled.txt`](output/retry-disabled.txt), same +method, `demo.resilience.enabled=false`: + +``` +"invocations": 1, +"caller": "threw: com.ankurm.resilience.support.TransientException: attempt 1 failed" +``` + +No warning at startup, no log line at call time. + +## Self-invocation + +A call from inside the bean does not go through the proxy +([`retry-self-invocation.txt`](output/retry-self-invocation.txt)): one invocation, first exception +straight to the caller. Same rule as `@Transactional` and `@Async` - see the +[AOP article](https://ankurm.com/spring-aop-pointcuts-advice-types-aspect-not-firing/). + +## `final` methods + +The proxies here are CGLIB subclasses (`/demo/proxy/*` reports the type - the demo beans implement +no interface). A subclass cannot override a `final` method, so a `final @Retryable` method is +called directly on the target and never retried - the same silent failure measured for `@Async` in +the [@Async article](https://ankurm.com/spring-boot-4-async-executors-virtual-threads/). diff --git a/resilience/docs/03-retryable-measured.md b/resilience/docs/03-retryable-measured.md new file mode 100644 index 0000000..28dc8fc --- /dev/null +++ b/resilience/docs/03-retryable-measured.md @@ -0,0 +1,61 @@ +# 3. `@Retryable`, measured + +[← 2. Enabling](02-enabling.md) · [Index](../README.md) · Next: [4. @ConcurrencyLimit →](04-concurrency-limit.md) + +Every number here is from [`FlakyGateway`](../src/main/java/com/ankurm/resilience/retry/FlakyGateway.java), +which records each real invocation in [`CallLog`](../src/main/java/com/ankurm/resilience/support/CallLog.java). + +## Counting + +| Scenario | Invocations | Gaps (ms) | Caller got | Transcript | +|---|---|---|---|---| +| defaults, fails twice | 3 | 1009, 1001 | the value | [`retry-defaults.txt`](output/retry-defaults.txt) | +| defaults, always fails | **4** | 1001, 1001, 1002 | `TransientException: attempt 4 failed` | [`retry-exhausted.txt`](output/retry-exhausted.txt) | +| `maxRetries=5, delay=100, multiplier=2, maxDelay=500` | 6 | 101, 200, 401, 501, 501 | last exception | [`retry-exponential.txt`](output/retry-exponential.txt) | +| `delay=200, jitter=100` | 7 | 254, 211, 296, 294, 273, 293 | last exception | [`retry-jitter.txt`](output/retry-jitter.txt) | + +- The caller receives the **last original exception**, not a wrapper. (`RetryTemplate.execute` + throws `RetryException`; the annotation path unwraps it.) +- `maxDelay` caps the exponential sequence: 400 would have been 800. + +## Jitter only adds + +Jitter looks symmetric in the documentation. `ExponentialBackOff$ExponentialBackOffExecution.applyJitter` +(read with `javap -c` on spring-core 7.0.9) computes the range as +`[max(interval - j, initialInterval), min(interval + j, maxInterval)]`, where `j` is the jitter +scaled by `interval / initialInterval`. For the first delay - and for every delay when `multiplier` +is 1 - the lower bound is clamped to `delay` itself, so jitter can only lengthen the wait. The +transcript agrees: six gaps between 211 and 296 ms, none under 200. + +## What is retried + +| Scenario | Invocations | Why | +|---|---|---| +| `includes = IllegalStateException`, throws `IllegalArgumentException` | 1 | not included | +| `includes = IOException`, throws `UncheckedIOException(IOException)` | 3 | **causes are matched** | +| returns `CompletableFuture.failedFuture(...)` | **1** | the method returned normally | +| returns `Mono.fromCallable(...)` that errors | 3 | retried by re-subscription | + +The `CompletableFuture` row is the one to remember: an async client method that returns a future +is not protected by `@Retryable` at all ([`retry-future.txt`](output/retry-future.txt)). Reactive +types go through a Reactor `retryWhen`, so a `Mono` works. + +## `timeout` is a budget, not a timeout + +| Scenario | Invocations | Elapsed | | +|---|---|---|---| +| attempts of 300 ms, `delay=100`, `timeout=1000` | 3 | 1105 ms | stops once the budget is spent - after overrunning it by one attempt | +| one attempt of 1500 ms, `timeout=500` | 1 | **1502 ms** | the attempt is not interrupted | + +The budget is checked between attempts. Use a client-level timeout (HTTP client, JDBC query +timeout) for the call itself, and treat `timeout` as "stop retrying after roughly this long". + +## Events are not a "retry happened" signal + +`MethodRetryEvent` fires for every failure with `isRetryAborted() == false`, *including the last +one*, then once more with a `RetryException` and `isRetryAborted() == true`. For an exception that +is not even retryable you get `will retry` followed by `retry aborted` +([`retry-includes.txt`](output/retry-includes.txt)). A counter of events with `retryAborted=false` +counts failures, not retries. Spring registers no retry metrics of its own +([`metrics-names.txt`](output/metrics-names.txt)): `app.retry.failures` exists only because +[`RetryEvents`](../src/main/java/com/ankurm/resilience/retry/RetryEvents.java) creates it. diff --git a/resilience/docs/04-concurrency-limit.md b/resilience/docs/04-concurrency-limit.md new file mode 100644 index 0000000..34a4334 --- /dev/null +++ b/resilience/docs/04-concurrency-limit.md @@ -0,0 +1,33 @@ +# 4. `@ConcurrencyLimit`: BLOCK, REJECT, and nesting with retry + +[← 3. @Retryable measured](03-retryable-measured.md) · [Index](../README.md) · Next: [5. Retry and transactions →](05-retry-and-transactions.md) + +Ten virtual threads call a 200 ms method limited to 2 ([`ReportService`](../src/main/java/com/ankurm/resilience/limit/ReportService.java)): + +| Setup | Succeeded | Failed | Max inside | Elapsed | Transcript | +|---|---|---|---|---|---| +| no limit | 10 | - | 10 | 203 ms | [`limit-none.txt`](output/limit-none.txt) | +| `@ConcurrencyLimit(2)` (BLOCK) | 10 | - | 2 | 1006 ms | [`limit-block.txt`](output/limit-block.txt) | +| `policy = REJECT` | 2 | 8 × `InvocationRejectedException` | 2 | 205 ms | [`limit-reject.txt`](output/limit-reject.txt) | +| Resilience4j `@Bulkhead`, `max-wait-duration: 0` | 2 | 8 × `BulkheadFullException` | 2 | 231 ms | [`limit-r4j-bulkhead.txt`](output/limit-r4j-bulkhead.txt) | + +BLOCK queues callers indefinitely - there is no wait timeout. With virtual threads that is cheap in +memory, but an unbounded queue in front of a slow dependency is still an unbounded latency. +`InvocationRejectedException` extends `RejectedExecutionException`, so existing handlers for +executor rejection catch it. + +## Both annotations on one method + +`limitedAndRetried()` has `@ConcurrencyLimit(1)` and `@Retryable(maxRetries=1, delay=300)`; every +caller's first attempt fails. Two callers took **815 ms** ([`limit-limit-and-retry.txt`](output/limit-limit-and-retry.txt)): +about 2 × (50 + 300 + 50). The permit is held through the 300 ms back-off, because the limit is the +outer interceptor ([`proxy-reports.txt`](output/proxy-reports.txt)): + +``` +"org.springframework.resilience.annotation.ConcurrencyLimitBeanPostProcessor$ConcurrencyLimitInterceptor order=2147483647", +"org.springframework.resilience.annotation.RetryAnnotationBeanPostProcessor$RetryAnnotationInterceptor order=2147483647" +``` + +Both post-processors insert at the front of an existing proxy, so whichever runs second ends up +outermost. If you want the permit released between attempts, put the two annotations on different +beans. diff --git a/resilience/docs/05-retry-and-transactions.md b/resilience/docs/05-retry-and-transactions.md new file mode 100644 index 0000000..3fe52d9 --- /dev/null +++ b/resilience/docs/05-retry-and-transactions.md @@ -0,0 +1,51 @@ +# 5. Retry and transactions + +[← 4. @ConcurrencyLimit](04-concurrency-limit.md) · [Index](../README.md) · Next: [6. What is left for Resilience4j →](06-what-is-left-for-resilience4j.md) + +[`StockWriter.record`](../src/main/java/com/ankurm/resilience/tx/StockWriter.java) is +`@Retryable @Transactional`, inserts a row, and fails twice. + +## Which interceptor is outside + +`RetryAnnotationBeanPostProcessor`'s constructor calls `setBeforeExistingAdvisors(true)` (`javap -c` +on spring-context 7.0.9). The transaction advisor is applied first by the auto-proxy creator; the +retry advisor is then inserted at index 0 ([`proxy-stock-writer.txt`](output/proxy-stock-writer.txt)). +The `order` attribute of `@EnableResilientMethods` is the post-processor's order, not the +advice's position, and does not change this. + +## Alone: each attempt is its own transaction + +[`tx-standalone.txt`](output/tx-standalone.txt) - three connection holders, one row: + +``` + "attempts": [ + "attempt 1: transaction active=true, connection holder @6aa73704", + "attempt 2: transaction active=true, connection holder @1978b580", + "attempt 3: transaction active=true, connection holder @7d5534c2" + ], + "rows in stock_movement": 1 +``` + +This is what you want: the failed attempts rolled back their inserts. + +## Called from inside a transaction: the retry cannot help + +[`OrderFacade.placeOrder`](../src/main/java/com/ankurm/resilience/tx/OrderFacade.java) is +`@Transactional` and calls the same method ([`tx-joined.txt`](output/tx-joined.txt)): + +``` + "caller got": "org.springframework.transaction.UnexpectedRollbackException: Transaction rolled back because it has been marked as rollback-only", + "attempts": [ + "attempt 1: transaction active=true, connection holder @6135f416", + "attempt 2: transaction active=true, connection holder @6135f416", + "attempt 3: transaction active=true, connection holder @6135f416" + ], + "rows in stock_movement": 0 +``` + +Every attempt joins the outer transaction (`REQUIRED`). The first failure passes through the inner +`TransactionInterceptor`, which marks the *shared* transaction rollback-only. The third attempt +succeeds, the retry returns normally, and the outer commit throws. Three attempts, zero rows, an +exception the caller did not expect. The retry belongs at the outermost transactional boundary - +or the inner method needs `REQUIRES_NEW`. The [@Transactional article](https://ankurm.com/transactional-propagation-isolation-silent-failures/) +covers the rollback-only mechanics. diff --git a/resilience/docs/06-what-is-left-for-resilience4j.md b/resilience/docs/06-what-is-left-for-resilience4j.md new file mode 100644 index 0000000..7c5bd15 --- /dev/null +++ b/resilience/docs/06-what-is-left-for-resilience4j.md @@ -0,0 +1,68 @@ +# 6. What is left for Resilience4j + +[← 5. Retry and transactions](05-retry-and-transactions.md) · [Index](../README.md) · Next: [7. Migrating from spring-retry →](07-migrating-from-spring-retry.md) + +| Need | Spring Framework 7.0.9 | Resilience4j 2.4.0 | +|---|---|---| +| Retry with back-off and jitter | `@Retryable`, `RetryTemplate` | `@Retry` | +| Limit concurrent calls | `@ConcurrencyLimit` (BLOCK/REJECT) | `@Bulkhead` (semaphore or thread pool, with a wait timeout) | +| Circuit breaker | - | `@CircuitBreaker` | +| Rate limiter (calls per period) | - | `@RateLimiter` | +| Time limit on a call | - (`timeout` is a retry budget) | `@TimeLimiter` (async return types) | +| Declarative fallback / recovery | - (`@Recover` request declined, spring-framework#35685) | `fallbackMethod` | +| Metrics | none built in | `resilience4j.*` meters, Actuator endpoints, health | +| Per-instance config in properties | placeholders in `*String` attributes | `resilience4j..instances..*` | + +## The breaker, measured + +`sliding-window-size: 10`, `minimum-number-of-calls: 5`, `failure-rate-threshold: 50`, 2 s open +([`r4j-breaker.txt`](output/r4j-breaker.txt)): five failures reach the downstream, the breaker +opens, calls 6-8 fail in microseconds with `CallNotPermittedException`, and after 2.1 s the first +call goes through (`HALF_OPEN`), the second closes it. + +**Resilience4j's own defaults are `slidingWindowSize=100` and `minimumNumberOfCalls=100`** (read +from `CircuitBreakerConfig.ofDefaults()`), with 60 s open. A breaker left on defaults cannot open +until it has seen a hundred calls - in a low-traffic service, effectively never. + +## `maxAttempts` vs `maxRetries` + +[`r4j-retry.txt`](output/r4j-retry.txt): `max-attempts: 3` → **3** invocations. Spring's +`maxRetries = 3` → **4**. Same number, different meaning. + +## Both libraries on one method + +`chargeWithBoth()` carries `@CircuitBreaker(name="combo")` and Spring's `@Retryable(maxRetries=3, delay=10)` +([`r4j-combo.txt`](output/r4j-combo.txt), [`proxy-payments.txt`](output/proxy-payments.txt)): + +``` + "call 1": { + "caller got": "TransientException: payment gateway 503 (call 4)", + "method body ran": 4, + "elapsed (ms)": 35, + "breaker: buffered / failed / not permitted": "4 / 4 / 0", + "breaker state": "CLOSED" + }, + "call 2": { + "caller got": "CallNotPermittedException: CircuitBreaker 'combo' is OPEN and does not permit further calls", + "method body ran": 1, + "elapsed (ms)": 33, + "breaker: buffered / failed / not permitted": "5 / 5 / 3", + "breaker state": "OPEN" + }, +``` + +Spring's retry interceptor is the first advisor on the proxy - ahead of every Resilience4j aspect +(orders 2147483642-2147483646) - so it retries *around* the breaker. The breaker counts each attempt +as a separate call, and once it opens, the retry spends its remaining attempts on +`CallNotPermittedException`. With the default 1 s delay that is three seconds of waiting to be told +the circuit is open. + +If you combine them, add `excludes = CallNotPermittedException.class` to the `@Retryable` - or use +Resilience4j's `@Retry`, whose aspect order puts it outside the breaker deliberately and which you +can configure to ignore `CallNotPermittedException`. + +## TimeLimiter + +[`r4j-timelimiter.txt`](output/r4j-timelimiter.txt): a 2 s `CompletableFuture` under a 500 ms +limit fails with `TimeoutException` after 512 ms. It only applies to asynchronous return types - +there is no equivalent for a blocking method in either library. diff --git a/resilience/docs/07-migrating-from-spring-retry.md b/resilience/docs/07-migrating-from-spring-retry.md new file mode 100644 index 0000000..3580b2b --- /dev/null +++ b/resilience/docs/07-migrating-from-spring-retry.md @@ -0,0 +1,21 @@ +# 7. Migrating from spring-retry + +[← 6. What is left for Resilience4j](06-what-is-left-for-resilience4j.md) · [Index](../README.md) + +The spring-retry README now says the project "is no longer maintained as an open-source project" +and "has been superseded by Spring Framework 7". Its last release on Maven Central is 2.0.13. + +| spring-retry | Spring Framework 7 | Watch out | +|---|---|---| +| `@EnableRetry` | `@EnableResilientMethods` | | +| `@Retryable(maxAttempts = 3)` | `@Retryable(maxRetries = 2)` | **attempts vs retries** - `maxRetries = 3` is one call more | +| `@Retryable(retryFor = X.class)` | `@Retryable(includes = X.class)` (or `value`) | causes are matched too | +| `@Retryable(noRetryFor = X.class)` | `@Retryable(excludes = X.class)` | | +| `@Backoff(delay, multiplier, maxDelay)` | `delay`, `multiplier`, `maxDelay` attributes | default delay is 1000 ms in both | +| `@Backoff(random = true)` | `jitter` | jitter only adds delay (chapter 3) | +| `@Recover` | none - catch the exception in the caller | declined upstream | +| `RetryTemplate.builder()` | `new RetryTemplate(RetryPolicy.builder()...build())` | `execute` throws `RetryException`; `invoke` throws the original | +| stateful retry / `RetryContext` | none | | + +Keep spring-retry on the classpath only as long as something still uses `@Recover` or stateful +retry. Both libraries define an annotation named `Retryable` - check the import on every one. diff --git a/resilience/docs/output/api-surface.txt b/resilience/docs/output/api-surface.txt new file mode 100644 index 0000000..47e2be7 --- /dev/null +++ b/resilience/docs/output/api-surface.txt @@ -0,0 +1,31 @@ +# Public API of the resilience support, per Spring Framework release + +## 7.0.0 + @Retryable [delay, delayString, excludes, includes, jitter, jitterString, maxDelay, maxDelayString, maxRetries, maxRetriesString, multiplier, multiplierString, predicate, timeUnit, value] + @ConcurrencyLimit [limit, limitString, value] + RetryPolicy.Builder [backOff, build, delay, excludes, includes, jitter, maxDelay, maxRetries, multiplier, predicate] + RetryListener [beforeRetry, onRetryFailure, onRetryPolicyExhaustion, onRetryPolicyInterruption, onRetrySuccess] + +## 7.0.1 + @Retryable [delay, delayString, excludes, includes, jitter, jitterString, maxDelay, maxDelayString, maxRetries, maxRetriesString, multiplier, multiplierString, predicate, timeUnit, value] + @ConcurrencyLimit [limit, limitString, value] + RetryPolicy.Builder [backOff, build, delay, excludes, includes, jitter, maxDelay, maxRetries, multiplier, predicate] + RetryListener [beforeRetry, onRetryFailure, onRetryPolicyExhaustion, onRetryPolicyInterruption, onRetrySuccess] + +## 7.0.2 + @Retryable [delay, delayString, excludes, includes, jitter, jitterString, maxDelay, maxDelayString, maxRetries, maxRetriesString, multiplier, multiplierString, predicate, timeUnit, timeout, timeoutString, value] + @ConcurrencyLimit [limit, limitString, value] + RetryPolicy.Builder [backOff, build, delay, excludes, includes, jitter, maxDelay, maxRetries, multiplier, predicate, timeout] + RetryListener [beforeRetry, onRetryFailure, onRetryPolicyExhaustion, onRetryPolicyInterruption, onRetryPolicyTimeout, onRetrySuccess, onRetryableExecution] + +## 7.0.3 + @Retryable [delay, delayString, excludes, includes, jitter, jitterString, maxDelay, maxDelayString, maxRetries, maxRetriesString, multiplier, multiplierString, predicate, timeUnit, timeout, timeoutString, value] + @ConcurrencyLimit [limit, limitString, policy, value] + RetryPolicy.Builder [backOff, build, delay, excludes, includes, jitter, maxDelay, maxRetries, multiplier, predicate, timeout] + RetryListener [beforeRetry, onRetryFailure, onRetryPolicyExhaustion, onRetryPolicyInterruption, onRetryPolicyTimeout, onRetrySuccess, onRetryableExecution] + +## 7.0.9 + @Retryable [delay, delayString, excludes, includes, jitter, jitterString, maxDelay, maxDelayString, maxRetries, maxRetriesString, multiplier, multiplierString, predicate, timeUnit, timeout, timeoutString, value] + @ConcurrencyLimit [limit, limitString, policy, value] + RetryPolicy.Builder [backOff, build, delay, excludes, includes, jitter, maxDelay, maxRetries, multiplier, predicate, timeout] + RetryListener [beforeRetry, onRetryFailure, onRetryPolicyExhaustion, onRetryPolicyInterruption, onRetryPolicyTimeout, onRetrySuccess, onRetryableExecution] diff --git a/resilience/docs/output/limit-block.txt b/resilience/docs/output/limit-block.txt new file mode 100644 index 0000000..d7d0802 --- /dev/null +++ b/resilience/docs/output/limit-block.txt @@ -0,0 +1,10 @@ +# GET /demo/limit/block (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "callers": 10, + "succeeded": 10, + "failed": {}, + "max concurrently inside the method": 2, + "elapsed (ms)": 1006 +} diff --git a/resilience/docs/output/limit-limit-and-retry.txt b/resilience/docs/output/limit-limit-and-retry.txt new file mode 100644 index 0000000..823ab85 --- /dev/null +++ b/resilience/docs/output/limit-limit-and-retry.txt @@ -0,0 +1,10 @@ +# GET /demo/limit/limit-and-retry (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "callers": 2, + "succeeded": 2, + "failed": {}, + "max concurrently inside the method": 1, + "elapsed (ms)": 815 +} diff --git a/resilience/docs/output/limit-none.txt b/resilience/docs/output/limit-none.txt new file mode 100644 index 0000000..834016c --- /dev/null +++ b/resilience/docs/output/limit-none.txt @@ -0,0 +1,10 @@ +# GET /demo/limit/none (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "callers": 10, + "succeeded": 10, + "failed": {}, + "max concurrently inside the method": 10, + "elapsed (ms)": 203 +} diff --git a/resilience/docs/output/limit-r4j-bulkhead.txt b/resilience/docs/output/limit-r4j-bulkhead.txt new file mode 100644 index 0000000..06fd5ee --- /dev/null +++ b/resilience/docs/output/limit-r4j-bulkhead.txt @@ -0,0 +1,12 @@ +# GET /demo/limit/r4j-bulkhead (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "callers": 10, + "succeeded": 2, + "failed": { + "io.github.resilience4j.bulkhead.BulkheadFullException": 8 + }, + "max concurrently inside the method": 2, + "elapsed (ms)": 231 +} diff --git a/resilience/docs/output/limit-reject.txt b/resilience/docs/output/limit-reject.txt new file mode 100644 index 0000000..c3e36af --- /dev/null +++ b/resilience/docs/output/limit-reject.txt @@ -0,0 +1,12 @@ +# GET /demo/limit/reject (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "callers": 10, + "succeeded": 2, + "failed": { + "org.springframework.resilience.InvocationRejectedException": 8 + }, + "max concurrently inside the method": 2, + "elapsed (ms)": 205 +} diff --git a/resilience/docs/output/metrics-names.txt b/resilience/docs/output/metrics-names.txt new file mode 100644 index 0000000..603c01b --- /dev/null +++ b/resilience/docs/output/metrics-names.txt @@ -0,0 +1,14 @@ +# Spring retry has no metrics of its own; the only meter is the one RetryEvents registers. +$ curl /actuator/metrics | grep -iE 'retry|resilience4j' +app.retry.failures +resilience4j.bulkhead.available.concurrent.calls +resilience4j.bulkhead.max.allowed.concurrent.calls +resilience4j.circuitbreaker.buffered.calls +resilience4j.circuitbreaker.calls +resilience4j.circuitbreaker.failure.rate +resilience4j.circuitbreaker.not.permitted.calls +resilience4j.circuitbreaker.slow.call.rate +resilience4j.circuitbreaker.slow.calls +resilience4j.circuitbreaker.state +resilience4j.retry.calls +resilience4j.timelimiter.calls diff --git a/resilience/docs/output/proxy-gateway.txt b/resilience/docs/output/proxy-gateway.txt new file mode 100644 index 0000000..978c678 --- /dev/null +++ b/resilience/docs/output/proxy-gateway.txt @@ -0,0 +1,7 @@ +# GET /demo/proxy/gateway (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +[ + "proxy type: CGLIB", + "org.springframework.resilience.annotation.RetryAnnotationBeanPostProcessor$RetryAnnotationInterceptor order=2147483647" +] diff --git a/resilience/docs/output/proxy-payments.txt b/resilience/docs/output/proxy-payments.txt new file mode 100644 index 0000000..d916eaa --- /dev/null +++ b/resilience/docs/output/proxy-payments.txt @@ -0,0 +1,12 @@ +# GET /demo/proxy/payments (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +[ + "proxy type: CGLIB", + "org.springframework.resilience.annotation.RetryAnnotationBeanPostProcessor$RetryAnnotationInterceptor order=2147483647", + "org.springframework.aop.interceptor.ExposeInvocationInterceptor order=-2147483647", + "org.springframework.aop.aspectj.AspectJAroundAdvice order=2147483642", + "org.springframework.aop.aspectj.AspectJAroundAdvice order=2147483643", + "org.springframework.aop.aspectj.AspectJAroundAdvice order=2147483645", + "org.springframework.aop.aspectj.AspectJAroundAdvice order=2147483646" +] diff --git a/resilience/docs/output/proxy-reports.txt b/resilience/docs/output/proxy-reports.txt new file mode 100644 index 0000000..de5481a --- /dev/null +++ b/resilience/docs/output/proxy-reports.txt @@ -0,0 +1,8 @@ +# GET /demo/proxy/reports (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +[ + "proxy type: CGLIB", + "org.springframework.resilience.annotation.ConcurrencyLimitBeanPostProcessor$ConcurrencyLimitInterceptor order=2147483647", + "org.springframework.resilience.annotation.RetryAnnotationBeanPostProcessor$RetryAnnotationInterceptor order=2147483647" +] diff --git a/resilience/docs/output/proxy-stock-writer.txt b/resilience/docs/output/proxy-stock-writer.txt new file mode 100644 index 0000000..20854e2 --- /dev/null +++ b/resilience/docs/output/proxy-stock-writer.txt @@ -0,0 +1,8 @@ +# GET /demo/proxy/stock-writer (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +[ + "proxy type: CGLIB", + "org.springframework.resilience.annotation.RetryAnnotationBeanPostProcessor$RetryAnnotationInterceptor order=2147483647", + "org.springframework.transaction.interceptor.TransactionInterceptor order=2147483647" +] diff --git a/resilience/docs/output/r4j-breaker.txt b/resilience/docs/output/r4j-breaker.txt new file mode 100644 index 0000000..a702d7f --- /dev/null +++ b/resilience/docs/output/r4j-breaker.txt @@ -0,0 +1,22 @@ +# GET /demo/r4j/breaker (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "calls made": 11, + "calls that reached the downstream": 8, + "of which before the breaker opened": 5, + "outcomes": [ + "1: TransientException: payment gateway 503 (call 1) [state after: CLOSED]", + "2: TransientException: payment gateway 503 (call 2) [state after: CLOSED]", + "3: TransientException: payment gateway 503 (call 3) [state after: CLOSED]", + "4: TransientException: payment gateway 503 (call 4) [state after: CLOSED]", + "5: TransientException: payment gateway 503 (call 5) [state after: OPEN]", + "6: CallNotPermittedException: CircuitBreaker 'payments' is OPEN and does not permit further calls [state after: OPEN]", + "7: CallNotPermittedException: CircuitBreaker 'payments' is OPEN and does not permit further calls [state after: OPEN]", + "8: CallNotPermittedException: CircuitBreaker 'payments' is OPEN and does not permit further calls [state after: OPEN]", + "-- 2.1 s later, downstream recovered --", + "9: returned charged [state after: HALF_OPEN]", + "10: returned charged [state after: CLOSED]", + "11: returned charged [state after: CLOSED]" + ] +} diff --git a/resilience/docs/output/r4j-combo.txt b/resilience/docs/output/r4j-combo.txt new file mode 100644 index 0000000..b005c5b --- /dev/null +++ b/resilience/docs/output/r4j-combo.txt @@ -0,0 +1,31 @@ +# GET /demo/r4j/combo (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "call 1": { + "caller got": "TransientException: payment gateway 503 (call 4)", + "method body ran": 4, + "elapsed (ms)": 35, + "breaker: buffered / failed / not permitted": "4 / 4 / 0", + "breaker state": "CLOSED" + }, + "call 2": { + "caller got": "CallNotPermittedException: CircuitBreaker 'combo' is OPEN and does not permit further calls", + "method body ran": 1, + "elapsed (ms)": 33, + "breaker: buffered / failed / not permitted": "5 / 5 / 3", + "breaker state": "OPEN" + }, + "MethodRetryEvents": [ + "chargeWithBoth failed with TransientException -> will retry", + "chargeWithBoth failed with TransientException -> will retry", + "chargeWithBoth failed with TransientException -> will retry", + "chargeWithBoth failed with TransientException -> will retry", + "chargeWithBoth failed with RetryException -> retry aborted", + "chargeWithBoth failed with TransientException -> will retry", + "chargeWithBoth failed with CallNotPermittedException -> will retry", + "chargeWithBoth failed with CallNotPermittedException -> will retry", + "chargeWithBoth failed with CallNotPermittedException -> will retry", + "chargeWithBoth failed with RetryException -> retry aborted" + ] +} diff --git a/resilience/docs/output/r4j-retry.txt b/resilience/docs/output/r4j-retry.txt new file mode 100644 index 0000000..4e7778c --- /dev/null +++ b/resilience/docs/output/r4j-retry.txt @@ -0,0 +1,17 @@ +# GET /demo/r4j/retry (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "invocations": 3, + "started at (ms)": [ + 7, + 26, + 36 + ], + "gaps (ms)": [ + 19, + 10 + ], + "elapsed (ms)": 37, + "caller": "threw: com.ankurm.resilience.support.TransientException: payment gateway 503 (call 3)" +} diff --git a/resilience/docs/output/r4j-timelimiter.txt b/resilience/docs/output/r4j-timelimiter.txt new file mode 100644 index 0000000..eb12f39 --- /dev/null +++ b/resilience/docs/output/r4j-timelimiter.txt @@ -0,0 +1,12 @@ +# GET /demo/r4j/timelimiter (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "invocations": 1, + "started at (ms)": [ + 11 + ], + "gaps (ms)": [], + "elapsed (ms)": 512, + "caller": "threw: java.util.concurrent.TimeoutException: TimeLimiter 'slow' recorded a timeout exception." +} diff --git a/resilience/docs/output/retry-cause.txt b/resilience/docs/output/retry-cause.txt new file mode 100644 index 0000000..bb14b7b --- /dev/null +++ b/resilience/docs/output/retry-cause.txt @@ -0,0 +1,23 @@ +# GET /demo/retry/cause (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "invocations": 3, + "started at (ms)": [ + 1, + 11, + 23 + ], + "gaps (ms)": [ + 10, + 12 + ], + "elapsed (ms)": 26, + "caller": "threw: java.io.UncheckedIOException: java.io.IOException: socket reset on attempt 3", + "MethodRetryEvents": [ + "wrappedCause failed with UncheckedIOException -> will retry", + "wrappedCause failed with UncheckedIOException -> will retry", + "wrappedCause failed with UncheckedIOException -> will retry", + "wrappedCause failed with RetryException -> retry aborted" + ] +} diff --git a/resilience/docs/output/retry-defaults.txt b/resilience/docs/output/retry-defaults.txt new file mode 100644 index 0000000..fb8ec84 --- /dev/null +++ b/resilience/docs/output/retry-defaults.txt @@ -0,0 +1,21 @@ +# GET /demo/retry/defaults (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "invocations": 3, + "started at (ms)": [ + 27, + 1036, + 2037 + ], + "gaps (ms)": [ + 1009, + 1001 + ], + "elapsed (ms)": 2037, + "caller": "returned: ok after 3 invocation(s)", + "MethodRetryEvents": [ + "defaults failed with TransientException -> will retry", + "defaults failed with TransientException -> will retry" + ] +} diff --git a/resilience/docs/output/retry-disabled.txt b/resilience/docs/output/retry-disabled.txt new file mode 100644 index 0000000..43cd9ab --- /dev/null +++ b/resilience/docs/output/retry-disabled.txt @@ -0,0 +1,14 @@ +# The same @Retryable method with @EnableResilientMethods switched off (demo.resilience.enabled=false) + +{ + "invocations": 1, + "started at (ms)": [ + 0 + ], + "gaps (ms)": [], + "elapsed (ms)": 0, + "caller": "threw: com.ankurm.resilience.support.TransientException: attempt 1 failed" +} + +# Startup log lines mentioning retry or resilience: +(none) diff --git a/resilience/docs/output/retry-exhausted.txt b/resilience/docs/output/retry-exhausted.txt new file mode 100644 index 0000000..2d75d17 --- /dev/null +++ b/resilience/docs/output/retry-exhausted.txt @@ -0,0 +1,26 @@ +# GET /demo/retry/exhausted (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "invocations": 4, + "started at (ms)": [ + 0, + 1001, + 2002, + 3004 + ], + "gaps (ms)": [ + 1001, + 1001, + 1002 + ], + "elapsed (ms)": 3008, + "caller": "threw: com.ankurm.resilience.support.TransientException: attempt 4 failed", + "MethodRetryEvents": [ + "defaults failed with TransientException -> will retry", + "defaults failed with TransientException -> will retry", + "defaults failed with TransientException -> will retry", + "defaults failed with TransientException -> will retry", + "defaults failed with RetryException -> retry aborted" + ] +} diff --git a/resilience/docs/output/retry-exponential.txt b/resilience/docs/output/retry-exponential.txt new file mode 100644 index 0000000..1c246af --- /dev/null +++ b/resilience/docs/output/retry-exponential.txt @@ -0,0 +1,32 @@ +# GET /demo/retry/exponential (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "invocations": 6, + "started at (ms)": [ + 3, + 104, + 304, + 705, + 1206, + 1707 + ], + "gaps (ms)": [ + 101, + 200, + 401, + 501, + 501 + ], + "elapsed (ms)": 1707, + "caller": "threw: com.ankurm.resilience.support.TransientException: attempt 6 failed", + "MethodRetryEvents": [ + "exponential failed with TransientException -> will retry", + "exponential failed with TransientException -> will retry", + "exponential failed with TransientException -> will retry", + "exponential failed with TransientException -> will retry", + "exponential failed with TransientException -> will retry", + "exponential failed with TransientException -> will retry", + "exponential failed with RetryException -> retry aborted" + ] +} diff --git a/resilience/docs/output/retry-future.txt b/resilience/docs/output/retry-future.txt new file mode 100644 index 0000000..806a54f --- /dev/null +++ b/resilience/docs/output/retry-future.txt @@ -0,0 +1,12 @@ +# GET /demo/retry/future (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "invocations": 1, + "started at (ms)": [ + 0 + ], + "gaps (ms)": [], + "elapsed (ms)": 1, + "caller": "threw: com.ankurm.resilience.support.TransientException: attempt 1 failed" +} diff --git a/resilience/docs/output/retry-hang.txt b/resilience/docs/output/retry-hang.txt new file mode 100644 index 0000000..219d84a --- /dev/null +++ b/resilience/docs/output/retry-hang.txt @@ -0,0 +1,16 @@ +# GET /demo/retry/hang (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "invocations": 1, + "started at (ms)": [ + 1 + ], + "gaps (ms)": [], + "elapsed (ms)": 1502, + "caller": "threw: com.ankurm.resilience.support.TransientException: slow failure", + "MethodRetryEvents": [ + "hangingWithTimeout failed with TransientException -> will retry", + "hangingWithTimeout failed with RetryException -> retry aborted" + ] +} diff --git a/resilience/docs/output/retry-includes.txt b/resilience/docs/output/retry-includes.txt new file mode 100644 index 0000000..8e29d0f --- /dev/null +++ b/resilience/docs/output/retry-includes.txt @@ -0,0 +1,16 @@ +# GET /demo/retry/includes (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "invocations": 1, + "started at (ms)": [ + 1 + ], + "gaps (ms)": [], + "elapsed (ms)": 0, + "caller": "threw: java.lang.IllegalArgumentException: not in includes", + "MethodRetryEvents": [ + "onlyIllegalState failed with IllegalArgumentException -> will retry", + "onlyIllegalState failed with RetryException -> retry aborted" + ] +} diff --git a/resilience/docs/output/retry-jitter.txt b/resilience/docs/output/retry-jitter.txt new file mode 100644 index 0000000..9a645f8 --- /dev/null +++ b/resilience/docs/output/retry-jitter.txt @@ -0,0 +1,35 @@ +# GET /demo/retry/jitter (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "invocations": 7, + "started at (ms)": [ + 0, + 254, + 465, + 761, + 1055, + 1328, + 1621 + ], + "gaps (ms)": [ + 254, + 211, + 296, + 294, + 273, + 293 + ], + "elapsed (ms)": 1622, + "caller": "threw: com.ankurm.resilience.support.TransientException: attempt 7 failed", + "MethodRetryEvents": [ + "jittered failed with TransientException -> will retry", + "jittered failed with TransientException -> will retry", + "jittered failed with TransientException -> will retry", + "jittered failed with TransientException -> will retry", + "jittered failed with TransientException -> will retry", + "jittered failed with TransientException -> will retry", + "jittered failed with TransientException -> will retry", + "jittered failed with RetryException -> retry aborted" + ] +} diff --git a/resilience/docs/output/retry-mono.txt b/resilience/docs/output/retry-mono.txt new file mode 100644 index 0000000..38d78bc --- /dev/null +++ b/resilience/docs/output/retry-mono.txt @@ -0,0 +1,21 @@ +# GET /demo/retry/mono (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "invocations": 3, + "started at (ms)": [ + 37, + 65, + 75 + ], + "gaps (ms)": [ + 28, + 10 + ], + "elapsed (ms)": 75, + "caller": "returned: ok after 3 invocation(s)", + "MethodRetryEvents": [ + "mono failed with TransientException -> will retry", + "mono failed with TransientException -> will retry" + ] +} diff --git a/resilience/docs/output/retry-self-invocation.txt b/resilience/docs/output/retry-self-invocation.txt new file mode 100644 index 0000000..5364794 --- /dev/null +++ b/resilience/docs/output/retry-self-invocation.txt @@ -0,0 +1,12 @@ +# GET /demo/retry/self-invocation (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "invocations": 1, + "started at (ms)": [ + 0 + ], + "gaps (ms)": [], + "elapsed (ms)": 0, + "caller": "threw: com.ankurm.resilience.support.TransientException: attempt 1 failed" +} diff --git a/resilience/docs/output/retry-timeout.txt b/resilience/docs/output/retry-timeout.txt new file mode 100644 index 0000000..decc5b5 --- /dev/null +++ b/resilience/docs/output/retry-timeout.txt @@ -0,0 +1,23 @@ +# GET /demo/retry/timeout (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "invocations": 3, + "started at (ms)": [ + 303, + 705, + 1106 + ], + "gaps (ms)": [ + 402, + 401 + ], + "elapsed (ms)": 1105, + "caller": "threw: com.ankurm.resilience.support.TransientException: attempt 3 failed", + "MethodRetryEvents": [ + "slowWithTimeout failed with TransientException -> will retry", + "slowWithTimeout failed with TransientException -> will retry", + "slowWithTimeout failed with TransientException -> will retry", + "slowWithTimeout failed with RetryException -> retry aborted" + ] +} diff --git a/resilience/docs/output/tx-joined.txt b/resilience/docs/output/tx-joined.txt new file mode 100644 index 0000000..f039887 --- /dev/null +++ b/resilience/docs/output/tx-joined.txt @@ -0,0 +1,12 @@ +# GET /demo/tx/joined (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "caller got": "org.springframework.transaction.UnexpectedRollbackException: Transaction rolled back because it has been marked as rollback-only", + "attempts": [ + "attempt 1: transaction active=true, connection holder @6135f416", + "attempt 2: transaction active=true, connection holder @6135f416", + "attempt 3: transaction active=true, connection holder @6135f416" + ], + "rows in stock_movement": 0 +} diff --git a/resilience/docs/output/tx-standalone.txt b/resilience/docs/output/tx-standalone.txt new file mode 100644 index 0000000..bd265dc --- /dev/null +++ b/resilience/docs/output/tx-standalone.txt @@ -0,0 +1,12 @@ +# GET /demo/tx/standalone (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25) +# Source: src/main/java/com/ankurm/resilience/web/DemoController.java + +{ + "caller got": "success", + "attempts": [ + "attempt 1: transaction active=true, connection holder @6aa73704", + "attempt 2: transaction active=true, connection holder @1978b580", + "attempt 3: transaction active=true, connection holder @7d5534c2" + ], + "rows in stock_movement": 1 +} diff --git a/resilience/pom.xml b/resilience/pom.xml new file mode 100644 index 0000000..c39ed67 --- /dev/null +++ b/resilience/pom.xml @@ -0,0 +1,82 @@ + + + 4.0.0 + + + org.springframework.boot + spring-boot-starter-parent + 4.1.1 + + + + com.ankurm + resilience + 1.0.0 + resilience + Spring Framework 7 @Retryable and @ConcurrencyLimit, and what is left for Resilience4j + + + 25 + + 2.4.0 + + + + + org.springframework.boot + spring-boot-starter-webmvc + + + org.springframework.boot + spring-boot-starter-actuator + + + + org.springframework.boot + spring-boot-starter-jdbc + + + com.h2database + h2 + runtime + + + io.projectreactor + reactor-core + + + + + io.github.resilience4j + resilience4j-spring-boot4 + ${resilience4j.version} + + + org.springframework.boot + spring-boot-starter-aspectj + + + io.micrometer + micrometer-registry-prometheus + runtime + + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + diff --git a/resilience/scripts/demo-api-surface.sh b/resilience/scripts/demo-api-surface.sh new file mode 100644 index 0000000..369de27 --- /dev/null +++ b/resilience/scripts/demo-api-surface.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Which attributes exist in which 7.0.x release, read by reflection from the jars themselves. +# Needs spring-context/spring-core 7.0.0-7.0.3 in the local repository, e.g. +# mvn dependency:get -Dartifact=org.springframework:spring-context:7.0.0 (and 7.0.1, 7.0.2, 7.0.3, +# plus the matching spring-core) +# -> docs/output/api-surface.txt +set -uo pipefail +source "$(dirname "$0")/env.sh" +M2="${M2:-$HOME/.m2/repository}" +S="$M2/org/springframework" +cat > /tmp/api-surface.jsh <<'JSH' +String attrs(Class c) { var s = new java.util.TreeSet(); for (var m : c.getDeclaredMethods()) if (java.lang.reflect.Modifier.isPublic(m.getModifiers())) s.add(m.getName()); return s.toString(); } +System.out.println(" @Retryable " + attrs(org.springframework.resilience.annotation.Retryable.class)); +System.out.println(" @ConcurrencyLimit " + attrs(org.springframework.resilience.annotation.ConcurrencyLimit.class)); +System.out.println(" RetryPolicy.Builder " + attrs(org.springframework.core.retry.RetryPolicy.Builder.class)); +System.out.println(" RetryListener " + attrs(org.springframework.core.retry.RetryListener.class)); +/exit +JSH +{ + echo "# Public API of the resilience support, per Spring Framework release" + for v in 7.0.0 7.0.1 7.0.2 7.0.3 7.0.9; do + echo; echo "## $v" + CP="$S/spring-context/$v/spring-context-$v.jar:$S/spring-core/$v/spring-core-$v.jar:$S/spring-aop/7.0.9/spring-aop-7.0.9.jar:$S/spring-beans/7.0.9/spring-beans-7.0.9.jar:$(ls "$M2"/org/jspecify/jspecify/*/jspecify-*.jar | head -1)" + jshell --class-path "$CP" -q /tmp/api-surface.jsh 2>&1 | grep -v 'Picked up' | sed 's/^jshell> //' + done +} > "$OUT/api-surface.txt" +cat "$OUT/api-surface.txt" diff --git a/resilience/scripts/demo.sh b/resilience/scripts/demo.sh new file mode 100644 index 0000000..ff080f7 --- /dev/null +++ b/resilience/scripts/demo.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Run demo scenarios against a running application and commit each report. +# ./scripts/demo.sh retry defaults exhausted ... -> docs/output/retry-.txt +set -uo pipefail +source "$(dirname "$0")/env.sh" +group="$1"; shift +for scenario in "$@"; do + file="$OUT/$group-$scenario.txt" + { + echo "# GET /demo/$group/$scenario (Spring Framework 7.0.9, Resilience4j 2.4.0, JDK 25)" + echo "# Source: src/main/java/com/ankurm/resilience/web/DemoController.java" + echo + curl -s "$BASE/demo/$group/$scenario" | python3 -m json.tool + } > "$file" + echo "wrote $file" +done diff --git a/resilience/scripts/env.sh b/resilience/scripts/env.sh new file mode 100644 index 0000000..859741c --- /dev/null +++ b/resilience/scripts/env.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Shared settings for every script in this module. +MODULE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +OUT="$MODULE_DIR/docs/output" +JAR="$MODULE_DIR/target/resilience-1.0.0.jar" +PORT="${PORT:-8081}" +BASE="http://localhost:$PORT" +PIDFILE="$MODULE_DIR/target/app.pid" +LOG="$MODULE_DIR/target/app.log" +mkdir -p "$OUT" diff --git a/resilience/scripts/run-all.sh b/resilience/scripts/run-all.sh new file mode 100644 index 0000000..79b4a05 --- /dev/null +++ b/resilience/scripts/run-all.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Regenerate every transcript under docs/output/. About a minute and a half - several scenarios +# sleep through Spring's one-second default delay on purpose. +set -euo pipefail +cd "$(dirname "$0")/.." +source scripts/env.sh +mvn -q -DskipTests package +./scripts/run.sh +./scripts/demo.sh retry defaults exhausted exponential jitter includes cause future mono timeout hang self-invocation +./scripts/demo.sh tx standalone joined +./scripts/demo.sh limit none block reject limit-and-retry r4j-bulkhead +./scripts/demo.sh r4j retry breaker combo timelimiter +./scripts/demo.sh proxy payments stock-writer reports gateway +{ + echo "# Spring retry has no metrics of its own; the only meter is the one RetryEvents registers." + echo "\$ curl /actuator/metrics | grep -iE 'retry|resilience4j'" + curl -s "$BASE/actuator/metrics" | python3 -c 'import json,sys +print("\n".join(n for n in json.load(sys.stdin)["names"] if "retry" in n or "resilience4j" in n))' +} > "$OUT/metrics-names.txt" +./scripts/stop.sh +EXTRA_ARGS=--demo.resilience.enabled=false ./scripts/run.sh +{ + echo "# The same @Retryable method with @EnableResilientMethods switched off (demo.resilience.enabled=false)" + echo + curl -s "$BASE/demo/retry/defaults" | python3 -m json.tool + echo + echo "# Startup log lines mentioning retry or resilience:" + grep -E 'Retr|Resilient|ConcurrencyLimit' "$LOG" | grep -v 'Picked up' || echo "(none)" +} > "$OUT/retry-disabled.txt" +./scripts/stop.sh +./scripts/demo-api-surface.sh +echo "Regenerated: $(ls docs/output | wc -l) files in docs/output/" diff --git a/resilience/scripts/run.sh b/resilience/scripts/run.sh new file mode 100644 index 0000000..482879d --- /dev/null +++ b/resilience/scripts/run.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Start the application (optional comma-separated profiles) and wait until it +# answers. Any instance started by a previous run is stopped first, by PID file - never by +# pattern-matching the process list, which can match (and kill) the calling shell. +# ./scripts/run.sh +# EXTRA_ARGS=--demo.resilience.enabled=false ./scripts/run.sh +set -euo pipefail +source "$(dirname "$0")/env.sh" +"$MODULE_DIR/scripts/stop.sh" +PROFILES="${1:-}" +[ -f "$JAR" ] || (cd "$MODULE_DIR" && mvn -q -DskipTests package) +nohup java -jar "$JAR" --server.port="$PORT" ${PROFILES:+--spring.profiles.active=$PROFILES} ${EXTRA_ARGS:-} \ + > "$LOG" 2>&1 < /dev/null & +echo $! > "$PIDFILE" +for _ in $(seq 1 60); do + if curl -s -o /dev/null "$BASE/actuator/health"; then exit 0; fi + if ! kill -0 "$(cat "$PIDFILE")" 2>/dev/null; then echo "application exited - see $LOG" >&2; exit 1; fi + sleep 0.5 +done +echo "application did not start within 30s - see $LOG" >&2; exit 1 diff --git a/resilience/scripts/stop.sh b/resilience/scripts/stop.sh new file mode 100644 index 0000000..378cd88 --- /dev/null +++ b/resilience/scripts/stop.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +source "$(dirname "$0")/env.sh" +if [ -f "$PIDFILE" ]; then + PID="$(cat "$PIDFILE")" + kill "$PID" 2>/dev/null || true + for _ in $(seq 1 40); do kill -0 "$PID" 2>/dev/null || break; sleep 0.25; done + kill -9 "$PID" 2>/dev/null || true + rm -f "$PIDFILE" +fi diff --git a/resilience/src/main/java/com/ankurm/resilience/ResilienceApplication.java b/resilience/src/main/java/com/ankurm/resilience/ResilienceApplication.java new file mode 100644 index 0000000..bc50b06 --- /dev/null +++ b/resilience/src/main/java/com/ankurm/resilience/ResilienceApplication.java @@ -0,0 +1,12 @@ +package com.ankurm.resilience; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class ResilienceApplication { + + public static void main(String[] args) { + SpringApplication.run(ResilienceApplication.class, args); + } +} diff --git a/resilience/src/main/java/com/ankurm/resilience/ResilienceConfig.java b/resilience/src/main/java/com/ankurm/resilience/ResilienceConfig.java new file mode 100644 index 0000000..d60adb9 --- /dev/null +++ b/resilience/src/main/java/com/ankurm/resilience/ResilienceConfig.java @@ -0,0 +1,17 @@ +package com.ankurm.resilience; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty; +import org.springframework.context.annotation.Configuration; +import org.springframework.resilience.annotation.EnableResilientMethods; + +/** + * Spring Boot 4.1 does not switch the resilience annotations on for you. Without this, + * {@code @Retryable} and {@code @ConcurrencyLimit} are inert - no proxy, no error, no log line. + * The {@code demo.resilience.enabled=false} switch exists only so the article can show that + * (docs/02-enabling.md). + */ +@Configuration +@EnableResilientMethods +@ConditionalOnBooleanProperty(name = "demo.resilience.enabled", matchIfMissing = true) +public class ResilienceConfig { +} diff --git a/resilience/src/main/java/com/ankurm/resilience/limit/ReportService.java b/resilience/src/main/java/com/ankurm/resilience/limit/ReportService.java new file mode 100644 index 0000000..1c102fe --- /dev/null +++ b/resilience/src/main/java/com/ankurm/resilience/limit/ReportService.java @@ -0,0 +1,82 @@ +package com.ankurm.resilience.limit; + +import com.ankurm.resilience.support.CallLog; + +import org.springframework.resilience.annotation.ConcurrencyLimit; +import org.springframework.resilience.annotation.Retryable; +import org.springframework.stereotype.Service; + +/** + * An expensive operation that must not run more than twice at once. The default policy BLOCKS + * callers beyond the limit; REJECT (added in 7.0.3) throws + * {@code InvocationRejectedException} instead. See docs/04-concurrency-limit.md. + */ +@Service +public class ReportService { + + private final CallLog log; + + public ReportService(CallLog log) { + this.log = log; + } + + @ConcurrencyLimit(2) + public String blocking() { + return work(); + } + + @ConcurrencyLimit(limit = 2, policy = ConcurrencyLimit.ThrottlePolicy.REJECT) + public String rejecting() { + return work(); + } + + private final ThreadLocal attemptsOnThisThread = ThreadLocal.withInitial(() -> 0); + + /** + * Both annotations on one method. Every caller's first attempt fails and is retried after + * 300 ms. Whether the permit is held during that 300 ms depends on which interceptor is + * outermost - measured in docs/output/limit-and-retry.txt. + */ + @ConcurrencyLimit(1) + @Retryable(maxRetries = 1, delay = 300) + public String limitedAndRetried() { + int attempt = attemptsOnThisThread.get() + 1; + attemptsOnThisThread.set(attempt); + log.enter(); + log.record(); + try { + Thread.sleep(50); + if (attempt == 1) { + throw new IllegalStateException("first attempt fails"); + } + attemptsOnThisThread.remove(); + return "ok"; + } + catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + return "interrupted"; + } + finally { + log.exit(); + } + } + + public String unlimited() { + return work(); + } + + private String work() { + log.enter(); + log.record(); + try { + Thread.sleep(200); + } + catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + finally { + log.exit(); + } + return "rendered"; + } +} diff --git a/resilience/src/main/java/com/ankurm/resilience/r4j/PaymentClient.java b/resilience/src/main/java/com/ankurm/resilience/r4j/PaymentClient.java new file mode 100644 index 0000000..44b8597 --- /dev/null +++ b/resilience/src/main/java/com/ankurm/resilience/r4j/PaymentClient.java @@ -0,0 +1,95 @@ +package com.ankurm.resilience.r4j; + +import java.util.concurrent.CompletableFuture; + +import com.ankurm.resilience.support.CallLog; +import com.ankurm.resilience.support.TransientException; +import io.github.resilience4j.bulkhead.annotation.Bulkhead; +import io.github.resilience4j.circuitbreaker.annotation.CircuitBreaker; +import io.github.resilience4j.retry.annotation.Retry; +import io.github.resilience4j.timelimiter.annotation.TimeLimiter; + +import org.springframework.resilience.annotation.Retryable; +import org.springframework.stereotype.Service; + +/** + * The same kind of downstream, guarded by Resilience4j 2.4.0 instead - the things Spring Framework + * 7 does not do (circuit breaking, time limiting, a rejecting bulkhead with metrics), and one + * method that carries annotations from both libraries to show how they nest. + * Instances are configured in application.yaml. See docs/06-what-is-left-for-resilience4j.md. + */ +@Service +public class PaymentClient { + + private final CallLog log; + private volatile boolean failing = true; + + public PaymentClient(CallLog log) { + this.log = log; + } + + public void setFailing(boolean failing) { + this.failing = failing; + } + + private String call() { + int n = log.record(); + if (failing) { + throw new TransientException("payment gateway 503 (call " + n + ")"); + } + return "charged"; + } + + /** Resilience4j retry: maxAttempts counts the FIRST call too. */ + @Retry(name = "payments") + public String r4jRetry() { + return call(); + } + + @CircuitBreaker(name = "payments") + public String charge() { + return call(); + } + + /** Both libraries on one method. Which one is on the outside decides what the breaker counts. */ + @CircuitBreaker(name = "combo") + @Retryable(maxRetries = 3, delay = 10) + public String chargeWithBoth() { + return call(); + } + + @Bulkhead(name = "reports") + public String bulkheadReport() { + log.enter(); + log.record(); + try { + Thread.sleep(200); + } + catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + finally { + log.exit(); + } + return "rendered"; + } + + /** TimeLimiter only applies to asynchronous return types. */ + @TimeLimiter(name = "slow") + public CompletableFuture slowAsync() { + return CompletableFuture.supplyAsync(() -> { + log.record(); + sleep(2000); + return "too late"; + }); + } + + private static void sleep(long ms) { + try { + Thread.sleep(ms); + } + catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } +} diff --git a/resilience/src/main/java/com/ankurm/resilience/retry/FlakyGateway.java b/resilience/src/main/java/com/ankurm/resilience/retry/FlakyGateway.java new file mode 100644 index 0000000..be47d2b --- /dev/null +++ b/resilience/src/main/java/com/ankurm/resilience/retry/FlakyGateway.java @@ -0,0 +1,128 @@ +package com.ankurm.resilience.retry; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; + +import com.ankurm.resilience.support.CallLog; +import com.ankurm.resilience.support.TransientException; +import reactor.core.publisher.Mono; + +import org.springframework.resilience.annotation.Retryable; +import org.springframework.stereotype.Service; + +/** + * A downstream that fails a configurable number of times before succeeding. Every method records + * each real invocation in {@link CallLog}, so the demos count what actually ran. + * + *

Each method isolates one behaviour of Spring Framework 7's {@code @Retryable}; the + * docs/03-retryable-measured.md chapter has the transcript for each. + */ +@Service +public class FlakyGateway { + + private final CallLog log; + private final AtomicInteger failuresLeft = new AtomicInteger(); + + public FlakyGateway(CallLog log) { + this.log = log; + } + + /** Fail the next {@code n} invocations, then succeed. */ + public void failNext(int n) { + failuresLeft.set(n); + } + + private String attempt() { + int call = log.record(); + if (failuresLeft.getAndDecrement() > 0) { + throw new TransientException("attempt " + call + " failed"); + } + return "ok after " + call + " invocation(s)"; + } + + /** All defaults: any exception, maxRetries = 3, delay = 1000 ms, no back-off growth. */ + @Retryable + public String defaults() { + return attempt(); + } + + /** Exponential: 100, 200, 400, 800 ms - capped at maxDelay. */ + @Retryable(maxRetries = 5, delay = 100, multiplier = 2, maxDelay = 500) + public String exponential() { + return attempt(); + } + + /** delay 200 +/- jitter 100. */ + @Retryable(maxRetries = 6, delay = 200, jitter = 100) + public String jittered() { + return attempt(); + } + + /** Only IllegalStateException is retryable; everything else fails on the first attempt. */ + @Retryable(includes = IllegalStateException.class, delay = 10) + public String onlyIllegalState() { + log.record(); + throw new IllegalArgumentException("not in includes"); + } + + /** + * includes = IOException, but what is thrown is an UncheckedIOException (a RuntimeException) + * that WRAPS an IOException. Does the cause count? + */ + @Retryable(includes = IOException.class, maxRetries = 2, delay = 10) + public String wrappedCause() { + int call = log.record(); + throw new UncheckedIOException(new IOException("socket reset on attempt " + call)); + } + + /** + * A CompletableFuture that completes exceptionally. The method itself returns normally, so + * from the interceptor's point of view there is nothing to retry. + */ + @Retryable(delay = 10) + public CompletableFuture future() { + try { + return CompletableFuture.completedFuture(attempt()); + } + catch (RuntimeException ex) { + return CompletableFuture.failedFuture(ex); + } + } + + /** A Mono: retried by re-subscribing, so the deferred body runs again each time. */ + @Retryable(delay = 10) + public Mono mono() { + return Mono.fromCallable(this::attempt); + } + + /** Each attempt takes 300 ms; the whole retry sequence has a 1 s budget. */ + @Retryable(maxRetries = 10, delay = 100, timeout = 1000) + public String slowWithTimeout() { + sleep(300); + return attempt(); + } + + /** One attempt that hangs for 1.5 s, with a 500 ms budget. Is the attempt interrupted? */ + @Retryable(maxRetries = 3, delay = 10, timeout = 500) + public String hangingWithTimeout() { + log.record(); + sleep(1500); + throw new TransientException("slow failure"); + } + + /** Self-invocation: calls the @Retryable method on {@code this}, bypassing the proxy. */ + public String selfInvocation() { + return defaults(); + } + + private static void sleep(long ms) { + try { + Thread.sleep(ms); + } + catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } +} diff --git a/resilience/src/main/java/com/ankurm/resilience/retry/RetryEvents.java b/resilience/src/main/java/com/ankurm/resilience/retry/RetryEvents.java new file mode 100644 index 0000000..0eae0d2 --- /dev/null +++ b/resilience/src/main/java/com/ankurm/resilience/retry/RetryEvents.java @@ -0,0 +1,41 @@ +package com.ankurm.resilience.retry; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import io.micrometer.core.instrument.MeterRegistry; + +import org.springframework.context.event.EventListener; +import org.springframework.resilience.retry.MethodRetryEvent; +import org.springframework.stereotype.Component; + +/** + * Spring's retry support publishes a {@link MethodRetryEvent} per failed attempt and records no + * metrics of its own. This listener is the whole of the observability story unless you write + * one - here it keeps the events for the demos and turns them into a Micrometer counter. + */ +@Component +public class RetryEvents { + + private final List events = new CopyOnWriteArrayList<>(); + private final MeterRegistry registry; + + public RetryEvents(MeterRegistry registry) { + this.registry = registry; + } + + @EventListener + void on(MethodRetryEvent event) { + events.add(event.getMethod().getName() + " failed with " + event.getFailure().getClass().getSimpleName() + + (event.isRetryAborted() ? " -> retry aborted" : " -> will retry")); + registry.counter("app.retry.failures", + "method", event.getMethod().getName(), + "aborted", String.valueOf(event.isRetryAborted())).increment(); + } + + public List drain() { + List copy = List.copyOf(events); + events.clear(); + return copy; + } +} diff --git a/resilience/src/main/java/com/ankurm/resilience/support/CallLog.java b/resilience/src/main/java/com/ankurm/resilience/support/CallLog.java new file mode 100644 index 0000000..df791a8 --- /dev/null +++ b/resilience/src/main/java/com/ankurm/resilience/support/CallLog.java @@ -0,0 +1,66 @@ +package com.ankurm.resilience.support; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.springframework.stereotype.Component; + +/** + * Records every real invocation of a guarded method: when it happened (milliseconds after the + * first call) and on which thread. The demos report from this rather than from the retry + * machinery's own view, so what is counted is what actually ran. + */ +@Component +public class CallLog { + + private final List times = new ArrayList<>(); + private final List threads = new ArrayList<>(); + private final AtomicInteger inFlight = new AtomicInteger(); + private final AtomicInteger maxInFlight = new AtomicInteger(); + private long start; + + public synchronized void reset() { + times.clear(); + threads.clear(); + inFlight.set(0); + maxInFlight.set(0); + start = System.nanoTime(); + } + + public synchronized int record() { + times.add((System.nanoTime() - start) / 1_000_000); + threads.add(Thread.currentThread().isVirtual() ? "virtual" : Thread.currentThread().getName()); + return times.size(); + } + + public void enter() { + int now = inFlight.incrementAndGet(); + maxInFlight.accumulateAndGet(now, Math::max); + } + + public void exit() { + inFlight.decrementAndGet(); + } + + public synchronized int count() { + return times.size(); + } + + public synchronized List times() { + return List.copyOf(times); + } + + /** Gaps between consecutive invocations, in ms: the effective back-off. */ + public synchronized List gaps() { + List gaps = new ArrayList<>(); + for (int i = 1; i < times.size(); i++) { + gaps.add(times.get(i) - times.get(i - 1)); + } + return gaps; + } + + public int maxInFlight() { + return maxInFlight.get(); + } +} diff --git a/resilience/src/main/java/com/ankurm/resilience/support/TransientException.java b/resilience/src/main/java/com/ankurm/resilience/support/TransientException.java new file mode 100644 index 0000000..a10495f --- /dev/null +++ b/resilience/src/main/java/com/ankurm/resilience/support/TransientException.java @@ -0,0 +1,8 @@ +package com.ankurm.resilience.support; + +/** What a flaky downstream throws: worth retrying. */ +public class TransientException extends RuntimeException { + public TransientException(String message) { + super(message); + } +} diff --git a/resilience/src/main/java/com/ankurm/resilience/tx/OrderFacade.java b/resilience/src/main/java/com/ankurm/resilience/tx/OrderFacade.java new file mode 100644 index 0000000..368ab9b --- /dev/null +++ b/resilience/src/main/java/com/ankurm/resilience/tx/OrderFacade.java @@ -0,0 +1,24 @@ +package com.ankurm.resilience.tx; + +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +/** + * The same retryable, transactional method - called from inside an existing transaction. Every + * attempt joins the caller's transaction, the first failure marks it rollback-only, and the + * retry that "succeeds" cannot un-mark it. + */ +@Service +public class OrderFacade { + + private final StockWriter writer; + + public OrderFacade(StockWriter writer) { + this.writer = writer; + } + + @Transactional + public void placeOrder(String sku) { + writer.record(sku); + } +} diff --git a/resilience/src/main/java/com/ankurm/resilience/tx/StockWriter.java b/resilience/src/main/java/com/ankurm/resilience/tx/StockWriter.java new file mode 100644 index 0000000..29e225b --- /dev/null +++ b/resilience/src/main/java/com/ankurm/resilience/tx/StockWriter.java @@ -0,0 +1,62 @@ +package com.ankurm.resilience.tx; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import javax.sql.DataSource; + +import com.ankurm.resilience.support.TransientException; + +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.resilience.annotation.Retryable; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionSynchronizationManager; + +/** + * {@code @Retryable} and {@code @Transactional} on the same method. Each attempt inserts a row and + * the first two then fail. What ends up in the table - and which transaction each attempt ran in - + * shows which interceptor is outside the other. See docs/05-retry-and-transactions.md. + */ +@Service +public class StockWriter { + + private final JdbcTemplate jdbc; + private final DataSource dataSource; + private final AtomicInteger failuresLeft = new AtomicInteger(); + private final List attempts = new ArrayList<>(); + + public StockWriter(JdbcTemplate jdbc, DataSource dataSource) { + this.jdbc = jdbc; + this.dataSource = dataSource; + } + + public void failNext(int n) { + failuresLeft.set(n); + attempts.clear(); + jdbc.update("delete from stock_movement"); + } + + @Retryable(maxRetries = 3, delay = 10) + @Transactional + public void record(String sku) { + Object holder = TransactionSynchronizationManager.getResource(dataSource); + attempts.add("attempt " + (attempts.size() + 1) + + ": transaction active=" + TransactionSynchronizationManager.isActualTransactionActive() + + ", connection holder @" + Integer.toHexString(System.identityHashCode(holder))); + jdbc.update("insert into stock_movement(sku) values (?)", sku); + if (failuresLeft.getAndDecrement() > 0) { + throw new TransientException("deadlock victim (simulated)"); + } + } + + public int rows() { + Integer n = jdbc.queryForObject("select count(*) from stock_movement", Integer.class); + return n == null ? 0 : n; + } + + public List attempts() { + return List.copyOf(attempts); + } +} diff --git a/resilience/src/main/java/com/ankurm/resilience/web/DemoController.java b/resilience/src/main/java/com/ankurm/resilience/web/DemoController.java new file mode 100644 index 0000000..6e38068 --- /dev/null +++ b/resilience/src/main/java/com/ankurm/resilience/web/DemoController.java @@ -0,0 +1,252 @@ +package com.ankurm.resilience.web; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import com.ankurm.resilience.limit.ReportService; +import com.ankurm.resilience.r4j.PaymentClient; +import com.ankurm.resilience.retry.FlakyGateway; +import com.ankurm.resilience.retry.RetryEvents; +import com.ankurm.resilience.support.CallLog; +import com.ankurm.resilience.tx.OrderFacade; +import com.ankurm.resilience.tx.StockWriter; +import io.github.resilience4j.circuitbreaker.CircuitBreaker; +import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry; + +import org.springframework.aop.framework.Advised; +import org.springframework.aop.support.AopUtils; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RestController; + +/** + * One endpoint per scenario. Each resets the call log, runs the scenario, and reports what really + * happened: how many times the guarded method ran, the gaps between runs, and what the caller got + * back. scripts/demo-*.sh curl these and commit the output. Delete before shipping anything real. + */ +@RestController +public class DemoController { + + private final FlakyGateway gateway; + private final ReportService reports; + private final StockWriter writer; + private final OrderFacade facade; + private final PaymentClient payments; + private final CallLog log; + private final RetryEvents events; + private final CircuitBreakerRegistry breakers; + + public DemoController(FlakyGateway gateway, ReportService reports, StockWriter writer, OrderFacade facade, + PaymentClient payments, CallLog log, RetryEvents events, CircuitBreakerRegistry breakers) { + this.gateway = gateway; + this.reports = reports; + this.writer = writer; + this.facade = facade; + this.payments = payments; + this.log = log; + this.events = events; + this.breakers = breakers; + } + + @GetMapping("/demo/retry/{scenario}") + public Map retry(@PathVariable String scenario) { + log.reset(); + events.drain(); + return switch (scenario) { + case "defaults" -> run(2, gateway::defaults); + case "exhausted" -> run(99, gateway::defaults); + case "exponential" -> run(99, gateway::exponential); + case "jitter" -> run(99, gateway::jittered); + case "includes" -> run(0, gateway::onlyIllegalState); + case "cause" -> run(0, gateway::wrappedCause); + case "future" -> run(99, () -> gateway.future().get(5, TimeUnit.SECONDS)); + case "mono" -> run(2, () -> gateway.mono().block()); + case "timeout" -> run(99, gateway::slowWithTimeout); + case "hang" -> run(0, gateway::hangingWithTimeout); + case "self-invocation" -> run(2, gateway::selfInvocation); + default -> Map.of("error", "unknown scenario " + scenario); + }; + } + + @GetMapping("/demo/tx/{scenario}") + public Map tx(@PathVariable String scenario) { + events.drain(); + writer.failNext(2); + Map out = new LinkedHashMap<>(); + try { + if (scenario.equals("joined")) { + facade.placeOrder("SKU-1"); + } + else { + writer.record("SKU-1"); + } + out.put("caller got", "success"); + } + catch (RuntimeException ex) { + out.put("caller got", ex.getClass().getName() + ": " + ex.getMessage()); + } + out.put("attempts", writer.attempts()); + out.put("rows in stock_movement", writer.rows()); + return out; + } + + @GetMapping("/demo/limit/{scenario}") + public Map limit(@PathVariable String scenario) throws Exception { + log.reset(); + Callable task = switch (scenario) { + case "block" -> reports::blocking; + case "reject" -> reports::rejecting; + case "r4j-bulkhead" -> payments::bulkheadReport; + case "limit-and-retry" -> reports::limitedAndRetried; + default -> reports::unlimited; + }; + return concurrently(scenario.equals("limit-and-retry") ? 2 : 10, task); + } + + @GetMapping("/demo/r4j/{scenario}") + public Map r4j(@PathVariable String scenario) throws Exception { + log.reset(); + events.drain(); + payments.setFailing(true); + return switch (scenario) { + case "retry" -> run(0, payments::r4jRetry); + case "breaker" -> breaker(); + case "combo" -> combo(); + case "timelimiter" -> run(0, () -> payments.slowAsync().get()); + default -> Map.of("error", "unknown scenario " + scenario); + }; + } + + /** The advisor chain on a bean's proxy, outermost first. */ + @GetMapping("/demo/proxy/{bean}") + public List proxy(@PathVariable String bean) { + Object target = switch (bean) { + case "payments" -> payments; + case "stock-writer" -> writer; + case "reports" -> reports; + case "gateway" -> gateway; + default -> throw new IllegalArgumentException(bean); + }; + List chain = new ArrayList<>(); + chain.add("proxy type: " + (AopUtils.isCglibProxy(target) ? "CGLIB" : AopUtils.isJdkDynamicProxy(target) ? "JDK" : "none")); + if (AopUtils.isAopProxy(target) && target instanceof Advised advised) { + for (var advisor : advised.getAdvisors()) { + chain.add(advisor.getAdvice().getClass().getName() + + (advisor instanceof org.springframework.core.Ordered o ? " order=" + o.getOrder() : "")); + } + } + return chain; + } + + private Map breaker() throws InterruptedException { + CircuitBreaker cb = breakers.circuitBreaker("payments"); + cb.reset(); + List outcomes = new ArrayList<>(); + for (int i = 1; i <= 8; i++) { + outcomes.add(i + ": " + outcome(payments::charge) + " [state after: " + cb.getState() + "]"); + } + int reachedWhileClosed = log.count(); + Thread.sleep(2100); + payments.setFailing(false); + outcomes.add("-- 2.1 s later, downstream recovered --"); + for (int i = 9; i <= 11; i++) { + outcomes.add(i + ": " + outcome(payments::charge) + " [state after: " + cb.getState() + "]"); + } + Map out = new LinkedHashMap<>(); + out.put("calls made", 11); + out.put("calls that reached the downstream", log.count()); + out.put("of which before the breaker opened", reachedWhileClosed); + out.put("outcomes", outcomes); + return out; + } + + private Map combo() { + CircuitBreaker cb = breakers.circuitBreaker("combo"); + cb.reset(); + Map out = new LinkedHashMap<>(); + for (int call = 1; call <= 2; call++) { + int before = log.count(); + long start = System.nanoTime(); + String result = outcome(payments::chargeWithBoth); + Map row = new LinkedHashMap<>(); + row.put("caller got", result); + row.put("method body ran", log.count() - before); + row.put("elapsed (ms)", (System.nanoTime() - start) / 1_000_000); + row.put("breaker: buffered / failed / not permitted", cb.getMetrics().getNumberOfBufferedCalls() + " / " + + cb.getMetrics().getNumberOfFailedCalls() + " / " + cb.getMetrics().getNumberOfNotPermittedCalls()); + row.put("breaker state", cb.getState().toString()); + out.put("call " + call, row); + } + out.put("MethodRetryEvents", events.drain()); + return out; + } + + private Map run(int failures, Callable action) { + gateway.failNext(failures); + long start = System.nanoTime(); + Object result; + try { + result = "returned: " + action.call(); + } + catch (Exception ex) { + Throwable t = ex instanceof ExecutionException && ex.getCause() != null ? ex.getCause() : ex; + result = "threw: " + t.getClass().getName() + ": " + t.getMessage(); + } + long elapsed = (System.nanoTime() - start) / 1_000_000; + Map out = new LinkedHashMap<>(); + out.put("invocations", log.count()); + out.put("started at (ms)", log.times()); + out.put("gaps (ms)", log.gaps()); + out.put("elapsed (ms)", elapsed); + out.put("caller", result); + List ev = events.drain(); + if (!ev.isEmpty()) { + out.put("MethodRetryEvents", ev); + } + return out; + } + + private Map concurrently(int callers, Callable task) throws Exception { + long start = System.nanoTime(); + int ok = 0; + Map failures = new LinkedHashMap<>(); + try (var executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(); + for (int i = 0; i < callers; i++) { + futures.add(executor.submit(task)); + } + for (Future f : futures) { + try { + f.get(); + ok++; + } + catch (ExecutionException ex) { + failures.merge(ex.getCause().getClass().getName(), 1, Integer::sum); + } + } + } + Map out = new LinkedHashMap<>(); + out.put("callers", callers); + out.put("succeeded", ok); + out.put("failed", failures); + out.put("max concurrently inside the method", log.maxInFlight()); + out.put("elapsed (ms)", (System.nanoTime() - start) / 1_000_000); + return out; + } + + private static String outcome(Callable action) { + try { + return "returned " + action.call(); + } + catch (Exception ex) { + return ex.getClass().getSimpleName() + ": " + ex.getMessage(); + } + } +} diff --git a/resilience/src/main/resources/application.yaml b/resilience/src/main/resources/application.yaml new file mode 100644 index 0000000..b674cd6 --- /dev/null +++ b/resilience/src/main/resources/application.yaml @@ -0,0 +1,53 @@ +spring: + application: + name: resilience + threads: + virtual: + enabled: true + datasource: + url: jdbc:h2:mem:resilience;DB_CLOSE_DELAY=-1 + +server: + port: 8080 + +management: + endpoints: + web: + exposure: + include: health,metrics,circuitbreakers,circuitbreakerevents,retries,bulkheads,prometheus + endpoint: + health: + show-details: always + health: + circuitbreakers: + enabled: true + +# Resilience4j instances used by PaymentClient. Every value is spelled out: the library defaults +# (sliding window 100, minimum 100 calls, 60 s open) would need 100 calls before the breaker +# could open at all - see docs/06-what-is-left-for-resilience4j.md. +resilience4j: + retry: + instances: + payments: + max-attempts: 3 + wait-duration: 10ms + circuitbreaker: + instances: + payments: &breaker + sliding-window-size: 10 + minimum-number-of-calls: 5 + failure-rate-threshold: 50 + wait-duration-in-open-state: 2s + permitted-number-of-calls-in-half-open-state: 2 + automatic-transition-from-open-to-half-open-enabled: false + combo: *breaker + bulkhead: + instances: + reports: + max-concurrent-calls: 2 + max-wait-duration: 0 + timelimiter: + instances: + slow: + timeout-duration: 500ms + cancel-running-future: true diff --git a/resilience/src/main/resources/schema.sql b/resilience/src/main/resources/schema.sql new file mode 100644 index 0000000..2e144b4 --- /dev/null +++ b/resilience/src/main/resources/schema.sql @@ -0,0 +1,4 @@ +create table if not exists stock_movement ( + id bigint generated by default as identity primary key, + sku varchar(64) not null +); diff --git a/resilience/src/test/java/com/ankurm/resilience/ApiSurfaceTest.java b/resilience/src/test/java/com/ankurm/resilience/ApiSurfaceTest.java new file mode 100644 index 0000000..8bf7755 --- /dev/null +++ b/resilience/src/test/java/com/ankurm/resilience/ApiSurfaceTest.java @@ -0,0 +1,34 @@ +package com.ankurm.resilience; + +import java.util.Arrays; + +import org.junit.jupiter.api.Test; + +import org.springframework.resilience.annotation.ConcurrencyLimit; +import org.springframework.resilience.annotation.EnableResilientMethods; +import org.springframework.resilience.annotation.Retryable; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Defaults the article quotes, read from the annotations rather than from the documentation. */ +class ApiSurfaceTest { + + @Test + void retryableDefaults() throws Exception { + assertThat(Retryable.class.getMethod("maxRetries").getDefaultValue()).isEqualTo(3L); + assertThat(Retryable.class.getMethod("delay").getDefaultValue()).isEqualTo(1000L); + assertThat(Retryable.class.getMethod("multiplier").getDefaultValue()).isEqualTo(1.0d); + assertThat(Retryable.class.getMethod("timeout").getDefaultValue()).isEqualTo(0L); + } + + @Test + void concurrencyLimitHasTwoPolicies() { + assertThat(Arrays.stream(ConcurrencyLimit.ThrottlePolicy.values()).map(Enum::name)) + .containsExactly("BLOCK", "REJECT"); + } + + @Test + void enableResilientMethodsOrderIsLowestMinusOne() throws Exception { + assertThat(EnableResilientMethods.class.getMethod("order").getDefaultValue()).isEqualTo(Integer.MAX_VALUE - 1); + } +} diff --git a/resilience/src/test/java/com/ankurm/resilience/CompositionContractTest.java b/resilience/src/test/java/com/ankurm/resilience/CompositionContractTest.java new file mode 100644 index 0000000..a0d90bf --- /dev/null +++ b/resilience/src/test/java/com/ankurm/resilience/CompositionContractTest.java @@ -0,0 +1,104 @@ +package com.ankurm.resilience; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import com.ankurm.resilience.limit.ReportService; +import com.ankurm.resilience.r4j.PaymentClient; +import com.ankurm.resilience.support.CallLog; +import com.ankurm.resilience.tx.OrderFacade; +import com.ankurm.resilience.tx.StockWriter; +import io.github.resilience4j.circuitbreaker.CallNotPermittedException; +import io.github.resilience4j.circuitbreaker.CircuitBreaker; +import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.resilience.InvocationRejectedException; +import org.springframework.transaction.UnexpectedRollbackException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** How the resilience interceptors nest with transactions, with each other, and with Resilience4j. */ +@SpringBootTest +class CompositionContractTest { + + @Autowired StockWriter writer; + @Autowired OrderFacade facade; + @Autowired ReportService reports; + @Autowired PaymentClient payments; + @Autowired CircuitBreakerRegistry breakers; + @Autowired CallLog log; + + @BeforeEach + void reset() { + log.reset(); + payments.setFailing(true); + } + + @Test + void retryWrapsTheTransactionSoEachAttemptRollsBackOnItsOwn() { + writer.failNext(2); + writer.record("SKU-1"); + assertThat(writer.rows()).isEqualTo(1); + assertThat(writer.attempts()).hasSize(3); + } + + @Test + void insideAnOuterTransactionTheRetryCannotSaveIt() { + writer.failNext(2); + assertThatThrownBy(() -> facade.placeOrder("SKU-1")).isInstanceOf(UnexpectedRollbackException.class); + assertThat(writer.rows()).isZero(); + } + + @Test + void blockPolicyQueuesAndRejectPolicyThrows() throws Exception { + assertThat(runConcurrently(10, reports::blocking)).isEmpty(); + assertThat(log.maxInFlight()).isEqualTo(2); + log.reset(); + assertThat(runConcurrently(10, reports::rejecting)) + .hasSize(8).allSatisfy(t -> assertThat(t).isInstanceOf(InvocationRejectedException.class)); + } + + @Test + void springRetryIsOutsideTheResilience4jBreaker() { + CircuitBreaker cb = breakers.circuitBreaker("combo"); + cb.reset(); + assertThatThrownBy(payments::chargeWithBoth).isInstanceOf(RuntimeException.class); + assertThat(log.count()).isEqualTo(4); + assertThat(cb.getMetrics().getNumberOfBufferedCalls()).isEqualTo(4); // breaker saw every attempt + + assertThatThrownBy(payments::chargeWithBoth).isInstanceOf(CallNotPermittedException.class); + assertThat(cb.getMetrics().getNumberOfNotPermittedCalls()).isEqualTo(3); // retried an open circuit + } + + @Test + void resilience4jMaxAttemptsCountsTheFirstCall() { + assertThatThrownBy(payments::r4jRetry).isInstanceOf(RuntimeException.class); + assertThat(log.count()).isEqualTo(3); + } + + private static List runConcurrently(int n, java.util.concurrent.Callable task) throws Exception { + List failures = new ArrayList<>(); + try (var executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(); + for (int i = 0; i < n; i++) { + futures.add(executor.submit(task)); + } + for (Future f : futures) { + try { + f.get(); + } + catch (java.util.concurrent.ExecutionException ex) { + failures.add(ex.getCause()); + } + } + } + return failures; + } +} diff --git a/resilience/src/test/java/com/ankurm/resilience/RetryableContractTest.java b/resilience/src/test/java/com/ankurm/resilience/RetryableContractTest.java new file mode 100644 index 0000000..17a756f --- /dev/null +++ b/resilience/src/test/java/com/ankurm/resilience/RetryableContractTest.java @@ -0,0 +1,108 @@ +package com.ankurm.resilience; + +import java.io.UncheckedIOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import com.ankurm.resilience.retry.FlakyGateway; +import com.ankurm.resilience.support.CallLog; +import com.ankurm.resilience.support.TransientException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * Pins what Spring Framework 7.0.9's @Retryable actually does, counted by real invocations. + * The one-second default delay makes a few of these slow on purpose. + */ +@SpringBootTest +class RetryableContractTest { + + @Autowired + FlakyGateway gateway; + + @Autowired + CallLog log; + + @BeforeEach + void reset() { + log.reset(); + } + + @Test + void maxRetriesThreeMeansFourInvocations() { + gateway.failNext(99); + assertThatThrownBy(gateway::defaults).isInstanceOf(TransientException.class) + .hasMessage("attempt 4 failed"); // the LAST original exception, not a wrapper + assertThat(log.count()).isEqualTo(4); + assertThat(log.gaps()).allSatisfy(gap -> assertThat(gap).isBetween(950L, 1300L)); + } + + @Test + void exceptionNotInIncludesIsNotRetried() { + assertThatThrownBy(gateway::onlyIllegalState).isInstanceOf(IllegalArgumentException.class); + assertThat(log.count()).isEqualTo(1); + } + + @Test + void includesMatchesAWrappedCause() { + assertThatThrownBy(gateway::wrappedCause).isInstanceOf(UncheckedIOException.class); + assertThat(log.count()).isEqualTo(3); + } + + @Test + void aFailedCompletableFutureIsNotRetried() { + gateway.failNext(99); + assertThatThrownBy(() -> gateway.future().get(5, TimeUnit.SECONDS)) + .isInstanceOf(ExecutionException.class); + assertThat(log.count()).isEqualTo(1); + } + + @Test + void aMonoIsRetriedByResubscribing() { + gateway.failNext(2); + assertThat(gateway.mono().block()).isEqualTo("ok after 3 invocation(s)"); + assertThat(log.count()).isEqualTo(3); + } + + @Test + void timeoutIsABudgetCheckedBetweenAttemptsNotAnInterrupt() { + long start = System.nanoTime(); + assertThatThrownBy(gateway::hangingWithTimeout).isInstanceOf(TransientException.class); + long elapsed = (System.nanoTime() - start) / 1_000_000; + assertThat(log.count()).isEqualTo(1); + assertThat(elapsed).isGreaterThanOrEqualTo(1500); // the 500 ms budget did not cut it short + } + + @Test + void selfInvocationBypassesTheProxy() { + gateway.failNext(1); + assertThatThrownBy(gateway::selfInvocation).isInstanceOf(TransientException.class); + assertThat(log.count()).isEqualTo(1); + } + + @Nested + @SpringBootTest(properties = "demo.resilience.enabled=false") + class WithoutEnableResilientMethods { + + @Autowired + FlakyGateway gateway; + + @Autowired + CallLog log; + + @Test + void theAnnotationIsInert() { + log.reset(); + gateway.failNext(1); + assertThatThrownBy(gateway::defaults).isInstanceOf(TransientException.class); + assertThat(log.count()).isEqualTo(1); + } + } +}