Add llm-gateway module: routing, failover below the tool-calling advisor, per-provider circuit breakers, dollar caps and token limits, tenant-keyed cache; real OpenAI and Anthropic models against a local fake

Co-Authored-By: Claude Sonnet 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01JXVi2GMQ7bR5EmbUFdDj7N
2026-10-09 10:24:57 +00:00
parent 80cd21f89b
commit cdee85d3f4
51 changed files with 2404 additions and 0 deletions
@@ -0,0 +1,82 @@
+package com.ankurm.gateway;
+
+import java.time.Duration;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import io.github.bucket4j.TimeMeter;
+import io.github.resilience4j.circuitbreaker.CircuitBreakerConfig;
+import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry;
+import org.springframework.ai.anthropic.AnthropicChatModel;
+import org.springframework.ai.anthropic.AnthropicChatOptions;
+import org.springframework.ai.chat.model.ChatModel;
+import org.springframework.ai.embedding.EmbeddingModel;
+import org.springframework.ai.openai.OpenAiChatModel;
+import org.springframework.ai.openai.OpenAiChatOptions;
+import org.springframework.beans.factory.ObjectProvider;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+@EnableConfigurationProperties(GatewayProperties.class)
+public class GatewayConfig {
+
+    @Bean
+    Routes routes(GatewayProperties p) {
+        Map<String, Target> byName = new LinkedHashMap<>();
+        for (GatewayProperties.Provider c : p.providers()) {
+            byName.put(c.name(), new Target(c.name(), c.model(), chatModel(c), Price.of(c.inputPrice(), c.outputPrice())));
+        }
+        Map<String, List<Target>> table = new LinkedHashMap<>();
+        p.routes().forEach((hint, names) -> table.put(hint, names.stream().map(byName::get).toList()));
+        return new Routes(table);
+    }
+
+    /** No retries inside the SDK: the gateway's own failover is the retry policy. */
+    public static ChatModel chatModel(GatewayProperties.Provider c) {
+        return switch (c.type()) {
+            case "openai" -> OpenAiChatModel.builder().options(OpenAiChatOptions.builder()
+                    .baseUrl(c.baseUrl()).apiKey(c.apiKey()).model(c.model()).maxRetries(0).build()).build();
+            case "anthropic" -> AnthropicChatModel.builder().options(AnthropicChatOptions.builder()
+                    .baseUrl(c.baseUrl()).apiKey(c.apiKey()).model(c.model()).maxTokens(1024).maxRetries(0).build())
+                    .build();
+            default -> throw new IllegalArgumentException("Unknown provider type " + c.type());
+        };
+    }
+
+    @Bean
+    CircuitBreakerRegistry breakers(GatewayProperties p) {
+        return registry(p.breaker());
+    }
+
+    /** One breaker per provider name, all built from the same settings. */
+    public static CircuitBreakerRegistry registry(GatewayProperties.Breaker b) {
+        return CircuitBreakerRegistry.of(CircuitBreakerConfig.custom()
+                .slidingWindowSize(b.slidingWindowSize())
+                .minimumNumberOfCalls(b.minimumCalls())
+                .failureRateThreshold(b.failureRateThreshold())
+                .waitDurationInOpenState(Duration.ofMillis(b.waitOpenMillis()))
+                .permittedNumberOfCallsInHalfOpenState(1)
+                .build());
+    }
+
+    @Bean
+    Budgets budgets(GatewayProperties p) {
+        return new Budgets(t -> p.tenant(t).budgetMicros());
+    }
+
+    @Bean
+    TokenLimiter tokenLimiter(GatewayProperties p) {
+        return new TokenLimiter(t -> p.tenant(t).tokensPerHour(), TimeMeter.SYSTEM_MILLISECONDS);
+    }
+
+    @Bean
+    Gateway gateway(Routes routes, CircuitBreakerRegistry breakers, Budgets budgets, TokenLimiter limiter,
+            ObjectProvider<EmbeddingModel> embeddings) {
+        EmbeddingModel em = embeddings.getIfAvailable();
+        return new Gateway(routes, targets -> new Failover(targets, breakers, budgets), limiter,
+                em == null ? null : new SemanticCache(em, 0.92), Map.of("refund_order", new OrderTools()));
+    }
+}