Add llm-gateway module: routing, failover below the tool-calling advisor, per-provider circuit breakers, dollar caps and token limits, tenant-keyed cache; real OpenAI and Anthropic models against a local fake

Co-Authored-By: Claude Sonnet 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01JXVi2GMQ7bR5EmbUFdDj7N
2026-10-09 10:24:57 +00:00
parent 80cd21f89b
commit cdee85d3f4
51 changed files with 2404 additions and 0 deletions
@@ -0,0 +1,48 @@
+package com.ankurm.gateway;
+
+import java.io.IOException;
+
+import com.anthropic.errors.AnthropicIoException;
+import com.anthropic.errors.AnthropicServiceException;
+import com.openai.errors.OpenAIIoException;
+import com.openai.errors.OpenAIServiceException;
+
+/**
+ * Decides whether a failed model call is worth sending to another provider. Same idea as the
+ * {@code providers} module: the vendor SDKs throw unrelated exception families, so walk the cause
+ * chain for an HTTP status.
+ */
+public final class Failures {
+
+    private Failures() {
+    }
+
+    public static int status(Throwable t) {
+        for (Throwable c = t; c != null; c = c.getCause()) {
+            if (c instanceof ProviderFailure e) {
+                return e.status();
+            }
+            if (c instanceof OpenAIServiceException e) {
+                return e.statusCode();
+            }
+            if (c instanceof AnthropicServiceException e) {
+                return e.statusCode();
+            }
+        }
+        return -1;
+    }
+
+    /** True for overload, rate limit, 5xx and network failures. A 400 or 401 is the caller's problem. */
+    public static boolean isTransient(Throwable t) {
+        int status = status(t);
+        if (status != -1) {
+            return status == 408 || status == 429 || status >= 500;
+        }
+        for (Throwable c = t; c != null; c = c.getCause()) {
+            if (c instanceof OpenAIIoException || c instanceof AnthropicIoException || c instanceof IOException) {
+                return true;
+            }
+        }
+        return false;
+    }
+}