Add llm-gateway module: routing, failover below the tool-calling advisor, per-provider circuit breakers, dollar caps and token limits, tenant-keyed cache; real OpenAI and Anthropic models against a local fake

Co-Authored-By: Claude Sonnet 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01JXVi2GMQ7bR5EmbUFdDj7N
2026-10-09 10:24:57 +00:00
parent 80cd21f89b
commit cdee85d3f4
51 changed files with 2404 additions and 0 deletions
@@ -0,0 +1,59 @@
+package com.ankurm.gateway;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import org.springframework.ai.embedding.EmbeddingModel;
+
+/**
+ * Serves a stored answer when a new question embeds close enough to an old one. The key includes
+ * the tenant: a cache shared across tenants answers tenant B with tenant A's data.
+ */
+public final class SemanticCache {
+
+    private record Entry(float[] vector, String answer) {
+    }
+
+    private final EmbeddingModel embeddings;
+
+    private final double threshold;
+
+    private final Map<String, List<Entry>> byKey = new ConcurrentHashMap<>();
+
+    public SemanticCache(EmbeddingModel embeddings, double threshold) {
+        this.embeddings = embeddings;
+        this.threshold = threshold;
+    }
+
+    public Optional<String> lookup(String tenant, String feature, String question) {
+        float[] q = embeddings.embed(question);
+        Entry best = null;
+        double bestScore = -1;
+        for (Entry e : byKey.getOrDefault(tenant + "|" + feature, List.of())) {
+            double s = cosine(q, e.vector());
+            if (s >= threshold && s > bestScore) {
+                best = e;
+                bestScore = s;
+            }
+        }
+        return Optional.ofNullable(best).map(Entry::answer);
+    }
+
+    public void put(String tenant, String feature, String question, String answer) {
+        byKey.computeIfAbsent(tenant + "|" + feature, k -> new CopyOnWriteArrayList<>())
+                .add(new Entry(embeddings.embed(question), answer));
+    }
+
+    static double cosine(float[] a, float[] b) {
+        double dot = 0, na = 0, nb = 0;
+        for (int i = 0; i < a.length; i++) {
+            dot += a[i] * b[i];
+            na += a[i] * a[i];
+            nb += b[i] * b[i];
+        }
+        return dot / (Math.sqrt(na) * Math.sqrt(nb) + 1e-10);
+    }
+}