asmhatre/spring-ai
1
0
Fork 0
Code Activity

Add llm-gateway module: routing, failover below the tool-calling advisor, per-provider circuit breakers, dollar caps and token limits, tenant-keyed cache; real OpenAI and Anthropic models against a local fake

Browse Source 
Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01JXVi2GMQ7bR5EmbUFdDj7N
This commit is contained in:
Claude
2026-10-09 10:24:57 +00:00
parent 80cd21f89b
commit cdee85d3f4
51 changed files with 2404 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
llm-gateway/src/test/java/com/ankurm/gateway/HttpTest.java
+133
View File
@@ -0,0 +1,133 @@
package com.ankurm.gateway;
import static org.assertj.core.api.Assertions.assertThat;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import com.ankurm.gateway.support.FakeVendors;
import com.ankurm.gateway.support.Transcript;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.web.server.LocalServerPort;
import org.springframework.test.context.DynamicPropertyRegistry;
import org.springframework.test.context.DynamicPropertySource;
/** The whole application over HTTP: real beans, real models, a fake at the far end. */
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
class HttpTest {
static final FakeVendors VENDORS;
static {
try {
VENDORS = new FakeVendors();
}
catch (java.io.IOException e) {
throw new ExceptionInInitializerError(e);
}
}
@DynamicPropertySource
static void providers(DynamicPropertyRegistry r) {
r.add("gateway.providers[0].name", () -> "openai");
r.add("gateway.providers[0].type", () -> "openai");
r.add("gateway.providers[0].base-url", () -> VENDORS.url() + "/v1");
r.add("gateway.providers[0].api-key", () -> "test-key");
r.add("gateway.providers[0].model", () -> "gpt-6.1-sol");
r.add("gateway.providers[0].input-price", () -> "2.00");
r.add("gateway.providers[0].output-price", () -> "10.00");
r.add("gateway.providers[1].name", () -> "anthropic");
r.add("gateway.providers[1].type", () -> "anthropic");
r.add("gateway.providers[1].base-url", () -> VENDORS.url());
r.add("gateway.providers[1].api-key", () -> "test-key");
r.add("gateway.providers[1].model", () -> "claude-sonnet-5-5");
r.add("gateway.providers[1].input-price", () -> "2.00");
r.add("gateway.providers[1].output-price", () -> "10.00");
r.add("gateway.tenants.poor.budget-micros", () -> "1000");
r.add("gateway.tenants.poor.tokens-per-hour", () -> "100000");
}
@LocalServerPort
int port;
private final HttpClient http = HttpClient.newHttpClient();
@BeforeEach
void reset() {
VENDORS.openai.reset();
VENDORS.anthropic.reset();
}
@AfterAll
static void stop() {
VENDORS.close();
}
private HttpResponse<String> post(String tenant, String json) throws Exception {
HttpRequest.Builder b = HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + port + "/v1/gateway/complete"))
.header("Content-Type", "application/json").POST(HttpRequest.BodyPublishers.ofString(json));
if (tenant != null) {
b.header("X-Tenant-Id", tenant);
}
return http.send(b.build(), HttpResponse.BodyHandlers.ofString());
}
private static String body(String hint, String extra) {
return "{\"featureTag\":\"support\",\"userMessage\":\"What is your return policy?\",\"modelHint\":\"" + hint
+ "\",\"maxTokens\":256" + extra + "}";
}
@Test
void statusCodesAndBodiesOverHttp() throws Exception {
try (Transcript t = new Transcript("10-http.txt", "The gateway over HTTP (real beans, real models, fake vendors)")) {
VENDORS.openai.then(200, FakeVendors.openaiText("30 days.", 47, 52));
HttpResponse<String> ok = post("acme", body("smart", ""));
t.line("1. normal call -> %d %s", ok.statusCode(), ok.body());
assertThat(ok.statusCode()).isEqualTo(200);
assertThat(ok.body()).contains("\"provider\":\"openai\"").contains("\"costMicros\":614");
reset();
VENDORS.openai.then(503, FakeVendors.error("server_error", "overloaded"));
VENDORS.anthropic.then(200, FakeVendors.anthropicText("30 days.", 47, 52));
HttpResponse<String> failover = post("acme", body("smart", ""));
t.line("2. openai returns 503 -> %d %s", failover.statusCode(), failover.body());
assertThat(failover.body()).contains("\"provider\":\"anthropic\"");
reset();
VENDORS.openai.always(503, FakeVendors.error("server_error", "overloaded"));
VENDORS.anthropic.always(529, FakeVendors.error("overloaded_error", "overloaded"));
HttpResponse<String> down = post("acme", body("smart", ""));
t.line("3. both providers down -> %d Retry-After=%s %s", down.statusCode(),
down.headers().firstValue("Retry-After").orElse("-"), down.body());
assertThat(down.statusCode()).isEqualTo(503);
reset();
HttpResponse<String> hint = post("acme", body("locla", ""));
t.line("4. unknown hint -> %d %s", hint.statusCode(), hint.body());
assertThat(hint.statusCode()).isEqualTo(400);
assertThat(VENDORS.openai.requests() + VENDORS.anthropic.requests()).isZero();
HttpResponse<String> tool = post("acme", body("smart", ",\"tools\":[\"drop_tables\"]"));
t.line("5. tool not allow-listed -> %d %s", tool.statusCode(), tool.body());
assertThat(tool.statusCode()).isEqualTo(400);
HttpResponse<String> poor = post("poor", body("smart", ""));
t.line("6. tenant over its cap -> %d %s", poor.statusCode(), poor.body());
assertThat(poor.statusCode()).isEqualTo(429);
assertThat(VENDORS.openai.requests() + VENDORS.anthropic.requests()).isZero();
HttpResponse<String> spoof = post("poor", body("smart", ",\"tenantId\":\"acme\""));
t.line("7. body claims tenant acme, header says poor -> %d (the header wins)", spoof.statusCode());
assertThat(spoof.statusCode()).isEqualTo(429);
HttpResponse<String> none = post(null, body("smart", ""));
t.line("8. no X-Tenant-Id header -> %d", none.statusCode());
assertThat(none.statusCode()).isEqualTo(400);
}
}
}