461 lines
15 KiB
TypeScript
Executable File
461 lines
15 KiB
TypeScript
Executable File
// Wave-3 compat route families: alias prefixes must be byte-identical to the
|
|
// canonical routes (and equally governed); GenAI and Cohere surfaces
|
|
// translate both directions.
|
|
|
|
import { assertEquals } from "@std/assert";
|
|
import { createHandler } from "../../apps/gateway/main.ts";
|
|
import {
|
|
type AppContext,
|
|
NullToolExecutor,
|
|
VERSION,
|
|
} from "../../apps/gateway/context.ts";
|
|
import { ProviderManager } from "../../packages/providers/src/mod.ts";
|
|
import { Metrics } from "../../packages/telemetry/src/metrics.ts";
|
|
import { LogBus } from "../../packages/telemetry/src/logbus.ts";
|
|
import { VirtualKeyManager } from "../../packages/governance/src/virtual_keys.ts";
|
|
import { MCPRegistry } from "../../packages/mcp/src/registry.ts";
|
|
import { PluginManager } from "../../packages/plugins/src/lifecycle.ts";
|
|
import {
|
|
jsonResponse,
|
|
MockProvider,
|
|
openAIChatBody,
|
|
readSSE,
|
|
sseResponse,
|
|
} from "../../packages/testing/src/mod.ts";
|
|
|
|
function makeContext(mockUrl: string): AppContext {
|
|
return {
|
|
providers: new ProviderManager([{
|
|
id: "openai",
|
|
type: "openai",
|
|
apiKey: "sk-test",
|
|
baseUrl: mockUrl,
|
|
enabled: true,
|
|
models: ["gpt-4o"],
|
|
priority: 0,
|
|
retry: { maxRetries: 0 },
|
|
}], "openai"),
|
|
metrics: new Metrics(),
|
|
logBus: new LogBus(),
|
|
virtualKeys: new VirtualKeyManager(),
|
|
mcp: new MCPRegistry(),
|
|
plugins: new PluginManager(),
|
|
toolExecutor: new NullToolExecutor(),
|
|
version: VERSION,
|
|
};
|
|
}
|
|
|
|
const chatBody = JSON.stringify({
|
|
model: "openai/gpt-4o",
|
|
messages: [{ role: "user", content: "hei" }],
|
|
});
|
|
|
|
function post(path: string, body: string, token?: string): Request {
|
|
return new Request(`http://gateway.test${path}`, {
|
|
method: "POST",
|
|
headers: {
|
|
"Content-Type": "application/json",
|
|
...(token ? { "Authorization": `Bearer ${token}` } : {}),
|
|
},
|
|
body,
|
|
});
|
|
}
|
|
|
|
Deno.test("alias prefixes are byte-identical to the canonical routes", async () => {
|
|
const mock = new MockProvider(() => jsonResponse(openAIChatBody("aliased")));
|
|
const handler = createHandler(makeContext(mock.url));
|
|
try {
|
|
const canonical = await (await handler(
|
|
post("/v1/chat/completions", chatBody),
|
|
)).text();
|
|
for (const prefix of ["/openai", "/litellm", "/langchain", "/pydanticai"]) {
|
|
const aliased = await (await handler(
|
|
post(`${prefix}/v1/chat/completions`, chatBody),
|
|
)).text();
|
|
assertEquals(aliased, canonical, `${prefix} diverged from canonical`);
|
|
}
|
|
|
|
// Anthropic SDK base URL: /anthropic/v1/messages hits the compat route.
|
|
const messages = await handler(post(
|
|
"/anthropic/v1/messages",
|
|
JSON.stringify({
|
|
model: "openai/gpt-4o",
|
|
max_tokens: 32,
|
|
messages: [{ role: "user", content: "hei" }],
|
|
}),
|
|
));
|
|
assertEquals(messages.status, 200);
|
|
const anthropic = await messages.json() as { type: string; role: string };
|
|
assertEquals(anthropic.type, "message");
|
|
assertEquals(anthropic.role, "assistant");
|
|
} finally {
|
|
await mock.close();
|
|
}
|
|
});
|
|
|
|
Deno.test("alias prefixes are governed exactly like /v1", async () => {
|
|
const mock = new MockProvider(() => jsonResponse(openAIChatBody("x")));
|
|
const ctx = makeContext(mock.url);
|
|
ctx.virtualKeys.upsert({
|
|
id: "vk",
|
|
name: "gate",
|
|
token: "vk-compat-gate-token",
|
|
enabled: true,
|
|
usedRequests: 0,
|
|
usedCostMicroUsd: 0,
|
|
});
|
|
const handler = createHandler(ctx);
|
|
try {
|
|
const denied = await handler(post("/openai/v1/chat/completions", chatBody));
|
|
assertEquals(denied.status, 401);
|
|
await denied.body?.cancel();
|
|
const allowed = await handler(
|
|
post("/openai/v1/chat/completions", chatBody, "vk-compat-gate-token"),
|
|
);
|
|
assertEquals(allowed.status, 200);
|
|
await allowed.body?.cancel();
|
|
} finally {
|
|
await mock.close();
|
|
}
|
|
});
|
|
|
|
Deno.test("genai compat: generateContent translates both directions", async () => {
|
|
const mock = new MockProvider(() =>
|
|
jsonResponse(openAIChatBody("Hei fra GenAI", {
|
|
usage: { prompt_tokens: 5, completion_tokens: 7, total_tokens: 12 },
|
|
}))
|
|
);
|
|
const handler = createHandler(makeContext(mock.url));
|
|
try {
|
|
const res = await handler(post(
|
|
`/genai/v1beta/models/${
|
|
encodeURIComponent("openai/gpt-4o")
|
|
}:generateContent`,
|
|
JSON.stringify({
|
|
systemInstruction: { parts: [{ text: "Vær presis." }] },
|
|
contents: [
|
|
{ role: "user", parts: [{ text: "Hei" }, { text: " der" }] },
|
|
],
|
|
generationConfig: { temperature: 0.3, maxOutputTokens: 99 },
|
|
}),
|
|
));
|
|
assertEquals(res.status, 200);
|
|
const body = await res.json() as {
|
|
candidates: Array<{
|
|
content: { role: string; parts: Array<{ text: string }> };
|
|
finishReason: string;
|
|
}>;
|
|
usageMetadata: { totalTokenCount: number };
|
|
};
|
|
assertEquals(body.candidates[0].content.parts[0].text, "Hei fra GenAI");
|
|
assertEquals(body.candidates[0].finishReason, "STOP");
|
|
assertEquals(body.usageMetadata.totalTokenCount, 12);
|
|
|
|
const sent = mock.calls[0].body as {
|
|
model: string;
|
|
messages: Array<{ role: string; content: string }>;
|
|
max_tokens: number;
|
|
temperature: number;
|
|
};
|
|
assertEquals(sent.model, "gpt-4o");
|
|
assertEquals(sent.messages[0], { role: "system", content: "Vær presis." });
|
|
assertEquals(sent.messages[1], { role: "user", content: "Hei der" });
|
|
assertEquals(sent.max_tokens, 99);
|
|
} finally {
|
|
await mock.close();
|
|
}
|
|
});
|
|
|
|
Deno.test("cohere compat: v2/chat translates both directions", async () => {
|
|
const mock = new MockProvider(() =>
|
|
jsonResponse(openAIChatBody("Hei fra Cohere-compat", {
|
|
usage: { prompt_tokens: 3, completion_tokens: 4, total_tokens: 7 },
|
|
}))
|
|
);
|
|
const handler = createHandler(makeContext(mock.url));
|
|
try {
|
|
const res = await handler(post(
|
|
"/cohere/v2/chat",
|
|
JSON.stringify({
|
|
model: "openai/gpt-4o",
|
|
messages: [{ role: "user", content: "Hei" }],
|
|
max_tokens: 50,
|
|
p: 0.8,
|
|
}),
|
|
));
|
|
assertEquals(res.status, 200);
|
|
const body = await res.json() as {
|
|
message: { content: Array<{ type: string; text: string }> };
|
|
finish_reason: string;
|
|
usage: { billed_units: { output_tokens: number } };
|
|
};
|
|
assertEquals(body.message.content[0].text, "Hei fra Cohere-compat");
|
|
assertEquals(body.finish_reason, "COMPLETE");
|
|
assertEquals(body.usage.billed_units.output_tokens, 4);
|
|
|
|
const sent = mock.calls[0].body as { top_p: number };
|
|
assertEquals(sent.top_p, 0.8);
|
|
} finally {
|
|
await mock.close();
|
|
}
|
|
});
|
|
|
|
// ------------------------------------------------- streaming translation
|
|
|
|
/** Canonical chat.completion.chunk frames with a usage-bearing finish chunk. */
|
|
function canonicalStreamFrames(
|
|
texts: string[],
|
|
usage: {
|
|
prompt_tokens: number;
|
|
completion_tokens: number;
|
|
total_tokens: number;
|
|
},
|
|
): string[] {
|
|
const frames = texts.map((text, i) =>
|
|
`data: ${
|
|
JSON.stringify({
|
|
id: "chatcmpl-mock",
|
|
object: "chat.completion.chunk",
|
|
created: 1700000000,
|
|
model: "mock-model",
|
|
choices: [{
|
|
index: 0,
|
|
delta: i === 0
|
|
? { role: "assistant", content: text }
|
|
: { content: text },
|
|
finish_reason: null,
|
|
}],
|
|
})
|
|
}\n\n`
|
|
);
|
|
frames.push(
|
|
`data: ${
|
|
JSON.stringify({
|
|
id: "chatcmpl-mock",
|
|
object: "chat.completion.chunk",
|
|
created: 1700000000,
|
|
model: "mock-model",
|
|
choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
|
|
usage,
|
|
})
|
|
}\n\n`,
|
|
);
|
|
frames.push("data: [DONE]\n\n");
|
|
return frames;
|
|
}
|
|
|
|
function streamAwareMock(frames: string[], jsonText: string): MockProvider {
|
|
return new MockProvider((call) =>
|
|
(call.body as { stream?: boolean }).stream
|
|
? sseResponse(frames)
|
|
: jsonResponse(openAIChatBody(jsonText))
|
|
);
|
|
}
|
|
|
|
interface GenAIChunk {
|
|
candidates?: Array<{
|
|
content?: { role?: string; parts?: Array<{ text?: string }> };
|
|
finishReason?: string;
|
|
index?: number;
|
|
}>;
|
|
usageMetadata?: {
|
|
promptTokenCount?: number;
|
|
candidatesTokenCount?: number;
|
|
totalTokenCount?: number;
|
|
};
|
|
modelVersion?: string;
|
|
}
|
|
|
|
Deno.test("genai compat: streamGenerateContent emits GenAI SSE chunks", async () => {
|
|
const mock = streamAwareMock(
|
|
canonicalStreamFrames(["Hei", " fra", " GenAI"], {
|
|
prompt_tokens: 5,
|
|
completion_tokens: 7,
|
|
total_tokens: 12,
|
|
}),
|
|
"unused",
|
|
);
|
|
const handler = createHandler(makeContext(mock.url));
|
|
try {
|
|
const res = await handler(post(
|
|
`/genai/v1beta/models/${
|
|
encodeURIComponent("openai/gpt-4o")
|
|
}:streamGenerateContent`,
|
|
JSON.stringify({
|
|
contents: [{ role: "user", parts: [{ text: "Hei" }] }],
|
|
}),
|
|
));
|
|
assertEquals(res.status, 200);
|
|
assertEquals(res.headers.get("Content-Type"), "text/event-stream");
|
|
// The upstream received a streaming request.
|
|
assertEquals((mock.calls[0].body as { stream?: boolean }).stream, true);
|
|
|
|
const raw = await readSSE(res);
|
|
// GenAI's SSE surface has no [DONE] sentinel.
|
|
assertEquals(raw.some((e) => e === "[DONE]"), false);
|
|
const events = raw as unknown as GenAIChunk[];
|
|
const text = events
|
|
.flatMap((e) => e.candidates?.[0]?.content?.parts ?? [])
|
|
.map((p) => p.text ?? "")
|
|
.join("");
|
|
assertEquals(text, "Hei fra GenAI");
|
|
const last = events.at(-1)!;
|
|
assertEquals(last.candidates?.[0].finishReason, "STOP");
|
|
assertEquals(last.candidates?.[0].content?.role, "model");
|
|
assertEquals(last.usageMetadata?.totalTokenCount, 12);
|
|
assertEquals(last.usageMetadata?.candidatesTokenCount, 7);
|
|
} finally {
|
|
await mock.close();
|
|
}
|
|
});
|
|
|
|
interface CohereEvent {
|
|
type: string;
|
|
delta?: {
|
|
message?: {
|
|
role?: string;
|
|
content?: { type?: string; text?: string };
|
|
tool_calls?: { function?: { arguments?: string } };
|
|
};
|
|
finish_reason?: string;
|
|
usage?: {
|
|
billed_units?: { input_tokens?: number; output_tokens?: number };
|
|
};
|
|
};
|
|
}
|
|
|
|
Deno.test("cohere compat: v2/chat stream emits Cohere events", async () => {
|
|
const mock = streamAwareMock(
|
|
canonicalStreamFrames(["Hei", " fra", " Cohere"], {
|
|
prompt_tokens: 3,
|
|
completion_tokens: 4,
|
|
total_tokens: 7,
|
|
}),
|
|
"unused",
|
|
);
|
|
const handler = createHandler(makeContext(mock.url));
|
|
try {
|
|
const res = await handler(post(
|
|
"/cohere/v2/chat",
|
|
JSON.stringify({
|
|
model: "openai/gpt-4o",
|
|
messages: [{ role: "user", content: "Hei" }],
|
|
stream: true,
|
|
}),
|
|
));
|
|
assertEquals(res.status, 200);
|
|
assertEquals(res.headers.get("Content-Type"), "text/event-stream");
|
|
assertEquals((mock.calls[0].body as { stream?: boolean }).stream, true);
|
|
|
|
const raw = await readSSE(res);
|
|
assertEquals(raw.some((e) => e === "[DONE]"), false);
|
|
const events = raw as unknown as CohereEvent[];
|
|
assertEquals(events[0].type, "message-start");
|
|
const text = events
|
|
.filter((e) => e.type === "content-delta")
|
|
.map((e) => e.delta?.message?.content?.text ?? "")
|
|
.join("");
|
|
assertEquals(text, "Hei fra Cohere");
|
|
const end = events.at(-1)!;
|
|
assertEquals(end.type, "message-end");
|
|
assertEquals(end.delta?.finish_reason, "COMPLETE");
|
|
assertEquals(end.delta?.usage?.billed_units?.input_tokens, 3);
|
|
assertEquals(end.delta?.usage?.billed_units?.output_tokens, 4);
|
|
} finally {
|
|
await mock.close();
|
|
}
|
|
});
|
|
|
|
// --------------------------------------------- aggregator ingress broadening
|
|
|
|
Deno.test("aggregator prefixes accept GenAI + Cohere native shapes", async () => {
|
|
const mock = streamAwareMock(
|
|
canonicalStreamFrames(["ag"], {
|
|
prompt_tokens: 1,
|
|
completion_tokens: 1,
|
|
total_tokens: 2,
|
|
}),
|
|
"agg",
|
|
);
|
|
const handler = createHandler(makeContext(mock.url));
|
|
try {
|
|
for (const prefix of ["/litellm", "/langchain", "/pydanticai"]) {
|
|
// GenAI native shape under the aggregator prefix (was 404 pre-broadening).
|
|
const genai = await handler(post(
|
|
`${prefix}/genai/v1beta/models/${
|
|
encodeURIComponent("openai/gpt-4o")
|
|
}:generateContent`,
|
|
JSON.stringify({
|
|
contents: [{ role: "user", parts: [{ text: "x" }] }],
|
|
}),
|
|
));
|
|
assertEquals(genai.status, 200, `${prefix} genai`);
|
|
const g = await genai.json() as {
|
|
candidates: Array<{ content: { parts: Array<{ text: string }> } }>;
|
|
};
|
|
assertEquals(g.candidates[0].content.parts[0].text, "agg");
|
|
|
|
// Cohere native shape under the aggregator prefix.
|
|
const cohere = await handler(post(
|
|
`${prefix}/cohere/v2/chat`,
|
|
JSON.stringify({
|
|
model: "openai/gpt-4o",
|
|
messages: [{ role: "user", content: "x" }],
|
|
}),
|
|
));
|
|
assertEquals(cohere.status, 200, `${prefix} cohere`);
|
|
const c = await cohere.json() as {
|
|
message: { content: Array<{ text: string }> };
|
|
};
|
|
assertEquals(c.message.content[0].text, "agg");
|
|
}
|
|
|
|
// Streaming also works under an aggregator prefix (GenAI SSE).
|
|
const stream = await handler(post(
|
|
`/litellm/genai/v1beta/models/${
|
|
encodeURIComponent("openai/gpt-4o")
|
|
}:streamGenerateContent`,
|
|
JSON.stringify({ contents: [{ role: "user", parts: [{ text: "x" }] }] }),
|
|
));
|
|
assertEquals(stream.status, 200);
|
|
assertEquals(stream.headers.get("Content-Type"), "text/event-stream");
|
|
await stream.body?.cancel();
|
|
|
|
// Every upstream call landed on the OpenAI /chat/completions surface.
|
|
assertEquals(
|
|
mock.calls.every((call) => call.path === "/chat/completions"),
|
|
true,
|
|
);
|
|
} finally {
|
|
await mock.close();
|
|
}
|
|
});
|
|
|
|
Deno.test("aggregator prefixes answer Bedrock shapes with a documented 501", async () => {
|
|
const mock = new MockProvider(() => jsonResponse(openAIChatBody("x")));
|
|
const handler = createHandler(makeContext(mock.url));
|
|
try {
|
|
const paths = [
|
|
"/litellm/bedrock/model/anthropic.claude-3/converse",
|
|
"/litellm/bedrock/model/anthropic.claude-3/converse-stream",
|
|
"/litellm/model/anthropic.claude-3/converse",
|
|
"/langchain/bedrock/model/x/converse",
|
|
"/pydanticai/model/x/converse-stream",
|
|
];
|
|
for (const path of paths) {
|
|
const res = await handler(post(path, JSON.stringify({ messages: [] })));
|
|
assertEquals(res.status, 501, path);
|
|
const body = await res.json() as {
|
|
error: { type: string; message: string };
|
|
};
|
|
assertEquals(body.error.type, "not_implemented");
|
|
// The 501 is self-documenting: it names Bedrock (not a silent 404).
|
|
assertEquals(body.error.message.includes("Bedrock"), true, path);
|
|
}
|
|
// No Bedrock-shaped request was dispatched to a provider.
|
|
assertEquals(mock.calls.length, 0);
|
|
} finally {
|
|
await mock.close();
|
|
}
|
|
});
|