From 1c707b076a2b91e2095024215aabf4a5788f31cb Mon Sep 17 00:00:00 2001 From: khoaminh2957 Date: Sun, 24 May 2026 14:49:55 +0700 Subject: [PATCH] feat(ai/router): add SmartRouter unifying LLM provider endpoints (#286) Adds a litellm-style smart router under JS/edgechains/arakoodev/src/ai/src/lib/router that exposes a single, provider-agnostic chat-completion API and dispatches to the appropriate provider based on the model prefix (gpt-* -> OpenAI, claude-* -> Anthropic, gemini-* -> Google, command* -> Cohere). Explicit "provider/model" overrides and runtime adapter registration are also supported. Includes 20 vitest cases covering provider resolution, per-adapter request/response shaping (with axios mocked), router dispatch, default provider fallback, error wrapping, and request validation. Closes #286 --- JS/edgechains/arakoodev/src/ai/src/index.ts | 21 + .../arakoodev/src/ai/src/lib/router/README.md | 115 ++++++ .../src/ai/src/lib/router/SmartRouter.ts | 183 ++++++++ .../lib/router/adapters/anthropicAdapter.ts | 99 +++++ .../src/lib/router/adapters/cohereAdapter.ts | 111 +++++ .../src/lib/router/adapters/googleAdapter.ts | 102 +++++ .../src/lib/router/adapters/openaiAdapter.ts | 64 +++ .../arakoodev/src/ai/src/lib/router/index.ts | 18 + .../src/ai/src/lib/router/providerResolver.ts | 77 ++++ .../arakoodev/src/ai/src/lib/router/types.ts | 80 ++++ .../src/ai/src/tests/smartRouter.test.ts | 389 ++++++++++++++++++ 11 files changed, 1259 insertions(+) create mode 100644 JS/edgechains/arakoodev/src/ai/src/lib/router/README.md create mode 100644 JS/edgechains/arakoodev/src/ai/src/lib/router/SmartRouter.ts create mode 100644 JS/edgechains/arakoodev/src/ai/src/lib/router/adapters/anthropicAdapter.ts create mode 100644 JS/edgechains/arakoodev/src/ai/src/lib/router/adapters/cohereAdapter.ts create mode 100644 JS/edgechains/arakoodev/src/ai/src/lib/router/adapters/googleAdapter.ts create mode 100644 JS/edgechains/arakoodev/src/ai/src/lib/router/adapters/openaiAdapter.ts create mode 100644 JS/edgechains/arakoodev/src/ai/src/lib/router/index.ts create mode 100644 JS/edgechains/arakoodev/src/ai/src/lib/router/providerResolver.ts create mode 100644 JS/edgechains/arakoodev/src/ai/src/lib/router/types.ts create mode 100644 JS/edgechains/arakoodev/src/ai/src/tests/smartRouter.test.ts diff --git a/JS/edgechains/arakoodev/src/ai/src/index.ts b/JS/edgechains/arakoodev/src/ai/src/index.ts index 2c98f37dc..9a049c917 100644 --- a/JS/edgechains/arakoodev/src/ai/src/index.ts +++ b/JS/edgechains/arakoodev/src/ai/src/index.ts @@ -3,3 +3,24 @@ export { GeminiAI } from "./lib/gemini/gemini.js"; export { LlamaAI } from "./lib/llama/llama.js"; export { RetellAI } from "./lib/retell-ai/retell.js"; export { RetellWebClient } from "./lib/retell-ai/retellWebClient.js"; +export { + SmartRouter, + RouterError, + OpenAIAdapter, + AnthropicAdapter, + GoogleAdapter, + CohereAdapter, + resolveProvider, +} from "./lib/router/index.js"; +export type { + SmartRouterOptions, + RouterRequest, + RouterResponse, + RouterMessage, + RouterRole, + RouterUsage, + ProviderAdapter, + ProviderKeys, + ProviderName, + ResolvedModel, +} from "./lib/router/index.js"; diff --git a/JS/edgechains/arakoodev/src/ai/src/lib/router/README.md b/JS/edgechains/arakoodev/src/ai/src/lib/router/README.md new file mode 100644 index 000000000..5ff1bdaac --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/lib/router/README.md @@ -0,0 +1,115 @@ +# SmartRouter — Unified LLM Provider Router + +`SmartRouter` is EdgeChains' answer to Python's +[`litellm`](https://github.com/BerriAI/litellm): one TypeScript API that +dispatches chat-completion requests to the right provider based on the +model name, with a normalized request and response shape. + +```ts +import { SmartRouter } from "@arakoodev/edgechains.js/ai"; + +const router = new SmartRouter({ + openai: process.env.OPENAI_API_KEY, + anthropic: process.env.ANTHROPIC_API_KEY, + google: process.env.GOOGLE_API_KEY, + cohere: process.env.COHERE_API_KEY, +}); + +const res = await router.complete({ + model: "gpt-4o", // or "claude-3-5-sonnet-20241022", "gemini-1.5-pro", "command-r-plus" + messages: [ + { role: "system", content: "You are concise." }, + { role: "user", content: "Say hello." }, + ], + temperature: 0.2, + max_tokens: 256, +}); + +console.log(res.content, res.provider, res.usage); +``` + +## How routing works + +The provider is inferred from the **model prefix**: + +| Prefix | Provider | +| -------------------------- | ----------- | +| `gpt-*`, `o1-*`, `o3-*` | `openai` | +| `claude-*` | `anthropic` | +| `gemini-*`, `models/gemini-*` | `google` | +| `command*` | `cohere` | +| `llama-*`, `llama3-*` | `llama` | + +You can force a provider with the `"/"` syntax, +e.g. `"openai/some-finetune"` or `"anthropic/claude-experimental-id"`. + +If the prefix is unknown, the router falls back to `defaultProvider` +(when set) or throws `RouterError`. + +## Normalized shapes + +```ts +interface RouterRequest { + model: string; + messages: Array<{ role: "system" | "user" | "assistant"; content: string }>; + temperature?: number; + max_tokens?: number; +} + +interface RouterResponse { + content: string; + provider: "openai" | "anthropic" | "google" | "cohere" | "llama" | "unknown"; + model: string; + usage: { input_tokens: number; output_tokens: number }; + raw?: unknown; // full provider payload, kept for advanced use +} +``` + +System messages are pushed into each provider's correct slot +automatically (Anthropic's `system` field, Gemini's `systemInstruction`, +Cohere's `preamble`). + +## Custom or local providers + +Register an adapter at runtime — useful for Ollama, vLLM, a corporate +gateway, or any other provider: + +```ts +import { SmartRouter, ProviderAdapter } from "@arakoodev/edgechains.js/ai"; + +const ollama: ProviderAdapter = { + name: "llama", + async complete(req) { + // call your local endpoint, return a RouterResponse + }, +}; + +const router = new SmartRouter(); +router.register("llama", ollama); +``` + +## Error handling + +Every failure surfaces as `RouterError` with the originating provider, +the upstream HTTP status (when applicable), and the original error on +`.cause`: + +```ts +import { RouterError } from "@arakoodev/edgechains.js/ai"; + +try { + await router.complete({ model: "gpt-4o", messages }); +} catch (e) { + if (e instanceof RouterError) { + console.error(e.provider, e.status, e.message, e.cause); + } +} +``` + +## Scope of this MVP + +- Built-in adapters for **OpenAI**, **Anthropic**, **Google Gemini**, + and **Cohere**. +- Llama / other providers can be plugged in via `register()`. +- Streaming, tool calling, and embeddings are intentionally out of + scope for this first PR and will land in follow-ups. diff --git a/JS/edgechains/arakoodev/src/ai/src/lib/router/SmartRouter.ts b/JS/edgechains/arakoodev/src/ai/src/lib/router/SmartRouter.ts new file mode 100644 index 000000000..8ab0113de --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/lib/router/SmartRouter.ts @@ -0,0 +1,183 @@ +import { + ProviderAdapter, + ProviderKeys, + ProviderName, + RouterRequest, + RouterResponse, +} from "./types.js"; +import { resolveProvider } from "./providerResolver.js"; +import { OpenAIAdapter } from "./adapters/openaiAdapter.js"; +import { AnthropicAdapter } from "./adapters/anthropicAdapter.js"; +import { GoogleAdapter } from "./adapters/googleAdapter.js"; +import { CohereAdapter } from "./adapters/cohereAdapter.js"; + +export interface SmartRouterOptions extends ProviderKeys { + /** + * Optional provider used when the model prefix cannot be resolved. + * If unset, requests with unknown models throw ``RouterError``. + */ + defaultProvider?: ProviderName; + /** + * OpenAI organization id (forwarded to the OpenAI adapter only). + */ + openaiOrgId?: string; + /** + * Inject pre-built adapters (useful for tests / custom providers). + * Keys here take precedence over the auto-built adapters. + */ + adapters?: Partial>; +} + +/** + * Error thrown for router-level failures (unknown provider, missing key, + * upstream HTTP error). The original cause is preserved on ``.cause`` + * so callers can introspect provider-specific error responses. + */ +export class RouterError extends Error { + public readonly provider: ProviderName; + public readonly status?: number; + public readonly cause?: unknown; + + constructor( + message: string, + opts: { provider: ProviderName; status?: number; cause?: unknown } = { + provider: "unknown", + } + ) { + super(message); + this.name = "RouterError"; + this.provider = opts.provider; + this.status = opts.status; + this.cause = opts.cause; + } +} + +/** + * Unified, provider-agnostic router for LLM chat completions. + * + * ``SmartRouter`` accepts a normalized {@link RouterRequest}, dispatches + * it to the appropriate provider (inferred from ``request.model`` or + * forced via the ``"provider/model"`` syntax), and returns a normalized + * {@link RouterResponse}. The goal is to mirror the ergonomics of + * Python's ``litellm`` package in TypeScript. + * + * ```ts + * const router = new SmartRouter({ + * openai: process.env.OPENAI_API_KEY, + * anthropic: process.env.ANTHROPIC_API_KEY, + * }); + * + * const res = await router.complete({ + * model: "gpt-4o", + * messages: [{ role: "user", content: "Say hi" }], + * }); + * + * console.log(res.content, res.provider, res.usage); + * ``` + */ +export class SmartRouter { + private readonly adapters = new Map(); + private readonly defaultProvider?: ProviderName; + + constructor(options: SmartRouterOptions = {}) { + this.defaultProvider = options.defaultProvider; + + // Auto-build adapters for every provider whose key is present. + const openaiKey = options.openai ?? process.env.OPENAI_API_KEY; + if (openaiKey) { + this.adapters.set( + "openai", + new OpenAIAdapter({ apiKey: openaiKey, orgId: options.openaiOrgId }) + ); + } + const anthropicKey = options.anthropic ?? process.env.ANTHROPIC_API_KEY; + if (anthropicKey) { + this.adapters.set("anthropic", new AnthropicAdapter({ apiKey: anthropicKey })); + } + const googleKey = + options.google ?? process.env.GOOGLE_API_KEY ?? process.env.GEMINI_API_KEY; + if (googleKey) { + this.adapters.set("google", new GoogleAdapter({ apiKey: googleKey })); + } + const cohereKey = options.cohere ?? process.env.COHERE_API_KEY; + if (cohereKey) { + this.adapters.set("cohere", new CohereAdapter({ apiKey: cohereKey })); + } + + // Caller-provided adapters win, e.g. for tests or for ``llama`` + // (which has no built-in adapter in this MVP yet). + if (options.adapters) { + for (const [name, adapter] of Object.entries(options.adapters)) { + if (adapter) this.adapters.set(name as ProviderName, adapter); + } + } + } + + /** + * Register (or replace) an adapter for a given provider at runtime. + * Useful for plugging in custom providers (e.g. a local Ollama + * server) without forking the router. + */ + register(provider: ProviderName, adapter: ProviderAdapter): void { + this.adapters.set(provider, adapter); + } + + /** Return ``true`` if an adapter is registered for ``provider``. */ + supports(provider: ProviderName): boolean { + return this.adapters.has(provider); + } + + /** List every provider currently routable from this instance. */ + listProviders(): ProviderName[] { + return Array.from(this.adapters.keys()); + } + + /** + * Run a chat completion request. Throws {@link RouterError} when + * the provider cannot be resolved, no adapter is registered for the + * resolved provider, or the upstream call fails. + */ + async complete(req: RouterRequest): Promise { + if (!req || !req.model) { + throw new RouterError("RouterRequest.model is required", { provider: "unknown" }); + } + if (!Array.isArray(req.messages) || req.messages.length === 0) { + throw new RouterError("RouterRequest.messages must be a non-empty array", { + provider: "unknown", + }); + } + + let { provider, model } = resolveProvider(req.model); + if (provider === "unknown") { + if (this.defaultProvider && this.adapters.has(this.defaultProvider)) { + provider = this.defaultProvider; + } else { + throw new RouterError( + `Unable to route model "${req.model}". Use "provider/model" or register an adapter.`, + { provider: "unknown" } + ); + } + } + + const adapter = this.adapters.get(provider); + if (!adapter) { + throw new RouterError( + `No adapter registered for provider "${provider}". Did you pass the API key?`, + { provider } + ); + } + + try { + return await adapter.complete({ ...req, model }); + } catch (err: any) { + // Re-throw RouterErrors verbatim. + if (err instanceof RouterError) throw err; + const status = err?.response?.status; + const message = + err?.response?.data?.error?.message || + err?.message || + `Provider "${provider}" request failed`; + throw new RouterError(message, { provider, status, cause: err }); + } + } +} diff --git a/JS/edgechains/arakoodev/src/ai/src/lib/router/adapters/anthropicAdapter.ts b/JS/edgechains/arakoodev/src/ai/src/lib/router/adapters/anthropicAdapter.ts new file mode 100644 index 000000000..e4b520533 --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/lib/router/adapters/anthropicAdapter.ts @@ -0,0 +1,99 @@ +import axios from "axios"; +import { + ProviderAdapter, + ProviderName, + RouterMessage, + RouterRequest, + RouterResponse, +} from "../types.js"; + +const ANTHROPIC_MESSAGES_URL = "https://api.anthropic.com/v1/messages"; +const ANTHROPIC_VERSION = "2023-06-01"; +/** + * Anthropic requires ``max_tokens`` on every request, so we apply a + * conservative default when the caller does not specify one. + */ +const DEFAULT_MAX_TOKENS = 1024; + +/** + * Adapter for Anthropic's Messages API. + * + * The Anthropic API differs from OpenAI in two notable ways that this + * adapter normalizes away: + * + * 1. ``system`` is a top-level field, not a message role. Any messages + * with ``role: "system"`` are collapsed into a single ``system`` string. + * 2. ``max_tokens`` is required; we fall back to {@link DEFAULT_MAX_TOKENS}. + */ +export class AnthropicAdapter implements ProviderAdapter { + public readonly name: ProviderName = "anthropic"; + private readonly apiKey: string; + + constructor(opts: { apiKey: string }) { + this.apiKey = opts.apiKey; + } + + async complete(req: RouterRequest): Promise { + const { system, messages } = splitSystemMessages(req.messages); + + const body: Record = { + model: req.model, + messages: messages.map((m) => ({ + role: m.role, + content: m.content, + })), + max_tokens: req.max_tokens ?? DEFAULT_MAX_TOKENS, + }; + if (system) body.system = system; + if (req.temperature !== undefined) body.temperature = req.temperature; + + const headers: Record = { + "x-api-key": this.apiKey, + "anthropic-version": ANTHROPIC_VERSION, + "content-type": "application/json", + }; + + const response = await axios.post(ANTHROPIC_MESSAGES_URL, body, { headers }); + const data = response.data ?? {}; + + // Anthropic returns ``content: [{ type: "text", text: "..." }, ...]``. + // Concatenate every text block; non-text blocks (tool_use, etc.) are + // surfaced via ``raw`` for advanced consumers. + const content: string = Array.isArray(data.content) + ? data.content + .filter((blk: any) => blk && blk.type === "text" && typeof blk.text === "string") + .map((blk: any) => blk.text) + .join("") + : ""; + + const usage = data.usage || {}; + return { + content, + provider: this.name, + model: data.model || req.model, + usage: { + input_tokens: usage.input_tokens ?? 0, + output_tokens: usage.output_tokens ?? 0, + }, + raw: data, + }; + } +} + +/** + * Pull ``system`` messages out of the conversation and join them with a + * blank line. Anthropic expects the remaining messages to alternate + * between ``user`` and ``assistant``; we leave ordering to the caller. + */ +function splitSystemMessages(messages: RouterMessage[]): { + system: string; + messages: RouterMessage[]; +} { + const systemParts: string[] = []; + const rest: RouterMessage[] = []; + for (const m of messages) { + if (m.role === "system") systemParts.push(m.content); + else rest.push(m); + } + return { system: systemParts.join("\n\n"), messages: rest }; +} diff --git a/JS/edgechains/arakoodev/src/ai/src/lib/router/adapters/cohereAdapter.ts b/JS/edgechains/arakoodev/src/ai/src/lib/router/adapters/cohereAdapter.ts new file mode 100644 index 000000000..4440f0ff8 --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/lib/router/adapters/cohereAdapter.ts @@ -0,0 +1,111 @@ +import axios from "axios"; +import { + ProviderAdapter, + ProviderName, + RouterMessage, + RouterRequest, + RouterResponse, +} from "../types.js"; + +const COHERE_CHAT_URL = "https://api.cohere.ai/v1/chat"; + +/** + * Adapter for Cohere's Chat API (v1). + * + * The Cohere chat endpoint expects a single ``message`` (the most recent + * user turn), a ``preamble`` (system prompt), and a ``chat_history`` of + * prior turns. We project the router's flat message list onto that + * shape, taking the trailing user turn as ``message``. + */ +export class CohereAdapter implements ProviderAdapter { + public readonly name: ProviderName = "cohere"; + private readonly apiKey: string; + + constructor(opts: { apiKey: string }) { + this.apiKey = opts.apiKey; + } + + async complete(req: RouterRequest): Promise { + const { preamble, history, message } = toCohereShape(req.messages); + + const body: Record = { + model: req.model, + message, + }; + if (preamble) body.preamble = preamble; + if (history.length > 0) body.chat_history = history; + if (req.temperature !== undefined) body.temperature = req.temperature; + if (req.max_tokens !== undefined) body.max_tokens = req.max_tokens; + + const response = await axios.post(COHERE_CHAT_URL, body, { + headers: { + Authorization: `Bearer ${this.apiKey}`, + "content-type": "application/json", + }, + }); + const data = response.data ?? {}; + + const content: string = typeof data.text === "string" ? data.text : ""; + // Cohere reports token usage under either ``meta.billed_units`` or + // ``meta.tokens`` depending on API age; prefer billed_units. + const meta = data.meta || {}; + const billed = meta.billed_units || {}; + const tokens = meta.tokens || {}; + + return { + content, + provider: this.name, + model: req.model, + usage: { + input_tokens: billed.input_tokens ?? tokens.input_tokens ?? 0, + output_tokens: billed.output_tokens ?? tokens.output_tokens ?? 0, + }, + raw: data, + }; + } +} + +/** + * Map router messages into Cohere's chat shape. The final user turn + * (the last user-role message in the list) becomes ``message``; earlier + * turns go into ``chat_history`` with Cohere-style role names + * (``USER``/``CHATBOT``). System messages collapse into ``preamble``. + * + * If the conversation does not end in a user message, the trailing + * non-system message is used as ``message`` regardless of role — the + * Cohere API requires *something* there. + */ +function toCohereShape(messages: RouterMessage[]): { + preamble: string; + history: Array<{ role: "USER" | "CHATBOT"; message: string }>; + message: string; +} { + const preambleParts: string[] = []; + const nonSystem: RouterMessage[] = []; + for (const m of messages) { + if (m.role === "system") preambleParts.push(m.content); + else nonSystem.push(m); + } + + let message = ""; + let endIdx = nonSystem.length; + for (let i = nonSystem.length - 1; i >= 0; i--) { + if (nonSystem[i].role === "user") { + message = nonSystem[i].content; + endIdx = i; + break; + } + } + if (!message && nonSystem.length > 0) { + // No user message found — fall back to the trailing turn. + message = nonSystem[nonSystem.length - 1].content; + endIdx = nonSystem.length - 1; + } + + const history = nonSystem.slice(0, endIdx).map((m) => ({ + role: (m.role === "assistant" ? "CHATBOT" : "USER") as "USER" | "CHATBOT", + message: m.content, + })); + + return { preamble: preambleParts.join("\n\n"), history, message }; +} diff --git a/JS/edgechains/arakoodev/src/ai/src/lib/router/adapters/googleAdapter.ts b/JS/edgechains/arakoodev/src/ai/src/lib/router/adapters/googleAdapter.ts new file mode 100644 index 000000000..981266f08 --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/lib/router/adapters/googleAdapter.ts @@ -0,0 +1,102 @@ +import axios from "axios"; +import { + ProviderAdapter, + ProviderName, + RouterMessage, + RouterRequest, + RouterResponse, +} from "../types.js"; + +const GOOGLE_BASE = "https://generativelanguage.googleapis.com/v1beta/models"; + +/** + * Adapter for Google's Generative Language API (Gemini). + * + * Gemini's request schema is markedly different from OpenAI's: + * + * - URL is ``{base}/{model}:generateContent``. + * - ``system`` is sent as a separate ``systemInstruction`` field. + * - ``assistant`` is named ``model`` in Gemini-land. + * - Sampling parameters live under a ``generationConfig`` object. + */ +export class GoogleAdapter implements ProviderAdapter { + public readonly name: ProviderName = "google"; + private readonly apiKey: string; + + constructor(opts: { apiKey: string }) { + this.apiKey = opts.apiKey; + } + + async complete(req: RouterRequest): Promise { + const modelPath = req.model.startsWith("models/") + ? req.model.slice("models/".length) + : req.model; + const url = `${GOOGLE_BASE}/${encodeURIComponent(modelPath)}:generateContent`; + + const { systemInstruction, contents } = toGeminiContents(req.messages); + + const generationConfig: Record = {}; + if (req.temperature !== undefined) generationConfig.temperature = req.temperature; + if (req.max_tokens !== undefined) generationConfig.maxOutputTokens = req.max_tokens; + + const body: Record = { contents }; + if (systemInstruction) body.systemInstruction = systemInstruction; + if (Object.keys(generationConfig).length > 0) body.generationConfig = generationConfig; + + const response = await axios.post(url, body, { + headers: { + "content-type": "application/json", + "x-goog-api-key": this.apiKey, + }, + }); + const data = response.data ?? {}; + + const firstCandidate = data.candidates && data.candidates[0]; + const parts = + (firstCandidate && firstCandidate.content && firstCandidate.content.parts) || []; + const content: string = parts + .filter((p: any) => p && typeof p.text === "string") + .map((p: any) => p.text) + .join(""); + + const meta = data.usageMetadata || {}; + return { + content, + provider: this.name, + model: req.model, + usage: { + input_tokens: meta.promptTokenCount ?? 0, + output_tokens: meta.candidatesTokenCount ?? 0, + }, + raw: data, + }; + } +} + +/** + * Convert router-style messages into Gemini's ``contents`` array and + * extract any system instructions into a separate field. Roles are + * mapped ``assistant`` -> ``model`` (Gemini's naming), ``user`` stays as + * ``user``. System messages are pulled out entirely. + */ +function toGeminiContents(messages: RouterMessage[]): { + systemInstruction?: { parts: Array<{ text: string }> }; + contents: Array<{ role: "user" | "model"; parts: Array<{ text: string }> }>; +} { + const systemParts: Array<{ text: string }> = []; + const contents: Array<{ role: "user" | "model"; parts: Array<{ text: string }> }> = []; + + for (const m of messages) { + if (m.role === "system") { + systemParts.push({ text: m.content }); + continue; + } + const role: "user" | "model" = m.role === "assistant" ? "model" : "user"; + contents.push({ role, parts: [{ text: m.content }] }); + } + + return { + systemInstruction: systemParts.length > 0 ? { parts: systemParts } : undefined, + contents, + }; +} diff --git a/JS/edgechains/arakoodev/src/ai/src/lib/router/adapters/openaiAdapter.ts b/JS/edgechains/arakoodev/src/ai/src/lib/router/adapters/openaiAdapter.ts new file mode 100644 index 000000000..f57a83094 --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/lib/router/adapters/openaiAdapter.ts @@ -0,0 +1,64 @@ +import axios from "axios"; +import { + ProviderAdapter, + ProviderName, + RouterRequest, + RouterResponse, +} from "../types.js"; + +const OPENAI_CHAT_URL = "https://api.openai.com/v1/chat/completions"; + +/** + * Adapter for the OpenAI Chat Completions API. + * + * Translates the router's normalized request into OpenAI's wire format + * and projects the response back into {@link RouterResponse}. Multi-turn + * messages, system prompts, temperature, and max_tokens are all + * forwarded verbatim. + */ +export class OpenAIAdapter implements ProviderAdapter { + public readonly name: ProviderName = "openai"; + private readonly apiKey: string; + private readonly orgId: string; + + constructor(opts: { apiKey: string; orgId?: string }) { + this.apiKey = opts.apiKey; + this.orgId = opts.orgId || process.env.OPENAI_ORG_ID || ""; + } + + async complete(req: RouterRequest): Promise { + const body: Record = { + model: req.model, + messages: req.messages.map((m) => ({ + role: m.role, + content: m.content, + ...(m.name ? { name: m.name } : {}), + })), + }; + if (req.temperature !== undefined) body.temperature = req.temperature; + if (req.max_tokens !== undefined) body.max_tokens = req.max_tokens; + + const headers: Record = { + Authorization: `Bearer ${this.apiKey}`, + "content-type": "application/json", + }; + if (this.orgId) headers["OpenAI-Organization"] = this.orgId; + + const response = await axios.post(OPENAI_CHAT_URL, body, { headers }); + const data = response.data ?? {}; + const choice = (data.choices && data.choices[0]) || {}; + const content: string = (choice.message && choice.message.content) || ""; + const usage = data.usage || {}; + + return { + content, + provider: this.name, + model: data.model || req.model, + usage: { + input_tokens: usage.prompt_tokens ?? 0, + output_tokens: usage.completion_tokens ?? 0, + }, + raw: data, + }; + } +} diff --git a/JS/edgechains/arakoodev/src/ai/src/lib/router/index.ts b/JS/edgechains/arakoodev/src/ai/src/lib/router/index.ts new file mode 100644 index 000000000..09df8fc80 --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/lib/router/index.ts @@ -0,0 +1,18 @@ +export { SmartRouter, RouterError } from "./SmartRouter.js"; +export type { SmartRouterOptions } from "./SmartRouter.js"; +export { resolveProvider } from "./providerResolver.js"; +export type { ResolvedModel } from "./providerResolver.js"; +export type { + RouterMessage, + RouterRequest, + RouterResponse, + RouterRole, + RouterUsage, + ProviderAdapter, + ProviderKeys, + ProviderName, +} from "./types.js"; +export { OpenAIAdapter } from "./adapters/openaiAdapter.js"; +export { AnthropicAdapter } from "./adapters/anthropicAdapter.js"; +export { GoogleAdapter } from "./adapters/googleAdapter.js"; +export { CohereAdapter } from "./adapters/cohereAdapter.js"; diff --git a/JS/edgechains/arakoodev/src/ai/src/lib/router/providerResolver.ts b/JS/edgechains/arakoodev/src/ai/src/lib/router/providerResolver.ts new file mode 100644 index 000000000..f0db16840 --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/lib/router/providerResolver.ts @@ -0,0 +1,77 @@ +import { ProviderName } from "./types.js"; + +/** + * Result of resolving a fully-qualified model string into a provider and + * the canonical model id that should be forwarded to that provider. + */ +export interface ResolvedModel { + provider: ProviderName; + model: string; +} + +/** + * Ordered list of model-prefix -> provider rules. Order matters because + * the first matching rule wins; more specific prefixes therefore appear + * before more general ones. + */ +const PROVIDER_PREFIXES: Array<{ prefix: string; provider: ProviderName }> = [ + { prefix: "gpt-", provider: "openai" }, + { prefix: "o1-", provider: "openai" }, + { prefix: "o3-", provider: "openai" }, + { prefix: "text-embedding-", provider: "openai" }, + { prefix: "claude-", provider: "anthropic" }, + { prefix: "gemini-", provider: "google" }, + { prefix: "models/gemini-", provider: "google" }, + { prefix: "command-", provider: "cohere" }, + { prefix: "command", provider: "cohere" }, + { prefix: "llama-", provider: "llama" }, + { prefix: "llama3-", provider: "llama" }, + { prefix: "llama2-", provider: "llama" }, +]; + +const EXPLICIT_PROVIDERS = new Set([ + "openai", + "anthropic", + "google", + "cohere", + "llama", +]); + +/** + * Resolve a model string to a {@link ResolvedModel}. + * + * Two forms are accepted: + * 1. ``/`` — explicit provider, the segment after the + * slash is forwarded verbatim. + * 2. Bare model id — provider is inferred from {@link PROVIDER_PREFIXES}. + * + * Unknown models resolve to ``{ provider: "unknown", model }`` so callers + * can decide whether to throw or fall through to a default provider. + */ +export function resolveProvider(model: string): ResolvedModel { + if (!model || typeof model !== "string") { + return { provider: "unknown", model: String(model ?? "") }; + } + + const trimmed = model.trim(); + + // 1) Explicit "/" form. + const slashIdx = trimmed.indexOf("/"); + if (slashIdx > 0) { + const head = trimmed.slice(0, slashIdx).toLowerCase() as ProviderName; + const tail = trimmed.slice(slashIdx + 1); + if (EXPLICIT_PROVIDERS.has(head) && tail.length > 0) { + return { provider: head, model: tail }; + } + } + + // 2) Prefix-based inference. + const lower = trimmed.toLowerCase(); + for (const { prefix, provider } of PROVIDER_PREFIXES) { + if (lower.startsWith(prefix)) { + return { provider, model: trimmed }; + } + } + + return { provider: "unknown", model: trimmed }; +} diff --git a/JS/edgechains/arakoodev/src/ai/src/lib/router/types.ts b/JS/edgechains/arakoodev/src/ai/src/lib/router/types.ts new file mode 100644 index 000000000..2f1c880f7 --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/lib/router/types.ts @@ -0,0 +1,80 @@ +/** + * Shared type definitions for the SmartRouter. + * + * The router exposes a single, provider-agnostic API surface (inspired by + * Python's litellm) that normalizes requests / responses across the + * underlying providers (OpenAI, Anthropic, Google Gemini, Cohere, ...). + */ + +export type RouterRole = "system" | "user" | "assistant"; + +export interface RouterMessage { + role: RouterRole; + content: string; + /** Optional name field (OpenAI-compatible). */ + name?: string; +} + +/** + * Provider identifier. New providers can be added without breaking the + * router by extending this union and registering a corresponding adapter. + */ +export type ProviderName = + | "openai" + | "anthropic" + | "google" + | "cohere" + | "llama" + | "unknown"; + +export interface RouterRequest { + /** + * Fully qualified model name. The router infers the provider from the + * model prefix (e.g. ``gpt-*`` -> openai, ``claude-*`` -> anthropic, + * ``gemini-*`` -> google, ``command-*`` -> cohere, ``llama-*`` -> llama). + * + * A provider can be forced by prefixing the model with ``/``, + * e.g. ``"openai/gpt-4o"`` or ``"anthropic/claude-3-5-sonnet"``. + */ + model: string; + messages: RouterMessage[]; + temperature?: number; + max_tokens?: number; + /** Streaming is reserved for a follow-up PR; ignored by the MVP adapters. */ + stream?: boolean; +} + +export interface RouterUsage { + input_tokens: number; + output_tokens: number; +} + +export interface RouterResponse { + /** Concatenated assistant text content. */ + content: string; + /** Resolved provider that served the request. */ + provider: ProviderName; + /** Resolved model id sent to the provider. */ + model: string; + usage: RouterUsage; + /** Raw provider response, kept for debugging / advanced use. */ + raw?: unknown; +} + +export interface ProviderKeys { + openai?: string; + anthropic?: string; + google?: string; + cohere?: string; + llama?: string; +} + +/** + * Minimal contract every provider adapter must satisfy. Adapters are + * intentionally stateless beyond their API key so the router can fan out + * concurrent requests safely. + */ +export interface ProviderAdapter { + readonly name: ProviderName; + complete(req: RouterRequest): Promise; +} diff --git a/JS/edgechains/arakoodev/src/ai/src/tests/smartRouter.test.ts b/JS/edgechains/arakoodev/src/ai/src/tests/smartRouter.test.ts new file mode 100644 index 000000000..70ea56c5b --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/tests/smartRouter.test.ts @@ -0,0 +1,389 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import axios from "axios"; +import { + SmartRouter, + RouterError, + resolveProvider, + OpenAIAdapter, + AnthropicAdapter, + GoogleAdapter, + CohereAdapter, +} from "../lib/router/index.js"; +import type { ProviderAdapter, RouterRequest, RouterResponse } from "../lib/router/index.js"; + +vi.mock("axios", () => { + const post = vi.fn(); + return { default: { post }, post }; +}); + +const axiosPost = (axios as unknown as { post: ReturnType }).post; + +beforeEach(() => { + axiosPost.mockReset(); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +// ───────────────────────────────────────────────────────────────────────── +// resolveProvider — pure function, no network. +// ───────────────────────────────────────────────────────────────────────── + +describe("resolveProvider", () => { + it("infers OpenAI from gpt-* and o*- prefixes", () => { + expect(resolveProvider("gpt-4o").provider).toBe("openai"); + expect(resolveProvider("gpt-3.5-turbo").provider).toBe("openai"); + expect(resolveProvider("o1-preview").provider).toBe("openai"); + expect(resolveProvider("o3-mini").provider).toBe("openai"); + }); + + it("infers Anthropic from claude-*", () => { + expect(resolveProvider("claude-3-5-sonnet-20241022").provider).toBe("anthropic"); + expect(resolveProvider("claude-sonnet-4-5").provider).toBe("anthropic"); + }); + + it("infers Google from gemini-* and models/gemini-*", () => { + expect(resolveProvider("gemini-1.5-pro").provider).toBe("google"); + expect(resolveProvider("models/gemini-1.5-flash").provider).toBe("google"); + }); + + it("infers Cohere from command*", () => { + expect(resolveProvider("command-r-plus").provider).toBe("cohere"); + expect(resolveProvider("command").provider).toBe("cohere"); + }); + + it("infers Llama from llama-*", () => { + expect(resolveProvider("llama-3-70b").provider).toBe("llama"); + expect(resolveProvider("llama3-8b").provider).toBe("llama"); + }); + + it("honors explicit provider/model syntax", () => { + expect(resolveProvider("anthropic/claude-foo").provider).toBe("anthropic"); + expect(resolveProvider("anthropic/claude-foo").model).toBe("claude-foo"); + expect(resolveProvider("openai/some-custom-model").provider).toBe("openai"); + expect(resolveProvider("openai/some-custom-model").model).toBe("some-custom-model"); + }); + + it("returns 'unknown' for unrecognized models", () => { + expect(resolveProvider("mystery-model-v9").provider).toBe("unknown"); + expect(resolveProvider("").provider).toBe("unknown"); + }); +}); + +// ───────────────────────────────────────────────────────────────────────── +// OpenAIAdapter +// ───────────────────────────────────────────────────────────────────────── + +describe("OpenAIAdapter", () => { + it("sends a correctly-shaped request and normalizes the response", async () => { + axiosPost.mockResolvedValueOnce({ + data: { + model: "gpt-4o-2024-05-13", + choices: [{ message: { role: "assistant", content: "hello world" } }], + usage: { prompt_tokens: 12, completion_tokens: 5 }, + }, + }); + + const adapter = new OpenAIAdapter({ apiKey: "sk-test" }); + const res = await adapter.complete({ + model: "gpt-4o", + messages: [{ role: "user", content: "hi" }], + temperature: 0.2, + max_tokens: 50, + }); + + expect(res.provider).toBe("openai"); + expect(res.content).toBe("hello world"); + expect(res.model).toBe("gpt-4o-2024-05-13"); + expect(res.usage).toEqual({ input_tokens: 12, output_tokens: 5 }); + + expect(axiosPost).toHaveBeenCalledTimes(1); + const [url, body, opts] = axiosPost.mock.calls[0]; + expect(url).toBe("https://api.openai.com/v1/chat/completions"); + expect(body).toMatchObject({ + model: "gpt-4o", + temperature: 0.2, + max_tokens: 50, + }); + expect(body.messages).toEqual([{ role: "user", content: "hi" }]); + expect(opts.headers.Authorization).toBe("Bearer sk-test"); + }); +}); + +// ───────────────────────────────────────────────────────────────────────── +// AnthropicAdapter — system extraction + default max_tokens. +// ───────────────────────────────────────────────────────────────────────── + +describe("AnthropicAdapter", () => { + it("collapses system messages and concatenates text blocks", async () => { + axiosPost.mockResolvedValueOnce({ + data: { + model: "claude-3-5-sonnet-20241022", + content: [ + { type: "text", text: "part one " }, + { type: "text", text: "part two" }, + { type: "tool_use", id: "ignored" }, + ], + usage: { input_tokens: 9, output_tokens: 7 }, + }, + }); + + const adapter = new AnthropicAdapter({ apiKey: "ak-test" }); + const res = await adapter.complete({ + model: "claude-3-5-sonnet-20241022", + messages: [ + { role: "system", content: "be terse" }, + { role: "user", content: "hi" }, + ], + }); + + expect(res.provider).toBe("anthropic"); + expect(res.content).toBe("part one part two"); + expect(res.usage).toEqual({ input_tokens: 9, output_tokens: 7 }); + + const [url, body, opts] = axiosPost.mock.calls[0]; + expect(url).toBe("https://api.anthropic.com/v1/messages"); + expect(body).toMatchObject({ + model: "claude-3-5-sonnet-20241022", + system: "be terse", + max_tokens: 1024, // default applied + }); + expect(body.messages).toEqual([{ role: "user", content: "hi" }]); + expect(opts.headers["x-api-key"]).toBe("ak-test"); + expect(opts.headers["anthropic-version"]).toBe("2023-06-01"); + }); +}); + +// ───────────────────────────────────────────────────────────────────────── +// GoogleAdapter — role rewrite + generationConfig. +// ───────────────────────────────────────────────────────────────────────── + +describe("GoogleAdapter", () => { + it("translates router messages to Gemini contents and reads usageMetadata", async () => { + axiosPost.mockResolvedValueOnce({ + data: { + candidates: [ + { + content: { + role: "model", + parts: [{ text: "Gemini answer" }], + }, + }, + ], + usageMetadata: { promptTokenCount: 3, candidatesTokenCount: 8 }, + }, + }); + + const adapter = new GoogleAdapter({ apiKey: "g-test" }); + const res = await adapter.complete({ + model: "gemini-1.5-pro", + messages: [ + { role: "system", content: "stay brief" }, + { role: "user", content: "hello" }, + { role: "assistant", content: "hi there" }, + { role: "user", content: "follow up" }, + ], + temperature: 0.4, + max_tokens: 128, + }); + + expect(res.provider).toBe("google"); + expect(res.content).toBe("Gemini answer"); + expect(res.usage).toEqual({ input_tokens: 3, output_tokens: 8 }); + + const [url, body, opts] = axiosPost.mock.calls[0]; + expect(url).toBe( + "https://generativelanguage.googleapis.com/v1beta/models/gemini-1.5-pro:generateContent" + ); + expect(body.contents).toEqual([ + { role: "user", parts: [{ text: "hello" }] }, + { role: "model", parts: [{ text: "hi there" }] }, + { role: "user", parts: [{ text: "follow up" }] }, + ]); + expect(body.systemInstruction).toEqual({ parts: [{ text: "stay brief" }] }); + expect(body.generationConfig).toEqual({ temperature: 0.4, maxOutputTokens: 128 }); + expect(opts.headers["x-goog-api-key"]).toBe("g-test"); + }); +}); + +// ───────────────────────────────────────────────────────────────────────── +// CohereAdapter — chat_history + preamble derivation. +// ───────────────────────────────────────────────────────────────────────── + +describe("CohereAdapter", () => { + it("splits history from the trailing user message and uses preamble", async () => { + axiosPost.mockResolvedValueOnce({ + data: { + text: "Cohere reply", + meta: { billed_units: { input_tokens: 4, output_tokens: 2 } }, + }, + }); + + const adapter = new CohereAdapter({ apiKey: "co-test" }); + const res = await adapter.complete({ + model: "command-r-plus", + messages: [ + { role: "system", content: "be helpful" }, + { role: "user", content: "first" }, + { role: "assistant", content: "ack" }, + { role: "user", content: "second" }, + ], + }); + + expect(res.provider).toBe("cohere"); + expect(res.content).toBe("Cohere reply"); + expect(res.usage).toEqual({ input_tokens: 4, output_tokens: 2 }); + + const [url, body, opts] = axiosPost.mock.calls[0]; + expect(url).toBe("https://api.cohere.ai/v1/chat"); + expect(body).toMatchObject({ + model: "command-r-plus", + message: "second", + preamble: "be helpful", + }); + expect(body.chat_history).toEqual([ + { role: "USER", message: "first" }, + { role: "CHATBOT", message: "ack" }, + ]); + expect(opts.headers.Authorization).toBe("Bearer co-test"); + }); +}); + +// ───────────────────────────────────────────────────────────────────────── +// SmartRouter end-to-end behavior with stubbed adapters. +// ───────────────────────────────────────────────────────────────────────── + +function makeStubAdapter(name: any, content = "stub-response"): ProviderAdapter { + return { + name, + complete: vi.fn(async (req: RouterRequest): Promise => ({ + content, + provider: name, + model: req.model, + usage: { input_tokens: 1, output_tokens: 1 }, + raw: { req }, + })), + } as ProviderAdapter; +} + +describe("SmartRouter", () => { + it("routes gpt-* requests to the OpenAI adapter", async () => { + const openai = makeStubAdapter("openai", "from-openai"); + const router = new SmartRouter({ + adapters: { openai, anthropic: makeStubAdapter("anthropic") }, + }); + const res = await router.complete({ + model: "gpt-4o", + messages: [{ role: "user", content: "hello" }], + }); + expect(res.content).toBe("from-openai"); + expect(res.provider).toBe("openai"); + expect((openai.complete as any).mock.calls[0][0].model).toBe("gpt-4o"); + }); + + it("routes claude-* requests to the Anthropic adapter", async () => { + const anthropic = makeStubAdapter("anthropic", "from-claude"); + const router = new SmartRouter({ + adapters: { openai: makeStubAdapter("openai"), anthropic }, + }); + const res = await router.complete({ + model: "claude-3-5-sonnet-20241022", + messages: [{ role: "user", content: "hi" }], + }); + expect(res.content).toBe("from-claude"); + expect(res.provider).toBe("anthropic"); + }); + + it("respects the explicit provider/model override and strips the prefix", async () => { + const openai = makeStubAdapter("openai", "explicit-openai"); + const router = new SmartRouter({ adapters: { openai } }); + const res = await router.complete({ + model: "openai/custom-model-id", + messages: [{ role: "user", content: "x" }], + }); + expect(res.content).toBe("explicit-openai"); + expect((openai.complete as any).mock.calls[0][0].model).toBe("custom-model-id"); + }); + + it("falls back to defaultProvider when the model prefix is unknown", async () => { + const anthropic = makeStubAdapter("anthropic", "default-fallback"); + const router = new SmartRouter({ + adapters: { anthropic }, + defaultProvider: "anthropic", + }); + const res = await router.complete({ + model: "totally-made-up-model", + messages: [{ role: "user", content: "?" }], + }); + expect(res.provider).toBe("anthropic"); + expect(res.content).toBe("default-fallback"); + }); + + it("throws RouterError when no adapter is registered for the resolved provider", async () => { + const router = new SmartRouter({ adapters: { openai: makeStubAdapter("openai") } }); + await expect( + router.complete({ + model: "claude-3-opus", + messages: [{ role: "user", content: "hi" }], + }) + ).rejects.toBeInstanceOf(RouterError); + }); + + it("throws RouterError for unresolvable models when no defaultProvider is set", async () => { + const router = new SmartRouter({ adapters: { openai: makeStubAdapter("openai") } }); + await expect( + router.complete({ + model: "mystery-xyz", + messages: [{ role: "user", content: "hi" }], + }) + ).rejects.toBeInstanceOf(RouterError); + }); + + it("validates request shape", async () => { + const router = new SmartRouter({ adapters: { openai: makeStubAdapter("openai") } }); + await expect( + router.complete({ model: "", messages: [{ role: "user", content: "x" }] } as any) + ).rejects.toBeInstanceOf(RouterError); + await expect( + router.complete({ model: "gpt-4o", messages: [] } as any) + ).rejects.toBeInstanceOf(RouterError); + }); + + it("supports register() for runtime adapter injection", async () => { + const router = new SmartRouter(); + expect(router.listProviders()).toEqual([]); + router.register("openai", makeStubAdapter("openai", "registered")); + expect(router.supports("openai")).toBe(true); + const res = await router.complete({ + model: "gpt-4o", + messages: [{ role: "user", content: "x" }], + }); + expect(res.content).toBe("registered"); + }); + + it("wraps upstream errors in RouterError preserving cause and status", async () => { + const failing: ProviderAdapter = { + name: "openai", + complete: vi.fn(async () => { + const err: any = new Error("Upstream went boom"); + err.response = { status: 503, data: { error: { message: "service down" } } }; + throw err; + }), + }; + const router = new SmartRouter({ adapters: { openai: failing } }); + try { + await router.complete({ + model: "gpt-4o", + messages: [{ role: "user", content: "x" }], + }); + throw new Error("expected RouterError"); + } catch (e: any) { + expect(e).toBeInstanceOf(RouterError); + expect(e.status).toBe(503); + expect(e.message).toBe("service down"); + expect(e.provider).toBe("openai"); + expect(e.cause).toBeDefined(); + } + }); +});