diff --git a/server/index.ts b/server/index.ts index 3d9306d2b..62ee28b0b 100644 --- a/server/index.ts +++ b/server/index.ts @@ -28,6 +28,7 @@ import { initLogCleanupInterval } from "@server/lib/cleanupLogs"; import { initAcmeCertSync } from "#dynamic/lib/acmeCertSync"; import { fetchServerIp } from "@server/lib/serverIpService"; import { startRebuildQueueProcessor } from "@server/lib/rebuildClientAssociations"; +import { initAiModelCatalog } from "@server/lib/aiModelCatalog"; async function startServers() { await setHostMeta(); @@ -46,6 +47,7 @@ async function startServers() { initLogCleanupInterval(); initAcmeCertSync(); startRebuildQueueProcessor(); + await initAiModelCatalog(); // Start all servers const apiServer = createApiServer(); diff --git a/server/lib/aiModelCatalog.ts b/server/lib/aiModelCatalog.ts new file mode 100644 index 000000000..39b3a70c9 --- /dev/null +++ b/server/lib/aiModelCatalog.ts @@ -0,0 +1,111 @@ +import fs from "node:fs"; +import axios from "axios"; +import config from "@server/lib/config"; +import logger from "@server/logger"; + +export type CatalogProvider = + | "openai" + | "anthropic" + | "gemini" + | "vertex" + | "azure" + | "bedrock"; + +export type AiModelCatalogEntry = { + provider: CatalogProvider; + model: string; + pricing: { + input: number | null; + output: number | null; + cacheRead: number | null; + reasoningOutput: number | null; + }; +}; + +let catalog: AiModelCatalogEntry[] = []; +let refreshTimer: NodeJS.Timeout | null = null; + +async function fetchFromFile(filePath: string): Promise { + try { + if (!fs.existsSync(filePath)) { + logger.warn( + `AI model catalog file not found at ${filePath}; cost calculation will fall back to unknown pricing` + ); + return null; + } + const raw = fs.readFileSync(filePath, "utf-8"); + const parsed = JSON.parse(raw) as { data: AiModelCatalogEntry[] }; + return parsed.data ?? []; + } catch (error) { + logger.warn("Failed to read AI model catalog file", { error }); + return null; + } +} + +async function fetchFromUpstream( + upstreamUrl: string +): Promise { + try { + const res = await axios.get<{ data: AiModelCatalogEntry[] }>( + upstreamUrl, + { timeout: 15_000 } + ); + return res.data?.data ?? []; + } catch (error: any) { + logger.warn( + `Failed to fetch AI model catalog from ${upstreamUrl}: ${error.message || error}` + ); + return null; + } +} + +async function refreshCatalog(): Promise { + const { file, upstream_url } = config.getRawConfig().ai.model_catalog; + + const fetched = file + ? await fetchFromFile(file) + : await fetchFromUpstream(upstream_url); + + if (fetched) { + catalog = fetched; + logger.debug( + `AI model catalog refreshed: ${catalog.length} models loaded` + ); + } else { + logger.debug( + "AI model catalog refresh failed; keeping previously loaded catalog in memory" + ); + } +} + +function scheduleNextRefresh(): void { + const { refresh_interval_min_hours, refresh_interval_max_hours } = + config.getRawConfig().ai.model_catalog; + + // Jittered rather than fixed so that many self-hosted instances don't + // all hit the upstream catalog endpoint at the same moment. + const minMs = refresh_interval_min_hours * 60 * 60 * 1000; + const maxMs = refresh_interval_max_hours * 60 * 60 * 1000; + const delayMs = minMs + Math.random() * Math.max(0, maxMs - minMs); + + if (refreshTimer) { + clearTimeout(refreshTimer); + } + refreshTimer = setTimeout(async () => { + await refreshCatalog(); + scheduleNextRefresh(); + }, delayMs); +} + +/** + * Loads the AI model pricing catalog into memory and schedules periodic + * background refreshes. Call once at server startup. + */ +export async function initAiModelCatalog(): Promise { + await refreshCatalog(); + scheduleNextRefresh(); +} + +export function getAiModelCatalog(): AiModelCatalogEntry[] { + return catalog; +} diff --git a/server/lib/aiModelPricing.ts b/server/lib/aiModelPricing.ts index 8c0eb6739..873e4213a 100644 --- a/server/lib/aiModelPricing.ts +++ b/server/lib/aiModelPricing.ts @@ -1,30 +1,10 @@ -import fs from "node:fs"; -import path from "node:path"; -import { APP_PATH } from "@server/lib/consts"; import type { AiProviderType } from "@server/lib/aiProviderDefaults"; import type { AiUsage } from "@server/lib/aiUsageExtraction"; -import logger from "@server/logger"; - -const MODELS_JSON_PATH = path.join(APP_PATH, "models.json"); - -export type CatalogProvider = - | "openai" - | "anthropic" - | "gemini" - | "vertex" - | "azure" - | "bedrock"; - -export type AiModelCatalogEntry = { - provider: CatalogProvider; - model: string; - pricing: { - input: number | null; - output: number | null; - cacheRead: number | null; - reasoningOutput: number | null; - }; -}; +import { + getAiModelCatalog, + type AiModelCatalogEntry, + type CatalogProvider +} from "@server/lib/aiModelCatalog"; export type AiModelPricing = { inputCostPerToken: number | null; @@ -56,34 +36,28 @@ const PROVIDER_CATALOG_MAP: Record< vercelAiGateway: null }; -let modelsByName: Map | null = null; +// Indexed view over the in-memory catalog, rebuilt only when +// getAiModelCatalog() returns a different array instance (i.e. after a +// background refresh swaps it out), not on every lookup. +let indexedCatalog: AiModelCatalogEntry[] | null = null; +let indexedByName: Map = new Map(); -function loadModels(): Map { - if (modelsByName) { - return modelsByName; +function getIndexedCatalog(): Map { + const catalog = getAiModelCatalog(); + if (catalog === indexedCatalog) { + return indexedByName; } const byName = new Map(); - try { - if (fs.existsSync(MODELS_JSON_PATH)) { - const raw = fs.readFileSync(MODELS_JSON_PATH, "utf-8"); - const parsed = JSON.parse(raw) as { data: AiModelCatalogEntry[] }; - for (const entry of parsed.data ?? []) { - if (!entry.model) continue; - const list = byName.get(entry.model) ?? []; - list.push(entry); - byName.set(entry.model, list); - } - } else { - logger.debug( - `AI model pricing file not found at ${MODELS_JSON_PATH}; cost calculation will fall back to unknown pricing` - ); - } - } catch (error) { - logger.warn("Failed to load AI model pricing file", { error }); + for (const entry of catalog) { + if (!entry.model) continue; + const list = byName.get(entry.model) ?? []; + list.push(entry); + byName.set(entry.model, list); } - modelsByName = byName; + indexedCatalog = catalog; + indexedByName = byName; return byName; } @@ -144,7 +118,7 @@ export function getModelPricing( return null; } - const byName = loadModels(); + const byName = getIndexedCatalog(); const catalogProvider = providerType === "custom" ? null : PROVIDER_CATALOG_MAP[providerType]; diff --git a/server/lib/readConfigFile.ts b/server/lib/readConfigFile.ts index 37418ed2f..d5c52f13a 100644 --- a/server/lib/readConfigFile.ts +++ b/server/lib/readConfigFile.ts @@ -401,6 +401,36 @@ export const configSchema = z disable_enterprise_features: z.boolean().optional() }) .optional(), + ai: z + .object({ + model_catalog: z + .object({ + upstream_url: z + .url() + .optional() + .default("https://api.fossorial.io/api/v1/models"), + // No default - only used when an operator wants to + // pin the catalog to a local file instead of + // fetching it from upstream_url. + file: z.string().optional(), + refresh_interval_min_hours: z + .number() + .positive() + .gt(0) + .optional() + .default(6), + refresh_interval_max_hours: z + .number() + .positive() + .gt(0) + .optional() + .default(12) + }) + .optional() + .prefault({}) + }) + .optional() + .prefault({}), dns: z .object({ nameservers: z