diff --git a/server/db/pg/schema/schema.ts b/server/db/pg/schema/schema.ts index 0f59b1c26..e253cd6d5 100644 --- a/server/db/pg/schema/schema.ts +++ b/server/db/pg/schema/schema.ts @@ -210,7 +210,11 @@ export const resources = pgTable( authDaemonPort: integer("authDaemonPort").default(22123), status: varchar("status") .$type<"pending" | "approved">() - .default("approved") + .default("approved"), + aiProviderId: integer("aiProviderId").references( + () => aiProviders.providerId, + { onDelete: "set null" } + ) }, (t) => [ index("idx_resources_fulldomain") @@ -221,6 +225,19 @@ export const resources = pgTable( ] ); +export const resourceAiModels = pgTable( + "resourceAiModels", + { + resourceId: integer("resourceId") + .notNull() + .references(() => resources.resourceId, { onDelete: "cascade" }), + modelId: integer("modelId") + .notNull() + .references(() => aiModels.modelId, { onDelete: "cascade" }) + }, + (t) => [primaryKey({ columns: [t.resourceId, t.modelId] })] +); + export const labels = pgTable("labels", { labelId: serial("labelId").primaryKey(), name: varchar("name").notNull(), @@ -475,11 +492,30 @@ export const siteResources = pgTable( fullDomain: varchar("fullDomain"), status: varchar("status") .$type<"pending" | "approved">() - .default("approved") + .default("approved"), + aiProviderId: integer("aiProviderId").references( + () => aiProviders.providerId, + { onDelete: "set null" } + ) }, (t) => [index("idx_siteresources_orgid_niceid").on(t.orgId, t.niceId)] ); +export const siteResourceAiModels = pgTable( + "siteResourceAiModels", + { + siteResourceId: integer("siteResourceId") + .notNull() + .references(() => siteResources.siteResourceId, { + onDelete: "cascade" + }), + modelId: integer("modelId") + .notNull() + .references(() => aiModels.modelId, { onDelete: "cascade" }) + }, + (t) => [primaryKey({ columns: [t.siteResourceId, t.modelId] })] +); + export const networks = pgTable( "networks", { @@ -1698,3 +1734,5 @@ export type UserPolicy = InferSelectModel; export type ResourcePolicyRule = InferSelectModel; export type AiProvider = InferSelectModel; export type AiModel = InferSelectModel; +export type ResourceAiModel = InferSelectModel; +export type SiteResourceAiModel = InferSelectModel; diff --git a/server/db/sqlite/schema/schema.ts b/server/db/sqlite/schema/schema.ts index 73e73f04a..3d3ff2ded 100644 --- a/server/db/sqlite/schema/schema.ts +++ b/server/db/sqlite/schema/schema.ts @@ -215,9 +215,26 @@ export const resources = sqliteTable("resources", { .$type<"site" | "remote" | "native">() .default("site"), authDaemonPort: integer("authDaemonPort").default(22123), - status: text("status").$type<"pending" | "approved">().default("approved") + status: text("status").$type<"pending" | "approved">().default("approved"), + aiProviderId: integer("aiProviderId").references( + () => aiProviders.providerId, + { onDelete: "set null" } + ) }); +export const resourceAiModels = sqliteTable( + "resourceAiModels", + { + resourceId: integer("resourceId") + .notNull() + .references(() => resources.resourceId, { onDelete: "cascade" }), + modelId: integer("modelId") + .notNull() + .references(() => aiModels.modelId, { onDelete: "cascade" }) + }, + (t) => [primaryKey({ columns: [t.resourceId, t.modelId] })] +); + export const labels = sqliteTable("labels", { labelId: integer("labelId").primaryKey({ autoIncrement: true }), name: text("name").notNull(), @@ -462,9 +479,28 @@ export const siteResources = sqliteTable("siteResources", { }), subdomain: text("subdomain"), fullDomain: text("fullDomain"), - status: text("status").$type<"pending" | "approved">().default("approved") + status: text("status").$type<"pending" | "approved">().default("approved"), + aiProviderId: integer("aiProviderId").references( + () => aiProviders.providerId, + { onDelete: "set null" } + ) }); +export const siteResourceAiModels = sqliteTable( + "siteResourceAiModels", + { + siteResourceId: integer("siteResourceId") + .notNull() + .references(() => siteResources.siteResourceId, { + onDelete: "cascade" + }), + modelId: integer("modelId") + .notNull() + .references(() => aiModels.modelId, { onDelete: "cascade" }) + }, + (t) => [primaryKey({ columns: [t.siteResourceId, t.modelId] })] +); + export const networks = sqliteTable("networks", { networkId: integer("networkId").primaryKey({ autoIncrement: true }), niceId: text("niceId"), @@ -1680,3 +1716,5 @@ export type RolePolicy = InferSelectModel; export type UserPolicy = InferSelectModel; export type AiProvider = InferSelectModel; export type AiModel = InferSelectModel; +export type ResourceAiModel = InferSelectModel; +export type SiteResourceAiModel = InferSelectModel; diff --git a/server/lib/traefik/TraefikConfigManager.ts b/server/lib/traefik/TraefikConfigManager.ts index cc7299ff7..73d46ee64 100644 --- a/server/lib/traefik/TraefikConfigManager.ts +++ b/server/lib/traefik/TraefikConfigManager.ts @@ -516,6 +516,9 @@ export class TraefikConfigManager { const maintenanceHost = config.getRawConfig().server.internal_hostname; const pangolinUIUrl = `http://${maintenanceHost}:${maintenancePort}`; + const aiGatewayUrl = `http://${maintenanceHost}:${ + config.getRawConfig().server.internal_port + }/api/v1/ai-gateway`; // logger.debug(`Fetching traefik config for exit node: ${currentExitNode}`); traefikConfig = await getTraefikConfig( @@ -528,7 +531,8 @@ export class TraefikConfigManager { ? false : config.getRawConfig().traefik.allow_raw_resources, // dont allow raw resources on saas otherwise use config pangolinUIUrl, // generate maintenance pages on cloud and hybrid - pangolinUIUrl // generate browser gateway targets on cloud and hybrid + pangolinUIUrl, // generate browser gateway targets on cloud and hybrid + aiGatewayUrl ); const domains = new Set(); diff --git a/server/lib/traefik/getTraefikConfig.ts b/server/lib/traefik/getTraefikConfig.ts index 3e84cd570..4f5dd3860 100644 --- a/server/lib/traefik/getTraefikConfig.ts +++ b/server/lib/traefik/getTraefikConfig.ts @@ -1,4 +1,4 @@ -import { db, targetHealthCheck, domains } from "@server/db"; +import { db, targetHealthCheck, domains, aiProviders } from "@server/db"; import { and, eq, @@ -45,7 +45,8 @@ export async function getTraefikConfig( generateLoginPageRouters = false, // UNUSED BUT USED IN PRIVATE allowRawResources = true, maintenancePageUiUrl: string | null = null, // UNUSED BUT USED IN PRIVATE - browserGatewayUiUrl: string | null = null // UNUSED BUT USED IN PRIVATE + browserGatewayUiUrl: string | null = null, // UNUSED BUT USED IN PRIVATE + aiGatewayUrl: string | null = null ): Promise { // Get resources with their targets and sites in a single optimized query // Start from sites on this exit node, then join to targets and resources @@ -209,8 +210,37 @@ export async function getTraefikConfig( }); }); + // Inference-mode resources have no targets/sites (their "backend" is the + // central AI gateway), so they can't be reached via the targets->sites + // join above - query them separately and include them on every exit node. + const inferenceResources = await db + .select({ + resourceId: resources.resourceId, + resourceName: resources.name, + fullDomain: resources.fullDomain, + ssl: resources.ssl, + subdomain: resources.subdomain, + domainId: resources.domainId, + enabled: resources.enabled, + domainCertResolver: domains.certResolver, + preferWildcardCert: domains.preferWildcardCert + }) + .from(resources) + .innerJoin( + aiProviders, + eq(resources.aiProviderId, aiProviders.providerId) + ) + .leftJoin(domains, eq(domains.domainId, resources.domainId)) + .where( + and( + eq(resources.mode, "inference"), + eq(resources.enabled, true), + eq(aiProviders.enabled, true) + ) + ); + // make sure we have at least one resource - if (resourcesMap.size === 0) { + if (resourcesMap.size === 0 && inferenceResources.length === 0) { return {}; } @@ -673,5 +703,90 @@ export async function getTraefikConfig( }; } } + + if (aiGatewayUrl) { + for (const ir of inferenceResources) { + if (!ir.enabled) continue; + if (!ir.domainId || !ir.fullDomain) continue; + + if (!config_output.http.routers) config_output.http.routers = {}; + if (!config_output.http.services) config_output.http.services = {}; + + const fullDomain = ir.fullDomain; + const irKey = `inference-r${ir.resourceId}`; + const routerName = `${irKey}-router`; + const serviceName = `${irKey}-service`; + const rule = `Host(\`${fullDomain}\`)`; + + const domainParts = fullDomain.split("."); + let wildCard; + if (domainParts.length <= 2) { + wildCard = `*.${domainParts.join(".")}`; + } else { + wildCard = `*.${domainParts.slice(1).join(".")}`; + } + if (!ir.subdomain) { + wildCard = fullDomain; + } + + const globalDefaultResolver = + config.getRawConfig().traefik.cert_resolver; + const globalDefaultPreferWildcard = + config.getRawConfig().traefik.prefer_wildcard_cert; + const resolverName = ir.domainCertResolver + ? ir.domainCertResolver.trim() + : globalDefaultResolver; + const preferWildcard = + ir.preferWildcardCert !== undefined && + ir.preferWildcardCert !== null + ? ir.preferWildcardCert + : globalDefaultPreferWildcard; + + const tls = { + certResolver: resolverName, + ...(preferWildcard ? { domains: [{ main: wildCard }] } : {}) + }; + + const additionalMiddlewares = + config.getRawConfig().traefik.additional_middlewares || []; + const routerMiddlewares = [ + badgerMiddlewareName, + ...additionalMiddlewares + ]; + + config_output.http.routers[routerName] = { + entryPoints: [ + ir.ssl + ? config.getRawConfig().traefik.https_entrypoint + : config.getRawConfig().traefik.http_entrypoint + ], + middlewares: routerMiddlewares, + service: serviceName, + rule, + priority: 100, + ...(ir.ssl ? { tls } : {}) + }; + + if (ir.ssl) { + config_output.http.routers[routerName + "-redirect"] = { + entryPoints: [ + config.getRawConfig().traefik.http_entrypoint + ], + middlewares: [redirectHttpsMiddlewareName], + service: serviceName, + rule, + priority: 100 + }; + } + + config_output.http.services[serviceName] = { + loadBalancer: { + servers: [{ url: `${aiGatewayUrl}/chat/completions` }], + passHostHeader: true + } + }; + } + } + return config_output; } diff --git a/server/private/lib/traefik/getTraefikConfig.ts b/server/private/lib/traefik/getTraefikConfig.ts index 4bdf204f0..475fb2523 100644 --- a/server/private/lib/traefik/getTraefikConfig.ts +++ b/server/private/lib/traefik/getTraefikConfig.ts @@ -41,7 +41,8 @@ import { siteNetworks, siteResources, Target, - targets + targets, + aiProviders } from "@server/db"; import { sanitize, @@ -86,7 +87,8 @@ export async function getTraefikConfig( generateLoginPageRouters = false, allowRawResources = true, maintenancePageUiUrl: string | null = null, - browserGatewayUiUrl: string | null = null + browserGatewayUiUrl: string | null = null, + aiGatewayUrl: string | null = null ): Promise { // Get resources with their targets and sites in a single optimized query // Start from sites on this exit node, then join to targets and resources @@ -393,6 +395,65 @@ export async function getTraefikConfig( ); } + // Inference-mode resources/siteResources have no targets/sites/network + // (their "backend" is the central AI gateway, not something on a site), + // so they can't be reached via the joins above - query them separately + // and include them on every exit node. + const inferenceResources = await db + .select({ + resourceId: resources.resourceId, + fullDomain: resources.fullDomain, + ssl: resources.ssl, + subdomain: resources.subdomain, + domainId: resources.domainId, + enabled: resources.enabled, + wildcard: resources.wildcard, + domainCertResolver: domains.certResolver, + preferWildcardCert: domains.preferWildcardCert + }) + .from(resources) + .innerJoin( + aiProviders, + eq(resources.aiProviderId, aiProviders.providerId) + ) + .leftJoin(domains, eq(domains.domainId, resources.domainId)) + .where( + and( + eq(resources.mode, "inference"), + eq(resources.enabled, true), + eq(aiProviders.enabled, true) + ) + ); + + let siteResourcesInference: { + siteResourceId: number; + alias: string | null; + ssl: boolean | null; + enabled: boolean | null; + }[] = []; + if (build == "enterprise") { + siteResourcesInference = await db + .select({ + siteResourceId: siteResources.siteResourceId, + alias: siteResources.alias, + ssl: siteResources.ssl, + enabled: siteResources.enabled + }) + .from(siteResources) + .innerJoin( + aiProviders, + eq(siteResources.aiProviderId, aiProviders.providerId) + ) + .where( + and( + eq(siteResources.mode, "inference"), + eq(siteResources.enabled, true), + eq(aiProviders.enabled, true), + isNotNull(siteResources.alias) + ) + ); + } + let validCerts: CertificateResult[] = []; if (privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) { // create a list of all domains to get certs for @@ -414,6 +475,17 @@ export async function getTraefikConfig( domains.add(bgResource.fullDomain); } } + // Include inference resource/siteResource domains + for (const ir of inferenceResources) { + if (ir.enabled && ir.ssl && ir.fullDomain) { + domains.add(ir.fullDomain); + } + } + for (const sr of siteResourcesInference) { + if (sr.enabled && sr.ssl && sr.alias) { + domains.add(sr.alias); + } + } // get the valid certs for these domains validCerts = await getValidCertificatesForDomains(domains, true); // we are caching here because this is called often // logger.debug(`Valid certs for domains: ${JSON.stringify(validCerts)}`); @@ -1445,6 +1517,199 @@ export async function getTraefikConfig( } } + if (aiGatewayUrl) { + // Public inference resources: same TLS/cert-resolver handling as + // plain http-mode resources, but the service points at the AI + // gateway instead of any real backend targets. + for (const ir of inferenceResources) { + if (!ir.enabled) continue; + if (!ir.domainId || !ir.fullDomain) continue; + + if (!config_output.http.routers) config_output.http.routers = {}; + if (!config_output.http.services) config_output.http.services = {}; + + const fullDomain = ir.fullDomain; + const irKey = `inference-r${ir.resourceId}`; + const routerName = `${irKey}-router`; + const serviceName = `${irKey}-service`; + + let rule: string; + if (ir.wildcard && fullDomain.startsWith("*.")) { + const escaped = fullDomain.slice(2).replace(/\./g, "\\."); + rule = `HostRegexp(\`^[^.]+\\.${escaped}$\`)`; + } else { + rule = `Host(\`${fullDomain}\`)`; + } + + let tls: any = {}; + if (!privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) { + const domainParts = fullDomain.split("."); + let wildCard; + if (domainParts.length <= 2) { + wildCard = `*.${domainParts.join(".")}`; + } else { + wildCard = `*.${domainParts.slice(1).join(".")}`; + } + if (!ir.subdomain) { + wildCard = fullDomain; + } + + const globalDefaultResolver = + config.getRawConfig().traefik.cert_resolver; + const globalDefaultPreferWildcard = + config.getRawConfig().traefik.prefer_wildcard_cert; + const resolverName = ir.domainCertResolver + ? ir.domainCertResolver.trim() + : globalDefaultResolver; + const preferWildcard = + ir.preferWildcardCert !== undefined && + ir.preferWildcardCert !== null + ? ir.preferWildcardCert + : globalDefaultPreferWildcard; + + tls = { + certResolver: resolverName, + ...(preferWildcard + ? { domains: [{ main: wildCard }] } + : {}) + }; + } else { + const matchingCert = validCerts.find( + (cert) => cert.queriedDomain === fullDomain + ); + if (!matchingCert) { + logger.debug( + `No matching certificate found for inference resource domain: ${fullDomain}` + ); + continue; + } + } + + const additionalMiddlewares = + config.getRawConfig().traefik.additional_middlewares || []; + const routerMiddlewares = [ + badgerMiddlewareName, + ...additionalMiddlewares + ]; + + if (ir.ssl) { + config_output.http.routers[routerName + "-redirect"] = { + entryPoints: [ + config.getRawConfig().traefik.http_entrypoint + ], + middlewares: [redirectHttpsMiddlewareName], + service: serviceName, + rule, + priority: 100 + }; + } + + config_output.http.routers[routerName] = { + entryPoints: [ + ir.ssl + ? config.getRawConfig().traefik.https_entrypoint + : config.getRawConfig().traefik.http_entrypoint + ], + middlewares: routerMiddlewares, + service: serviceName, + rule, + priority: 100, + ...(ir.ssl ? { tls } : {}) + }; + + config_output.http.services[serviceName] = { + loadBalancer: { + servers: [{ url: `${aiGatewayUrl}/chat/completions` }], + passHostHeader: true + } + }; + } + + // Private (siteResource) inference resources: routed by their alias + // instead of a public fullDomain, and deliberately WITHOUT the + // badger middleware - no per-user auth/policy stack exists for + // siteResources today (see plan doc), so gating here is + // reachability-only for now. + for (const sr of siteResourcesInference) { + if (!sr.enabled || !sr.alias) continue; + + if (!config_output.http.routers) config_output.http.routers = {}; + if (!config_output.http.services) config_output.http.services = {}; + + const alias = sr.alias; + const srKey = `inference-sr${sr.siteResourceId}`; + const routerName = `${srKey}-router`; + const serviceName = `${srKey}-service`; + const rule = `Host(\`${alias}\`)`; + + let tls: any = {}; + if (!privateConfig.getRawPrivateConfig().flags.use_pangolin_dns) { + const domainParts = alias.split("."); + const wildCard = + domainParts.length <= 2 + ? `*.${domainParts.join(".")}` + : `*.${domainParts.slice(1).join(".")}`; + + const globalDefaultResolver = + config.getRawConfig().traefik.cert_resolver; + const globalDefaultPreferWildcard = + config.getRawConfig().traefik.prefer_wildcard_cert; + + tls = { + certResolver: globalDefaultResolver, + ...(globalDefaultPreferWildcard + ? { domains: [{ main: wildCard }] } + : {}) + }; + } else { + const matchingCert = validCerts.find( + (cert) => cert.queriedDomain === alias + ); + if (!matchingCert) { + logger.debug( + `No matching certificate found for inference siteResource alias: ${alias}` + ); + continue; + } + } + + const additionalMiddlewares = + config.getRawConfig().traefik.additional_middlewares || []; + + if (sr.ssl) { + config_output.http.routers[routerName + "-redirect"] = { + entryPoints: [ + config.getRawConfig().traefik.http_entrypoint + ], + middlewares: [redirectHttpsMiddlewareName], + service: serviceName, + rule, + priority: 100 + }; + } + + config_output.http.routers[routerName] = { + entryPoints: [ + sr.ssl + ? config.getRawConfig().traefik.https_entrypoint + : config.getRawConfig().traefik.http_entrypoint + ], + middlewares: additionalMiddlewares, + service: serviceName, + rule, + priority: 100, + ...(sr.ssl ? { tls } : {}) + }; + + config_output.http.services[serviceName] = { + loadBalancer: { + servers: [{ url: `${aiGatewayUrl}/chat/completions` }], + passHostHeader: true + } + }; + } + } + if (generateLoginPageRouters) { const exitNodeLoginPages = await db .select({ diff --git a/server/private/routers/hybrid.ts b/server/private/routers/hybrid.ts index 06799bcd6..e8650e234 100644 --- a/server/private/routers/hybrid.ts +++ b/server/private/routers/hybrid.ts @@ -351,6 +351,7 @@ hybridRouter.get( } const pangolinUIUrl = config.getRawConfig().app.dashboard_url; // points to the dashboard to serve from there + const aiGatewayUrl = `${config.getRawConfig().app.dashboard_url}/api/v1/ai-gateway`; try { const traefikConfig = await getTraefikConfig( @@ -360,7 +361,8 @@ hybridRouter.get( false, // Dont include login pages, true, // allow raw resources pangolinUIUrl, // dont generate maintenance page - pangolinUIUrl // generate browser gateway targets + pangolinUIUrl, // generate browser gateway targets + aiGatewayUrl ); return response(res, { diff --git a/server/routers/aiGateway/chatCompletions.ts b/server/routers/aiGateway/chatCompletions.ts new file mode 100644 index 000000000..cddcf8404 --- /dev/null +++ b/server/routers/aiGateway/chatCompletions.ts @@ -0,0 +1,256 @@ +import { Request, Response } from "express"; +import { and, eq } from "drizzle-orm"; +import { + AiProvider, + aiModels, + aiProviders, + db, + resourceAiModels, + resources, + siteResourceAiModels, + siteResources +} from "@server/db"; +import config from "@server/lib/config"; +import { decrypt } from "@server/lib/crypto"; +import { + AiProviderAuthType, + AiProviderRoutingMode, + AiProviderType, + resolveAiProviderConfig +} from "@server/lib/aiProviderDefaults"; +import logger from "@server/logger"; +import HttpCode from "@server/types/HttpCode"; + +type ResolvedTarget = { + provider: AiProvider; + // null = no restriction; every enabled model on the provider is allowed + allowedModelIds: number[] | null; +}; + +async function resolveTarget(host: string): Promise { + const [resourceRow] = await db + .select({ resourceId: resources.resourceId, provider: aiProviders }) + .from(resources) + .innerJoin( + aiProviders, + eq(resources.aiProviderId, aiProviders.providerId) + ) + .where( + and( + eq(resources.fullDomain, host), + eq(resources.mode, "inference"), + eq(resources.enabled, true), + eq(aiProviders.enabled, true) + ) + ) + .limit(1); + + if (resourceRow) { + const restrictions = await db + .select({ modelId: resourceAiModels.modelId }) + .from(resourceAiModels) + .where(eq(resourceAiModels.resourceId, resourceRow.resourceId)); + + return { + provider: resourceRow.provider, + allowedModelIds: restrictions.length + ? restrictions.map((r) => r.modelId) + : null + }; + } + + const [siteResourceRow] = await db + .select({ + siteResourceId: siteResources.siteResourceId, + provider: aiProviders + }) + .from(siteResources) + .innerJoin( + aiProviders, + eq(siteResources.aiProviderId, aiProviders.providerId) + ) + .where( + and( + eq(siteResources.alias, host), + eq(siteResources.mode, "inference"), + eq(siteResources.enabled, true), + eq(aiProviders.enabled, true) + ) + ) + .limit(1); + + if (siteResourceRow) { + const restrictions = await db + .select({ modelId: siteResourceAiModels.modelId }) + .from(siteResourceAiModels) + .where( + eq( + siteResourceAiModels.siteResourceId, + siteResourceRow.siteResourceId + ) + ); + + return { + provider: siteResourceRow.provider, + allowedModelIds: restrictions.length + ? restrictions.map((r) => r.modelId) + : null + }; + } + + return null; +} + +// Generic OpenAI-wire-compatible passthrough. Anthropic's native API uses a +// different path/schema; everything else here is OpenAI-compatible today. +function getCompletionsPath(type: AiProviderType): string { + if (type === "anthropic") { + return "/v1/messages"; + } + return "/chat/completions"; +} + +export async function chatCompletions(req: Request, res: Response): Promise { + try { + const host = (req.headers.host || "").split(":")[0]; + if (!host) { + return res + .status(HttpCode.BAD_REQUEST) + .json({ error: { message: "Missing Host header" } }); + } + + const target = await resolveTarget(host); + if (!target) { + return res.status(HttpCode.NOT_FOUND).json({ + error: { + message: "No inference resource found for this host" + } + }); + } + + const { provider, allowedModelIds } = target; + const requestedModel = + typeof req.body?.model === "string" ? req.body.model : undefined; + + if (allowedModelIds) { + if (!requestedModel) { + return res.status(HttpCode.FORBIDDEN).json({ + error: { + message: + "This resource restricts access to specific models; a model must be specified" + } + }); + } + + const [matchedModel] = await db + .select({ modelId: aiModels.modelId }) + .from(aiModels) + .where( + and( + eq(aiModels.providerId, provider.providerId), + eq(aiModels.modelKey, requestedModel) + ) + ) + .limit(1); + + if ( + !matchedModel || + !allowedModelIds.includes(matchedModel.modelId) + ) { + return res.status(HttpCode.FORBIDDEN).json({ + error: { + message: `Model "${requestedModel}" is not permitted on this resource` + } + }); + } + } + + if (!provider.apiKey) { + return res.status(HttpCode.INTERNAL_SERVER_ERROR).json({ + error: { message: "AI provider has no API key configured" } + }); + } + + const secret = config.getRawConfig().server.secret!; + const apiKey = decrypt(provider.apiKey, secret); + + const { upstreamUrl, authType } = resolveAiProviderConfig({ + type: provider.type as AiProviderType, + upstreamUrl: provider.upstreamUrl, + authType: provider.authType as AiProviderAuthType | null, + routingMode: provider.routingMode as AiProviderRoutingMode | null + }); + + if (!upstreamUrl) { + return res.status(HttpCode.INTERNAL_SERVER_ERROR).json({ + error: { + message: "AI provider has no upstream URL configured" + } + }); + } + + const targetUrl = `${upstreamUrl.replace(/\/$/, "")}${getCompletionsPath( + provider.type as AiProviderType + )}`; + + const headers: Record = { + "Content-Type": "application/json" + }; + if (authType === "bearer") { + headers["Authorization"] = `Bearer ${apiKey}`; + } + + // No dedicated per-request TLS agent is wired up (no extra deps for + // this v1 gateway) - toggle the process-wide Node TLS check instead. + // Known limitation: this is not safe under concurrent requests mixing + // skipTlsVerification providers with strict ones. + const restoreTlsReject = process.env.NODE_TLS_REJECT_UNAUTHORIZED; + if (provider.skipTlsVerification) { + process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"; + } + + let upstreamRes: globalThis.Response; + try { + upstreamRes = await fetch(targetUrl, { + method: "POST", + headers, + body: JSON.stringify(req.body) + }); + } finally { + if (provider.skipTlsVerification) { + if (restoreTlsReject === undefined) { + delete process.env.NODE_TLS_REJECT_UNAUTHORIZED; + } else { + process.env.NODE_TLS_REJECT_UNAUTHORIZED = restoreTlsReject; + } + } + } + + const contentType = upstreamRes.headers.get("content-type") || ""; + const isStream = + req.body?.stream === true || + contentType.includes("text/event-stream"); + + res.status(upstreamRes.status); + res.setHeader("Content-Type", contentType || "application/json"); + + if (isStream && upstreamRes.body) { + res.flushHeaders(); + const reader = upstreamRes.body.getReader(); + while (true) { + const { done, value } = await reader.read(); + if (done) break; + res.write(value); + } + return res.end(); + } + + const text = await upstreamRes.text(); + return res.send(text); + } catch (error) { + logger.error(error); + return res.status(HttpCode.INTERNAL_SERVER_ERROR).json({ + error: { message: "Failed to proxy inference request" } + }); + } +} diff --git a/server/routers/aiGateway/index.ts b/server/routers/aiGateway/index.ts new file mode 100644 index 000000000..691ebebe8 --- /dev/null +++ b/server/routers/aiGateway/index.ts @@ -0,0 +1 @@ +export * from "./chatCompletions"; diff --git a/server/routers/internal.ts b/server/routers/internal.ts index 2fa5239cc..af12d4429 100644 --- a/server/routers/internal.ts +++ b/server/routers/internal.ts @@ -3,6 +3,7 @@ import * as gerbil from "@server/routers/gerbil"; import * as traefik from "@server/routers/traefik"; import * as resource from "./resource"; import * as badger from "./badger"; +import * as aiGateway from "@server/routers/aiGateway"; import * as auth from "@server/routers/auth"; import * as supporterKey from "@server/routers/supporterKey"; import * as idp from "@server/routers/idp"; @@ -63,3 +64,10 @@ internalRouter.use("/badger", badgerRouter); badgerRouter.post("/verify-session", badger.verifyResourceSession); badgerRouter.post("/exchange-session", badger.exchangeSession); + +// AI inference gateway - minimal chat-completions proxy for inference-mode +// resources/siteResources +internalRouter.post( + "/ai-gateway/chat/completions", + aiGateway.chatCompletions +); diff --git a/server/routers/resource/createResource.ts b/server/routers/resource/createResource.ts index 2ff7b9ae3..e56896e55 100644 --- a/server/routers/resource/createResource.ts +++ b/server/routers/resource/createResource.ts @@ -90,11 +90,22 @@ const createHttpResourceSchema = z domainId: z.string(), stickySession: z.boolean().optional(), postAuthPath: z.string().nullable().optional(), - mode: z.enum(["http", "ssh", "rdp", "vnc", "tcp", "udp"]).optional(), + mode: z + .enum(["http", "ssh", "rdp", "vnc", "tcp", "udp", "inference"]) + .optional(), // SSH Settings pamMode: z.enum(["passthrough", "push"]).optional(), authDaemonPort: z.int().positive().optional(), - authDaemonMode: z.enum(["site", "remote", "native"]).optional() + authDaemonMode: z.enum(["site", "remote", "native"]).optional(), + // Inference settings + aiProviderId: z + .number() + .int() + .positive() + .optional() + .describe( + "For inference-mode resources: the AI provider this resource proxies chat completions to." + ) }) .refine( (data) => { @@ -365,7 +376,8 @@ async function createHttpResource( mode, authDaemonPort, authDaemonMode, - pamMode + pamMode, + aiProviderId } = parsedBody.data; const subdomain = parsedBody.data.subdomain; const stickySession = parsedBody.data.stickySession; @@ -552,7 +564,8 @@ async function createHttpResource( postAuthPath: postAuthPath, wildcard, health: "unknown", - defaultResourcePolicyId: defaultPolicy.resourcePolicyId + defaultResourcePolicyId: defaultPolicy.resourcePolicyId, + aiProviderId: aiProviderId ?? null }) .returning(); diff --git a/server/routers/resource/updateResource.ts b/server/routers/resource/updateResource.ts index f3e0c3909..c67cb2795 100644 --- a/server/routers/resource/updateResource.ts +++ b/server/routers/resource/updateResource.ts @@ -120,6 +120,15 @@ const updateHttpResourceBodySchema = z .optional() .describe( "ID of the resource policy to apply to this resource. Set to null to remove the resource policy and fall back to the inline policy settings." + ), + aiProviderId: z + .number() + .int() + .positive() + .nullable() + .optional() + .describe( + "For inference-mode resources: the AI provider this resource proxies chat completions to. Set to null to unlink." ) }) .refine((data) => Object.keys(data).length > 0, { diff --git a/server/routers/siteResource/createSiteResource.ts b/server/routers/siteResource/createSiteResource.ts index 1a4cd9a4d..9a6f91ad7 100644 --- a/server/routers/siteResource/createSiteResource.ts +++ b/server/routers/siteResource/createSiteResource.ts @@ -78,7 +78,15 @@ const createSiteResourceSchema = z authDaemonMode: z.enum(["site", "remote", "native"]).optional(), pamMode: z.enum(["passthrough", "push"]).optional(), domainId: z.string().optional(), // only used for http mode, we need this to verify the alias is unique within the org - subdomain: z.string().optional() // only used for http mode, we need this to verify the alias is unique within the org + subdomain: z.string().optional(), // only used for http mode, we need this to verify the alias is unique within the org + aiProviderId: z + .number() + .int() + .positive() + .optional() + .describe( + "For inference-mode site resources: the AI provider this resource proxies chat completions to." + ) }) .strict() .refine( @@ -322,7 +330,8 @@ export async function createSiteResource( authDaemonMode, pamMode, domainId, - subdomain + subdomain, + aiProviderId } = parsedBody.data; // Backward compatibility: merge deprecated siteId into siteIds array @@ -594,7 +603,8 @@ export async function createSiteResource( domainId, subdomain: finalSubdomain, fullDomain, - requiresExitNodeConnection: mode === "inference" // in the future we might want to have different modes that do this + requiresExitNodeConnection: mode === "inference", // in the future we might want to have different modes that do this + aiProviderId: aiProviderId ?? null }; if (isLicensedSshPam) { if (authDaemonPort !== undefined) diff --git a/server/routers/siteResource/updateSiteResource.ts b/server/routers/siteResource/updateSiteResource.ts index 0ab928d04..0ae57c087 100644 --- a/server/routers/siteResource/updateSiteResource.ts +++ b/server/routers/siteResource/updateSiteResource.ts @@ -78,7 +78,16 @@ const updateSiteResourceSchema = z authDaemonMode: z.enum(["site", "remote", "native"]).optional(), pamMode: z.enum(["passthrough", "push"]).optional(), domainId: z.string().optional(), - subdomain: z.string().optional() + subdomain: z.string().optional(), + aiProviderId: z + .number() + .int() + .positive() + .nullable() + .optional() + .describe( + "For inference-mode site resources: the AI provider this resource proxies chat completions to. Set to null to unlink." + ) }) .strict() .refine( @@ -329,7 +338,8 @@ export async function updateSiteResource( authDaemonMode, pamMode, domainId, - subdomain + subdomain, + aiProviderId } = parsedBody.data; // Backward compatibility: merge deprecated siteId into siteIds array @@ -594,6 +604,7 @@ export async function updateSiteResource( networkId: mode === "inference" ? null : undefined, requiresExitNodeConnection: mode !== undefined ? mode === "inference" : undefined, + aiProviderId: aiProviderId, ...sshPamSet }) .where(and(eq(siteResources.siteResourceId, siteResourceId))) diff --git a/server/routers/traefik/traefikConfigProvider.ts b/server/routers/traefik/traefikConfigProvider.ts index 04cb30530..898ac42ee 100644 --- a/server/routers/traefik/traefikConfigProvider.ts +++ b/server/routers/traefik/traefikConfigProvider.ts @@ -20,6 +20,9 @@ export async function traefikConfigProvider( const maintenancePort = config.getRawConfig().server.next_port; const maintenanceHost = config.getRawConfig().server.internal_hostname; const pangolinUIUrl = `http://${maintenanceHost}:${maintenancePort}`; + const aiGatewayUrl = `http://${maintenanceHost}:${ + config.getRawConfig().server.internal_port + }/api/v1/ai-gateway`; const traefikConfig = await getTraefikConfig( currentExitNodeId, @@ -28,7 +31,8 @@ export async function traefikConfigProvider( build != "oss", // generate the login pages on the cloud and and enterprise, config.getRawConfig().traefik.allow_raw_resources, pangolinUIUrl, - pangolinUIUrl + pangolinUIUrl, + aiGatewayUrl ); if (traefikConfig?.http?.middlewares) {