import { db, targetHealthCheck, domains, aiProviders, resourceAiProviders, siteResources, exitNodes } from "@server/db"; import { and, eq, inArray, or, isNull, ne, isNotNull, desc, sql } from "drizzle-orm"; import logger from "@server/logger"; import config from "@server/lib/config"; import { resources, sites, Target, targets } from "@server/db"; import createPathRewriteMiddleware from "./middleware"; import { sanitize, encodePath, validatePathRewriteConfig } from "./utils"; import regionalCache from "@server/lib/cache"; import { AI_GATEWAY_TRUST_HEADER, AI_GATEWAY_RESOURCE_TYPE_HEADER, getAiGatewayTrustToken } from "@server/lib/aiGatewayTrust"; const redirectHttpsMiddlewareName = "redirect-to-https"; const badgerMiddlewareName = "badger"; // Define extended target type with site information type TargetWithSite = Target & { resourceId: number; targetId: number; ip: string | null; method: string | null; port: number | null; internalPort: number | null; enabled: boolean; health: string | null; site: { siteId: number; type: string; subnet: string | null; exitNodeId: number | null; online: boolean; }; }; export async function getTraefikConfig( exitNodeId: number, siteTypes: string[], filterOutNamespaceDomains = false, // UNUSED BUT USED IN PRIVATE generateLoginPageRouters = false, // UNUSED BUT USED IN PRIVATE allowRawResources = true, maintenancePageUiUrl: string | null = null, // UNUSED BUT USED IN PRIVATE browserGatewayUiUrl: string | null = null, // UNUSED BUT USED IN PRIVATE aiGatewayUrl: string | null = null ): Promise { // Get the exit node but cache it for 5 minutes to avoid hitting the DB too often const exitNodeCacheKey = `exitNode:${exitNodeId}`; let exitNode = await regionalCache.get( exitNodeCacheKey ); if (!exitNode) { [exitNode] = await db .select() .from(exitNodes) .where(eq(exitNodes.exitNodeId, exitNodeId)) .limit(1); await regionalCache.set(exitNodeCacheKey, exitNode, 300); } // Get resources with their targets and sites in a single optimized query // Start from sites on this exit node, then join to targets and resources const resourcesWithTargetsAndSites = await db .select({ // Resource fields resourceId: resources.resourceId, resourceName: resources.name, fullDomain: resources.fullDomain, ssl: resources.ssl, proxyPort: resources.proxyPort, subdomain: resources.subdomain, domainId: resources.domainId, enabled: resources.enabled, stickySession: resources.stickySession, tlsServerName: resources.tlsServerName, setHostHeader: resources.setHostHeader, enableProxy: resources.enableProxy, headers: resources.headers, proxyProtocol: resources.proxyProtocol, proxyProtocolVersion: resources.proxyProtocolVersion, mode: resources.mode, // Target fields targetId: targets.targetId, targetEnabled: targets.enabled, ip: targets.ip, method: targets.method, port: targets.port, internalPort: targets.internalPort, hcHealth: targetHealthCheck.hcHealth, path: targets.path, pathMatchType: targets.pathMatchType, rewritePath: targets.rewritePath, rewritePathType: targets.rewritePathType, priority: targets.priority, // Site fields siteId: sites.siteId, siteType: sites.type, siteOnline: sites.online, subnet: sites.exitNodeSubnet, exitNodeId: sites.exitNodeId, // Domain cert resolver fields domainCertResolver: domains.certResolver, preferWildcardCert: domains.preferWildcardCert }) .from(sites) .innerJoin(targets, eq(targets.siteId, sites.siteId)) .innerJoin(resources, eq(resources.resourceId, targets.resourceId)) .leftJoin(domains, eq(domains.domainId, resources.domainId)) .leftJoin( targetHealthCheck, eq(targetHealthCheck.targetId, targets.targetId) ) .where( and( eq(targets.enabled, true), eq(resources.enabled, true), or( eq(sites.exitNodeId, exitNodeId), and( isNull(sites.exitNodeId), sql`(${siteTypes.includes("local") ? 1 : 0} = 1)`, // only allow local sites if "local" is in siteTypes eq(sites.type, "local") ) ), inArray(sites.type, siteTypes), allowRawResources ? inArray(resources.mode, ["http", "udp", "tcp"]) // allow all three : eq(resources.mode, "http") ) ) .orderBy(desc(targets.priority), targets.targetId); // stable ordering // Group by resource and include targets with their unique site data const resourcesMap = new Map(); resourcesWithTargetsAndSites.forEach((row) => { const resourceId = row.resourceId; const resourceName = sanitize(row.resourceName) || ""; const targetPath = encodePath(row.path); // Use encodePath to avoid collisions (e.g. "/a/b" vs "/a-b") const pathMatchType = row.pathMatchType || ""; const rewritePath = row.rewritePath || ""; const rewritePathType = row.rewritePathType || ""; const priority = row.priority ?? 100; // Create a unique key combining resourceId, path config, and rewrite config const pathKey = [ targetPath, pathMatchType, rewritePath, rewritePathType ] .filter(Boolean) .join("-"); const mapKey = [resourceId, pathKey].filter(Boolean).join("-"); const key = sanitize(mapKey); if (!resourcesMap.has(mapKey)) { const validation = validatePathRewriteConfig( row.path, row.pathMatchType, row.rewritePath, row.rewritePathType ); if (!validation.isValid) { logger.error( `Invalid path rewrite configuration for resource ${resourceId}: ${validation.error}` ); return; } resourcesMap.set(mapKey, { resourceId: row.resourceId, name: resourceName, key: key, fullDomain: row.fullDomain, ssl: row.ssl, mode: row.mode, proxyPort: row.proxyPort, subdomain: row.subdomain, domainId: row.domainId, enabled: row.enabled, stickySession: row.stickySession, tlsServerName: row.tlsServerName, setHostHeader: row.setHostHeader, enableProxy: row.enableProxy, targets: [], headers: row.headers, proxyProtocol: row.proxyProtocol, proxyProtocolVersion: row.proxyProtocolVersion ?? 1, path: row.path, // the targets will all have the same path pathMatchType: row.pathMatchType, // the targets will all have the same pathMatchType rewritePath: row.rewritePath, rewritePathType: row.rewritePathType, priority: priority, // Store domain cert resolver fields domainCertResolver: row.domainCertResolver, preferWildcardCert: row.preferWildcardCert }); } resourcesMap.get(mapKey).targets.push({ resourceId: row.resourceId, targetId: row.targetId, ip: row.ip, method: row.method, port: row.port, internalPort: row.internalPort, enabled: row.targetEnabled, health: row.hcHealth, site: { siteId: row.siteId, type: row.siteType, subnet: row.subnet, exitNodeId: row.exitNodeId, online: row.siteOnline } }); }); // Inference-mode resources have no targets/sites (their "backend" is the // central AI gateway), so they can't be reached via the targets->sites // join above - query them separately and include them on every exit node. const inferenceResources = await db .selectDistinct({ resourceId: resources.resourceId, resourceName: resources.name, fullDomain: resources.fullDomain, ssl: resources.ssl, subdomain: resources.subdomain, domainId: resources.domainId, enabled: resources.enabled, wildcard: resources.wildcard, domainCertResolver: domains.certResolver, preferWildcardCert: domains.preferWildcardCert }) .from(resources) // .innerJoin( // resourceAiProviders, // eq(resources.resourceId, resourceAiProviders.resourceId) // ) // .innerJoin( // aiProviders, // eq(resourceAiProviders.providerId, aiProviders.providerId) // ) .leftJoin(domains, eq(domains.domainId, resources.domainId)) .where( and( eq(resources.mode, "inference"), eq(resources.enabled, true) // eq(aiProviders.enabled, true) ) ); // make sure we have at least one resource if (resourcesMap.size === 0 && inferenceResources.length === 0) { return {}; } const config_output: any = { http: { middlewares: { [redirectHttpsMiddlewareName]: { redirectScheme: { scheme: "https" } } } } }; // get the key and the resource for (const [, resource] of resourcesMap.entries()) { const targets = resource.targets as TargetWithSite[]; const key = resource.key; const routerName = `${key}-${resource.name}-router`; const serviceName = `${key}-${resource.name}-service`; const fullDomain = `${resource.fullDomain}`; const transportName = `${key}-transport`; const headersMiddlewareName = `${key}-headers-middleware`; if (!resource.enabled) { continue; } if (resource.mode === "http") { if (!resource.domainId || !resource.fullDomain) { continue; } // Initialize routers and services if they don't exist if (!config_output.http.routers) { config_output.http.routers = {}; } if (!config_output.http.services) { config_output.http.services = {}; } const domainParts = fullDomain.split("."); let wildCard; if (domainParts.length <= 2) { wildCard = `*.${domainParts.join(".")}`; } else { wildCard = `*.${domainParts.slice(1).join(".")}`; } if (!resource.subdomain) { wildCard = resource.fullDomain; } const globalDefaultResolver = config.getRawConfig().traefik.cert_resolver; const globalDefaultPreferWildcard = config.getRawConfig().traefik.prefer_wildcard_cert; const domainCertResolver = resource.domainCertResolver; const preferWildcardCert = resource.preferWildcardCert; let resolverName: string | undefined; let preferWildcard: boolean | undefined; // Handle both letsencrypt & custom cases if (domainCertResolver) { resolverName = domainCertResolver.trim(); } else { resolverName = globalDefaultResolver; } if ( preferWildcardCert !== undefined && preferWildcardCert !== null ) { preferWildcard = preferWildcardCert; } else { preferWildcard = globalDefaultPreferWildcard; } const tls = { certResolver: resolverName, ...(preferWildcard ? { domains: [ { main: wildCard } ] } : {}) }; const additionalMiddlewares = config.getRawConfig().traefik.additional_middlewares || []; const routerMiddlewares = [ badgerMiddlewareName, ...additionalMiddlewares ]; // Handle path rewriting middleware if ( resource.rewritePath !== null && resource.path !== null && resource.pathMatchType && resource.rewritePathType ) { // Create a unique middleware name const rewriteMiddlewareName = `rewrite-r${resource.resourceId}-${key}`; try { const rewriteResult = createPathRewriteMiddleware( rewriteMiddlewareName, resource.path, resource.pathMatchType, resource.rewritePath, resource.rewritePathType ); // Initialize middlewares object if it doesn't exist if (!config_output.http.middlewares) { config_output.http.middlewares = {}; } // the middleware to the config Object.assign( config_output.http.middlewares, rewriteResult.middlewares ); // middlewares to the router middleware chain if (rewriteResult.chain) { // For chained middlewares (like stripPrefix + addPrefix) routerMiddlewares.push(...rewriteResult.chain); } else { // Single middleware routerMiddlewares.push(rewriteMiddlewareName); } // logger.debug( // `Created path rewrite middleware ${rewriteMiddlewareName}: ${resource.pathMatchType}(${resource.path}) -> ${resource.rewritePathType}(${resource.rewritePath})` // ); } catch (error) { logger.error( `Failed to create path rewrite middleware for resource ${resource.resourceId}: ${error}` ); } } // Handle custom headers middleware if (resource.headers || resource.setHostHeader) { const headersObj: { [key: string]: string } = {}; if (resource.headers) { let headersArr: { name: string; value: string }[] = []; try { headersArr = JSON.parse(resource.headers) as { name: string; value: string; }[]; } catch (e) { logger.warn( `Failed to parse headers for resource ${resource.resourceId}: ${e}` ); } headersArr.forEach((header) => { headersObj[header.name] = header.value; }); } if (resource.setHostHeader) { headersObj["Host"] = resource.setHostHeader; } if (Object.keys(headersObj).length > 0) { if (!config_output.http.middlewares) { config_output.http.middlewares = {}; } config_output.http.middlewares[headersMiddlewareName] = { headers: { customRequestHeaders: headersObj } }; routerMiddlewares.push(headersMiddlewareName); } } // Build routing rules let rule = `Host(\`${fullDomain}\`)`; // priority logic let priority: number; if (resource.priority && resource.priority != 100) { priority = resource.priority; } else { priority = 100; if (resource.path && resource.pathMatchType) { priority += 10; if (resource.pathMatchType === "exact") { priority += 5; } else if (resource.pathMatchType === "prefix") { priority += 3; } else if (resource.pathMatchType === "regex") { priority += 2; } if (resource.path === "/") { priority = 1; // lowest for catch-all } } } if (resource.path && resource.pathMatchType) { // priority += 1; // add path to rule based on match type let path = resource.path; // if the path doesn't start with a /, add it if (!path.startsWith("/")) { path = `/${path}`; } if (resource.pathMatchType === "exact") { rule += ` && Path(\`${path}\`)`; } else if (resource.pathMatchType === "prefix") { rule += ` && PathPrefix(\`${path}\`)`; } else if (resource.pathMatchType === "regex") { rule += ` && PathRegexp(\`${resource.path}\`)`; // this is the raw path because it's a regex } } config_output.http.routers![routerName] = { entryPoints: [ resource.ssl ? config.getRawConfig().traefik.https_entrypoint : config.getRawConfig().traefik.http_entrypoint ], middlewares: routerMiddlewares, service: serviceName, rule: rule, priority: priority, ...(resource.ssl ? { tls } : {}) }; if (resource.ssl) { config_output.http.routers![routerName + "-redirect"] = { entryPoints: [ config.getRawConfig().traefik.http_entrypoint ], middlewares: [redirectHttpsMiddlewareName], service: serviceName, rule: rule, priority: priority }; } config_output.http.services![serviceName] = { loadBalancer: { servers: (() => { // Check if any sites are online // THIS IS SO THAT THERE IS SOME IMMEDIATE FEEDBACK // EVEN IF THE SITES HAVE NOT UPDATED YET FROM THE // RECEIVE BANDWIDTH ENDPOINT. // TODO: HOW TO HANDLE ^^^^^^ BETTER const anySitesOnline = targets.some( (target) => target.site.online ); return ( targets .filter((target) => { if (!target.enabled) { return false; } if (target.health == "unhealthy") { return false; } // If any sites are online, exclude offline sites if (anySitesOnline && !target.site.online) { return false; } if ( target.site.type === "local" || target.site.type === "wireguard" ) { if ( !target.ip || !target.port || !target.method ) { return false; } } else if (target.site.type === "newt") { if ( !target.internalPort || !target.method || !target.site.subnet ) { return false; } } return true; }) .map((target) => { if ( target.site.type === "local" || target.site.type === "wireguard" ) { return { url: `${target.method}://${target.ip}:${target.port}` }; } else if (target.site.type === "newt") { const ip = target.site.subnet!.split("/")[0]; return { url: `${target.method}://${ip}:${target.internalPort}` }; } }) // filter out duplicates .filter( (v, i, a) => a.findIndex( (t) => t && v && t.url === v.url ) === i ) ); })(), ...(resource.stickySession ? { sticky: { cookie: { name: "p_sticky", // TODO: make this configurable via config.yml like other cookies secure: resource.ssl, httpOnly: true } } } : {}) } }; // Add the serversTransport if TLS server name is provided if (resource.tlsServerName) { if (!config_output.http.serversTransports) { config_output.http.serversTransports = {}; } config_output.http.serversTransports![transportName] = { serverName: resource.tlsServerName, //unfortunately the following needs to be set. traefik doesn't merge the default serverTransport settings // if defined in the static config and here. if not set, self-signed certs won't work insecureSkipVerify: true }; config_output.http.services![ serviceName ].loadBalancer.serversTransport = transportName; } } else if (resource.mode === "tcp" || resource.mode === "udp") { // Non-HTTP (TCP/UDP) configuration if (!resource.enableProxy || !resource.proxyPort) { continue; } const protocol = resource.mode === "udp" ? "udp" : "tcp"; // all of the other ones are tcp const port = resource.proxyPort; if (!port) { continue; } if (!config_output[protocol]) { config_output[protocol] = { routers: {}, services: {} }; } config_output[protocol].routers[routerName] = { entryPoints: [`${protocol}-${port}`], service: serviceName, ...(protocol === "tcp" ? { rule: "HostSNI(`*`)" } : {}) }; const ppPrefix = config.getRawConfig().traefik.pp_transport_prefix; config_output[protocol].services[serviceName] = { loadBalancer: { servers: (() => { // Check if any sites are online const anySitesOnline = targets.some( (target) => target.site.online ); return targets .filter((target) => { if (!target.enabled) { return false; } // If any sites are online, exclude offline sites if (anySitesOnline && !target.site.online) { return false; } if ( target.site.type === "local" || target.site.type === "wireguard" ) { if (!target.ip || !target.port) { return false; } } else if (target.site.type === "newt") { if ( !target.internalPort || !target.site.subnet ) { return false; } } return true; }) .map((target) => { if ( target.site.type === "local" || target.site.type === "wireguard" ) { return { address: `${target.ip}:${target.port}` }; } else if (target.site.type === "newt") { const ip = target.site.subnet!.split("/")[0]; return { address: `${ip}:${target.internalPort}` }; } }); })(), ...(resource.proxyProtocol && protocol == "tcp" ? { serversTransport: `${ppPrefix}${resource.proxyProtocolVersion || 1}@file` // TODO: does @file here cause issues? } : {}), ...(resource.stickySession ? { sticky: { ipStrategy: { depth: 0, sourcePort: true } } } : {}) } }; } } if (aiGatewayUrl) { // The AI gateway may live on a different host than the inference // resource itself (e.g. a remote exit node forwarding to the // central dashboard over a tunnel). passHostHeader would forward // the resource's own Host, which that external host won't // recognize, so we pin the Host header to the gateway's own host // and smuggle the original resource host through in "p-host" // instead. let aiGatewayHost: string | undefined; try { aiGatewayHost = new URL(aiGatewayUrl).host; } catch { aiGatewayHost = undefined; } // The trust token is the same for every inference route on this exit // node, so it's defined once here and attached to each router below // instead of being duplicated into a per-resource middleware. Two // variants exist (public resource vs. siteResource) so the resource // type header lets the gateway know which kind of router the // request came through without re-deriving it from resourceId. const aiGatewayTrustMiddlewareNameResource = "ai-gateway-trust-headers-resource"; const aiGatewayTrustMiddlewareNameSiteResource = "ai-gateway-trust-headers-site-resource"; if (!config_output.http.middlewares) { config_output.http.middlewares = {}; } config_output.http.middlewares[aiGatewayTrustMiddlewareNameResource] = { headers: { customRequestHeaders: { [AI_GATEWAY_TRUST_HEADER]: getAiGatewayTrustToken(), [AI_GATEWAY_RESOURCE_TYPE_HEADER]: "resource" } } }; config_output.http.middlewares[ aiGatewayTrustMiddlewareNameSiteResource ] = { headers: { customRequestHeaders: { [AI_GATEWAY_TRUST_HEADER]: getAiGatewayTrustToken(), [AI_GATEWAY_RESOURCE_TYPE_HEADER]: "site-resource" } } }; // Public inference resources: same TLS/cert-resolver handling as // plain http-mode resources, but the service points at the AI // gateway instead of any real backend targets. for (const ir of inferenceResources) { if (!ir.enabled) continue; if (!ir.domainId || !ir.fullDomain) continue; if (!config_output.http.routers) config_output.http.routers = {}; if (!config_output.http.services) config_output.http.services = {}; const fullDomain = ir.fullDomain; const irKey = `inference-r${ir.resourceId}`; const routerName = `${irKey}-router`; const serviceName = `${irKey}-service`; let rule: string; if (ir.wildcard && fullDomain.startsWith("*.")) { const escaped = fullDomain.slice(2).replace(/\./g, "\\."); rule = `HostRegexp(\`^[^.]+\\.${escaped}$\`)`; } else { rule = `Host(\`${fullDomain}\`)`; } const domainParts = fullDomain.split("."); let wildCard; if (domainParts.length <= 2) { wildCard = `*.${domainParts.join(".")}`; } else { wildCard = `*.${domainParts.slice(1).join(".")}`; } if (!ir.subdomain) { wildCard = fullDomain; } const globalDefaultResolver = config.getRawConfig().traefik.cert_resolver; const globalDefaultPreferWildcard = config.getRawConfig().traefik.prefer_wildcard_cert; const resolverName = ir.domainCertResolver ? ir.domainCertResolver.trim() : globalDefaultResolver; const preferWildcard = ir.preferWildcardCert !== undefined && ir.preferWildcardCert !== null ? ir.preferWildcardCert : globalDefaultPreferWildcard; const tls = { certResolver: resolverName, ...(preferWildcard ? { domains: [{ main: wildCard }] } : {}) }; const irHeadersMiddlewareName = `${irKey}-headers-middleware`; if (!config_output.http.middlewares) { config_output.http.middlewares = {}; } config_output.http.middlewares[irHeadersMiddlewareName] = { headers: { customRequestHeaders: { ...(aiGatewayHost ? { Host: aiGatewayHost } : {}), "p-host": fullDomain } } }; const additionalMiddlewares = config.getRawConfig().traefik.additional_middlewares || []; const routerMiddlewares = [ badgerMiddlewareName, aiGatewayTrustMiddlewareNameResource, irHeadersMiddlewareName, ...additionalMiddlewares ]; if (ir.ssl) { config_output.http.routers[routerName + "-redirect"] = { entryPoints: [ config.getRawConfig().traefik.http_entrypoint ], middlewares: [redirectHttpsMiddlewareName], service: serviceName, rule, priority: 100 }; } config_output.http.routers[routerName] = { entryPoints: [ ir.ssl ? config.getRawConfig().traefik.https_entrypoint : config.getRawConfig().traefik.http_entrypoint ], middlewares: routerMiddlewares, service: serviceName, rule, priority: 100, ...(ir.ssl ? { tls } : {}) }; config_output.http.services[serviceName] = { loadBalancer: { servers: [{ url: aiGatewayUrl }] } }; } // Private (siteResource) inference resources: routed by their alias // instead of a public fullDomain, and deliberately WITHOUT the // badger middleware - no per-user auth/policy stack exists for // siteResources today, so gating here is reachability-only for now. const siteResourcesInference = await db .selectDistinct({ siteResourceId: siteResources.siteResourceId, fullDomain: siteResources.fullDomain, ssl: siteResources.ssl, enabled: siteResources.enabled }) .from(siteResources) .where( and( eq(siteResources.mode, "inference"), eq(siteResources.enabled, true), isNotNull(siteResources.fullDomain) ) ); for (const sr of siteResourcesInference) { if (!sr.enabled || !sr.fullDomain) continue; if (!config_output.http.routers) config_output.http.routers = {}; if (!config_output.http.services) config_output.http.services = {}; const fullDomain = sr.fullDomain; const srKey = `inference-sr${sr.siteResourceId}`; const routerName = `${srKey}-router`; const serviceName = `${srKey}-service`; const rule = `Host(\`${fullDomain}\`) && ClientIP(${exitNode.address})`; // restrict to coming from the exit node ip range that the client is connected to const domainParts = fullDomain.split("."); const wildCard = domainParts.length <= 2 ? `*.${domainParts.join(".")}` : `*.${domainParts.slice(1).join(".")}`; const globalDefaultResolver = config.getRawConfig().traefik.cert_resolver; const globalDefaultPreferWildcard = config.getRawConfig().traefik.prefer_wildcard_cert; const tls = { certResolver: globalDefaultResolver, ...(globalDefaultPreferWildcard ? { domains: [{ main: wildCard }] } : {}) }; const srHeadersMiddlewareName = `${srKey}-headers-middleware`; if (!config_output.http.middlewares) { config_output.http.middlewares = {}; } config_output.http.middlewares[srHeadersMiddlewareName] = { headers: { customRequestHeaders: { ...(aiGatewayHost ? { Host: aiGatewayHost } : {}), "p-host": fullDomain } } }; const additionalMiddlewares = config.getRawConfig().traefik.additional_middlewares || []; const routerMiddlewares = [ aiGatewayTrustMiddlewareNameSiteResource, srHeadersMiddlewareName, ...additionalMiddlewares ]; if (sr.ssl) { config_output.http.routers[routerName + "-redirect"] = { entryPoints: [ config.getRawConfig().traefik.http_entrypoint ], middlewares: [redirectHttpsMiddlewareName], service: serviceName, rule, priority: 200 // we want to match on the site resource first because the clientIP rule is more specific than the public inference resource rule, which is just the exit node IP range. so we give it a higher priority to ensure it matches first. }; } config_output.http.routers[routerName] = { entryPoints: [ sr.ssl ? config.getRawConfig().traefik.https_entrypoint : config.getRawConfig().traefik.http_entrypoint ], middlewares: routerMiddlewares, service: serviceName, rule, priority: 200, // we want to match on the site resource first because the clientIP rule is more specific than the public inference resource rule, which is just the exit node IP range. so we give it a higher priority to ensure it matches first. ...(sr.ssl ? { tls } : {}) }; config_output.http.services[serviceName] = { loadBalancer: { servers: [{ url: aiGatewayUrl }] } }; } } return config_output; }