Chungus

server/lib/exitNodes/exitNodeComms.ts

@@ -0,0 +1,86 @@
+import axios from "axios";
+import logger from "@server/logger";
+import { ExitNode } from "@server/db";
+
+interface ExitNodeRequest {
+    remoteType?: string;
+    localPath: string;
+    method?: "POST" | "DELETE" | "GET" | "PUT";
+    data?: any;
+    queryParams?: Record<string, string>;
+}
+
+/**
+ * Sends a request to an exit node, handling both remote and local exit nodes
+ * @param exitNode The exit node to send the request to
+ * @param request The request configuration
+ * @returns Promise<any> Response data for local nodes, undefined for remote nodes
+ */
+export async function sendToExitNode(
+    exitNode: ExitNode,
+    request: ExitNodeRequest
+): Promise<any> {
+    if (!exitNode.reachableAt) {
+        throw new Error(
+            `Exit node with ID ${exitNode.exitNodeId} is not reachable`
+        );
+    }
+
+    // Handle local exit node with HTTP API
+    const method = request.method || "POST";
+    let url = `${exitNode.reachableAt}${request.localPath}`;
+
+    // Add query parameters if provided
+    if (request.queryParams) {
+        const params = new URLSearchParams(request.queryParams);
+        url += `?${params.toString()}`;
+    }
+
+    try {
+        let response;
+
+        switch (method) {
+            case "POST":
+                response = await axios.post(url, request.data, {
+                    headers: {
+                        "Content-Type": "application/json"
+                    }
+                });
+                break;
+            case "DELETE":
+                response = await axios.delete(url);
+                break;
+            case "GET":
+                response = await axios.get(url);
+                break;
+            case "PUT":
+                response = await axios.put(url, request.data, {
+                    headers: {
+                        "Content-Type": "application/json"
+                    }
+                });
+                break;
+            default:
+                throw new Error(`Unsupported HTTP method: ${method}`);
+        }
+
+        logger.info(`Exit node request successful:`, {
+            method,
+            url,
+            status: response.data.status
+        });
+
+        return response.data;
+    } catch (error) {
+        if (axios.isAxiosError(error)) {
+            logger.error(
+                `Error making ${method} request (can Pangolin see Gerbil HTTP API?) for exit node at ${exitNode.reachableAt} (status: ${error.response?.status}): ${error.message}`
+            );
+        } else {
+            logger.error(
+                `Error making ${method} request for exit node at ${exitNode.reachableAt}: ${error}`
+            );
+        }
+        throw error;
+    }
+}

server/lib/exitNodes/exitNodes.ts

@@ -16,7 +16,7 @@ export async function verifyExitNodeOrgAccess(
     return { hasAccess: true, exitNode };
 }
 
-export async function listExitNodes(orgId: string, filterOnline = false) {
+export async function listExitNodes(orgId: string, filterOnline = false, noCloud = false) {
     // TODO: pick which nodes to send and ping better than just all of them that are not remote
     const allExitNodes = await db
         .select({
@@ -57,4 +57,9 @@ export function selectBestExitNode(
 
 export async function checkExitNodeOrg(exitNodeId: number, orgId: string) {
     return false;
-}
+}
+
+export async function resolveExitNodes(hostname: string, publicKey: string) {
+    // OSS version: simple implementation that returns empty array
+    return [];
+}

server/lib/exitNodes/getCurrentExitNodeId.ts

@@ -0,0 +1,33 @@
+import { eq } from "drizzle-orm";
+import { db, exitNodes } from "@server/db";
+import config from "@server/lib/config";
+
+let currentExitNodeId: number; // we really only need to look this up once per instance
+export async function getCurrentExitNodeId(): Promise<number> {
+    if (!currentExitNodeId) {
+        if (config.getRawConfig().gerbil.exit_node_name) {
+            const exitNodeName = config.getRawConfig().gerbil.exit_node_name!;
+            const [exitNode] = await db
+                .select({
+                    exitNodeId: exitNodes.exitNodeId
+                })
+                .from(exitNodes)
+                .where(eq(exitNodes.name, exitNodeName));
+            if (exitNode) {
+                currentExitNodeId = exitNode.exitNodeId;
+            }
+        } else {
+            const [exitNode] = await db
+                .select({
+                    exitNodeId: exitNodes.exitNodeId
+                })
+                .from(exitNodes)
+                .limit(1);
+
+            if (exitNode) {
+                currentExitNodeId = exitNode.exitNodeId;
+            }
+        }
+    }
+    return currentExitNodeId;
+}

server/lib/exitNodes/index.ts

@@ -1,2 +1,33 @@
-export * from "./exitNodes";
-export * from "./shared";
+import { build } from "@server/build";
+
+// Import both modules
+import * as exitNodesModule from "./exitNodes";
+import * as privateExitNodesModule from "./privateExitNodes";
+
+// Conditionally export exit nodes implementation based on build type
+const exitNodesImplementation = build === "oss" ? exitNodesModule : privateExitNodesModule;
+
+// Re-export all items from the selected implementation
+export const {
+    verifyExitNodeOrgAccess,
+    listExitNodes,
+    selectBestExitNode,
+    checkExitNodeOrg,
+    resolveExitNodes
+} = exitNodesImplementation;
+
+// Import communications modules
+import * as exitNodeCommsModule from "./exitNodeComms";
+import * as privateExitNodeCommsModule from "./privateExitNodeComms";
+
+// Conditionally export communications implementation based on build type
+const exitNodeCommsImplementation = build === "oss" ? exitNodeCommsModule : privateExitNodeCommsModule;
+
+// Re-export communications functions from the selected implementation
+export const {
+    sendToExitNode
+} = exitNodeCommsImplementation;
+
+// Re-export shared modules
+export * from "./subnet";
+export * from "./getCurrentExitNodeId";

server/lib/exitNodes/privateExitNodeComms.ts

@@ -0,0 +1,145 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import axios from "axios";
+import logger from "@server/logger";
+import { db, ExitNode, remoteExitNodes } from "@server/db";
+import { eq } from "drizzle-orm";
+import { sendToClient } from "../../routers/ws";
+import { config } from "../config";
+
+interface ExitNodeRequest {
+    remoteType?: string;
+    localPath: string;
+    method?: "POST" | "DELETE" | "GET" | "PUT";
+    data?: any;
+    queryParams?: Record<string, string>;
+}
+
+/**
+ * Sends a request to an exit node, handling both remote and local exit nodes
+ * @param exitNode The exit node to send the request to
+ * @param request The request configuration
+ * @returns Promise<any> Response data for local nodes, undefined for remote nodes
+ */
+export async function sendToExitNode(
+    exitNode: ExitNode,
+    request: ExitNodeRequest
+): Promise<any> {
+    if (exitNode.type === "remoteExitNode" && request.remoteType) {
+        const [remoteExitNode] = await db
+            .select()
+            .from(remoteExitNodes)
+            .where(eq(remoteExitNodes.exitNodeId, exitNode.exitNodeId))
+            .limit(1);
+
+        if (!remoteExitNode) {
+            throw new Error(
+                `Remote exit node with ID ${exitNode.exitNodeId} not found`
+            );
+        }
+
+        return sendToClient(remoteExitNode.remoteExitNodeId, {
+            type: request.remoteType,
+            data: request.data
+        });
+    } else {
+        let hostname = exitNode.reachableAt;
+
+        logger.debug(`Exit node details:`, {
+            type: exitNode.type,
+            name: exitNode.name,
+            reachableAt: exitNode.reachableAt,
+        });
+
+        logger.debug(`Configured local exit node name: ${config.getRawConfig().gerbil.exit_node_name}`);
+
+        if (exitNode.name == config.getRawConfig().gerbil.exit_node_name) {
+            hostname = config.getRawPrivateConfig().gerbil.local_exit_node_reachable_at;
+        }
+
+        if (!hostname) {
+            throw new Error(
+                `Exit node with ID ${exitNode.exitNodeId} is not reachable`
+            );
+        }
+
+        logger.debug(`Sending request to exit node at ${hostname}`, {
+            type: request.remoteType,
+            data: request.data
+        });
+
+        // Handle local exit node with HTTP API
+        const method = request.method || "POST";
+        let url = `${hostname}${request.localPath}`;
+
+        // Add query parameters if provided
+        if (request.queryParams) {
+            const params = new URLSearchParams(request.queryParams);
+            url += `?${params.toString()}`;
+        }
+
+        try {
+            let response;
+
+            switch (method) {
+                case "POST":
+                    response = await axios.post(url, request.data, {
+                        headers: {
+                            "Content-Type": "application/json"
+                        },
+                        timeout: 8000
+                    });
+                    break;
+                case "DELETE":
+                    response = await axios.delete(url, {
+                        timeout: 8000
+                    });
+                    break;
+                case "GET":
+                    response = await axios.get(url, {
+                        timeout: 8000
+                    });
+                    break;
+                case "PUT":
+                    response = await axios.put(url, request.data, {
+                        headers: {
+                            "Content-Type": "application/json"
+                        },
+                        timeout: 8000
+                    });
+                    break;
+                default:
+                    throw new Error(`Unsupported HTTP method: ${method}`);
+            }
+
+            logger.debug(`Exit node request successful:`, {
+                method,
+                url,
+                status: response.data.status
+            });
+
+            return response.data;
+        } catch (error) {
+            if (axios.isAxiosError(error)) {
+                logger.error(
+                    `Error making ${method} request (can Pangolin see Gerbil HTTP API?) for exit node at ${hostname} (status: ${error.response?.status}): ${error.message}`
+                );
+            } else {
+                logger.error(
+                    `Error making ${method} request for exit node at ${hostname}: ${error}`
+                );
+            }
+        }
+    }
+}

server/lib/exitNodes/privateExitNodes.ts

@@ -0,0 +1,379 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
+import {
+    db,
+    exitNodes,
+    exitNodeOrgs,
+    resources,
+    targets,
+    sites,
+    targetHealthCheck
+} from "@server/db";
+import logger from "@server/logger";
+import { ExitNodePingResult } from "@server/routers/newt";
+import { eq, and, or, ne, isNull } from "drizzle-orm";
+import axios from "axios";
+import config from "../config";
+
+/**
+ * Checks if an exit node is actually online by making HTTP requests to its endpoint/ping
+ * Makes up to 3 attempts in parallel with small delays, returns as soon as one succeeds
+ */
+async function checkExitNodeOnlineStatus(
+    endpoint: string | undefined
+): Promise<boolean> {
+    if (!endpoint || endpoint == "") {
+        // the endpoint can start out as a empty string
+        return false;
+    }
+
+    const maxAttempts = 3;
+    const timeoutMs = 5000; // 5 second timeout per request
+    const delayBetweenAttempts = 100; // 100ms delay between starting each attempt
+
+    // Create promises for all attempts with staggered delays
+    const attemptPromises = Array.from({ length: maxAttempts }, async (_, index) => {
+        const attemptNumber = index + 1;
+        
+        // Add delay before each attempt (except the first)
+        if (index > 0) {
+            await new Promise((resolve) => setTimeout(resolve, delayBetweenAttempts * index));
+        }
+
+        try {
+            const response = await axios.get(`http://${endpoint}/ping`, {
+                timeout: timeoutMs,
+                validateStatus: (status) => status === 200
+            });
+
+            if (response.status === 200) {
+                logger.debug(
+                    `Exit node ${endpoint} is online (attempt ${attemptNumber}/${maxAttempts})`
+                );
+                return { success: true, attemptNumber };
+            }
+            return { success: false, attemptNumber, error: 'Non-200 status' };
+        } catch (error) {
+            const errorMessage = error instanceof Error ? error.message : "Unknown error";
+            logger.debug(
+                `Exit node ${endpoint} ping failed (attempt ${attemptNumber}/${maxAttempts}): ${errorMessage}`
+            );
+            return { success: false, attemptNumber, error: errorMessage };
+        }
+    });
+
+    try {
+        // Wait for the first successful response or all to fail
+        const results = await Promise.allSettled(attemptPromises);
+        
+        // Check if any attempt succeeded
+        for (const result of results) {
+            if (result.status === 'fulfilled' && result.value.success) {
+                return true;
+            }
+        }
+
+        // All attempts failed
+        logger.warn(
+            `Exit node ${endpoint} is offline after ${maxAttempts} parallel attempts`
+        );
+        return false;
+    } catch (error) {
+        logger.warn(
+            `Unexpected error checking exit node ${endpoint}: ${error instanceof Error ? error.message : "Unknown error"}`
+        );
+        return false;
+    }
+}
+
+export async function verifyExitNodeOrgAccess(
+    exitNodeId: number,
+    orgId: string
+) {
+    const [result] = await db
+        .select({
+            exitNode: exitNodes,
+            exitNodeOrgId: exitNodeOrgs.exitNodeId
+        })
+        .from(exitNodes)
+        .leftJoin(
+            exitNodeOrgs,
+            and(
+                eq(exitNodeOrgs.exitNodeId, exitNodes.exitNodeId),
+                eq(exitNodeOrgs.orgId, orgId)
+            )
+        )
+        .where(eq(exitNodes.exitNodeId, exitNodeId));
+
+    if (!result) {
+        return { hasAccess: false, exitNode: null };
+    }
+
+    const { exitNode } = result;
+
+    // If the exit node is type "gerbil", access is allowed
+    if (exitNode.type === "gerbil") {
+        return { hasAccess: true, exitNode };
+    }
+
+    // If the exit node is type "remoteExitNode", check if it has org access
+    if (exitNode.type === "remoteExitNode") {
+        return { hasAccess: !!result.exitNodeOrgId, exitNode };
+    }
+
+    // For any other type, deny access
+    return { hasAccess: false, exitNode };
+}
+
+export async function listExitNodes(orgId: string, filterOnline = false, noCloud = false) {
+    const allExitNodes = await db
+        .select({
+            exitNodeId: exitNodes.exitNodeId,
+            name: exitNodes.name,
+            address: exitNodes.address,
+            endpoint: exitNodes.endpoint,
+            publicKey: exitNodes.publicKey,
+            listenPort: exitNodes.listenPort,
+            reachableAt: exitNodes.reachableAt,
+            maxConnections: exitNodes.maxConnections,
+            online: exitNodes.online,
+            lastPing: exitNodes.lastPing,
+            type: exitNodes.type,
+            orgId: exitNodeOrgs.orgId,
+            region: exitNodes.region
+        })
+        .from(exitNodes)
+        .leftJoin(
+            exitNodeOrgs,
+            eq(exitNodes.exitNodeId, exitNodeOrgs.exitNodeId)
+        )
+        .where(
+            or(
+                // Include all exit nodes that are NOT of type remoteExitNode
+                and(
+                    eq(exitNodes.type, "gerbil"),
+                    or(
+                        // only choose nodes that are in the same region
+                        eq(exitNodes.region, config.getRawPrivateConfig().app.region),
+                        isNull(exitNodes.region) // or for enterprise where region is not set
+                    )
+                ),
+                // Include remoteExitNode types where the orgId matches the newt's organization
+                and(
+                    eq(exitNodes.type, "remoteExitNode"),
+                    eq(exitNodeOrgs.orgId, orgId)
+                )
+            )
+        );
+
+    // Filter the nodes. If there are NO remoteExitNodes then do nothing. If there are then remove all of the non-remoteExitNodes
+    if (allExitNodes.length === 0) {
+        logger.warn("No exit nodes found for ping request!");
+        return [];
+    }
+
+    // Enhanced online checking: consider node offline if either DB says offline OR HTTP ping fails
+    const nodesWithRealOnlineStatus = await Promise.all(
+        allExitNodes.map(async (node) => {
+            // If database says it's online, verify with HTTP ping
+            let online: boolean;
+            if (filterOnline && node.type == "remoteExitNode") {
+                try {
+                    const isActuallyOnline = await checkExitNodeOnlineStatus( 
+                        node.endpoint
+                    );
+
+                    // set the item in the database if it is offline
+                    if (isActuallyOnline != node.online) {
+                        await db
+                            .update(exitNodes)
+                            .set({ online: isActuallyOnline })
+                            .where(eq(exitNodes.exitNodeId, node.exitNodeId));
+                    }
+                    online = isActuallyOnline;
+                } catch (error) {
+                    logger.warn(
+                        `Failed to check online status for exit node ${node.name} (${node.endpoint}): ${error instanceof Error ? error.message : "Unknown error"}`
+                    );
+                    online = false;
+                }
+            } else {
+                online = node.online;
+            }
+
+            return {
+                ...node,
+                online
+            };
+        })
+    );
+
+    const remoteExitNodes = nodesWithRealOnlineStatus.filter(
+        (node) =>
+            node.type === "remoteExitNode" && (!filterOnline || node.online)
+    );
+    const gerbilExitNodes = nodesWithRealOnlineStatus.filter(
+        (node) => node.type === "gerbil" && (!filterOnline || node.online) && !noCloud
+    );
+
+    // THIS PROVIDES THE FALL
+    const exitNodesList =
+        remoteExitNodes.length > 0 ? remoteExitNodes : gerbilExitNodes;
+
+    return exitNodesList;
+}
+
+/**
+ * Selects the most suitable exit node from a list of ping results.
+ *
+ * The selection algorithm follows these steps:
+ *
+ * 1. **Filter Invalid Nodes**: Excludes nodes with errors or zero weight.
+ *
+ * 2. **Sort by Latency**: Sorts valid nodes in ascending order of latency.
+ *
+ * 3. **Preferred Selection**:
+ *    - If the lowest-latency node has sufficient capacity (≥10% weight),
+ *      check if a previously connected node is also acceptable.
+ *    - The previously connected node is preferred if its latency is within
+ *      30ms or 15% of the best node’s latency.
+ *
+ * 4. **Fallback to Next Best**:
+ *    - If the lowest-latency node is under capacity, find the next node
+ *      with acceptable capacity.
+ *
+ * 5. **Final Fallback**:
+ *    - If no nodes meet the capacity threshold, fall back to the node
+ *      with the highest weight (i.e., most available capacity).
+ *
+ */
+export function selectBestExitNode(
+    pingResults: ExitNodePingResult[]
+): ExitNodePingResult | null {
+    const MIN_CAPACITY_THRESHOLD = 0.1;
+    const LATENCY_TOLERANCE_MS = 30;
+    const LATENCY_TOLERANCE_PERCENT = 0.15;
+
+    // Filter out invalid nodes
+    const validNodes = pingResults.filter((n) => !n.error && n.weight > 0);
+
+    if (validNodes.length === 0) {
+        logger.error("No valid exit nodes available");
+        return null;
+    }
+
+    // Sort by latency (ascending)
+    const sortedNodes = validNodes
+        .slice()
+        .sort((a, b) => a.latencyMs - b.latencyMs);
+    const lowestLatencyNode = sortedNodes[0];
+
+    logger.debug(
+        `Lowest latency node: ${lowestLatencyNode.exitNodeName} (${lowestLatencyNode.latencyMs} ms, weight=${lowestLatencyNode.weight.toFixed(2)})`
+    );
+
+    // If lowest latency node has enough capacity, check if previously connected node is acceptable
+    if (lowestLatencyNode.weight >= MIN_CAPACITY_THRESHOLD) {
+        const previouslyConnectedNode = sortedNodes.find(
+            (n) =>
+                n.wasPreviouslyConnected && n.weight >= MIN_CAPACITY_THRESHOLD
+        );
+
+        if (previouslyConnectedNode) {
+            const latencyDiff =
+                previouslyConnectedNode.latencyMs - lowestLatencyNode.latencyMs;
+            const percentDiff = latencyDiff / lowestLatencyNode.latencyMs;
+
+            if (
+                latencyDiff <= LATENCY_TOLERANCE_MS ||
+                percentDiff <= LATENCY_TOLERANCE_PERCENT
+            ) {
+                logger.info(
+                    `Sticking with previously connected node: ${previouslyConnectedNode.exitNodeName} ` +
+                        `(${previouslyConnectedNode.latencyMs} ms), latency diff = ${latencyDiff.toFixed(1)}ms ` +
+                        `/ ${(percentDiff * 100).toFixed(1)}%.`
+                );
+                return previouslyConnectedNode;
+            }
+        }
+
+        return lowestLatencyNode;
+    }
+
+    // Otherwise, find the next node (after the lowest) that has enough capacity
+    for (let i = 1; i < sortedNodes.length; i++) {
+        const node = sortedNodes[i];
+        if (node.weight >= MIN_CAPACITY_THRESHOLD) {
+            logger.info(
+                `Lowest latency node under capacity. Using next best: ${node.exitNodeName} ` +
+                    `(${node.latencyMs} ms, weight=${node.weight.toFixed(2)})`
+            );
+            return node;
+        }
+    }
+
+    // Fallback: pick the highest weight node
+    const fallbackNode = validNodes.reduce((a, b) =>
+        a.weight > b.weight ? a : b
+    );
+    logger.warn(
+        `No nodes with ≥10% weight. Falling back to highest capacity node: ${fallbackNode.exitNodeName}`
+    );
+    return fallbackNode;
+}
+
+export async function checkExitNodeOrg(exitNodeId: number, orgId: string) {
+    const [exitNodeOrg] = await db
+        .select()
+        .from(exitNodeOrgs)
+        .where(
+            and(
+                eq(exitNodeOrgs.exitNodeId, exitNodeId),
+                eq(exitNodeOrgs.orgId, orgId)
+            )
+        )
+        .limit(1);
+
+    if (!exitNodeOrg) {
+        return true;
+    }
+
+    return false;
+}
+
+export async function resolveExitNodes(hostname: string, publicKey: string) {
+    const resourceExitNodes = await db
+        .select({
+            endpoint: exitNodes.endpoint,
+            publicKey: exitNodes.publicKey,
+            orgId: resources.orgId
+        })
+        .from(resources)
+        .innerJoin(targets, eq(resources.resourceId, targets.resourceId))
+        .leftJoin(
+            targetHealthCheck,
+            eq(targetHealthCheck.targetId, targets.targetId)
+        )
+        .innerJoin(sites, eq(targets.siteId, sites.siteId))
+        .innerJoin(exitNodes, eq(sites.exitNodeId, exitNodes.exitNodeId))
+        .where(
+            and(
+                eq(resources.fullDomain, hostname),
+                ne(exitNodes.publicKey, publicKey),
+                ne(targetHealthCheck.hcHealth, "unhealthy")
+            )
+        );
+
+    return resourceExitNodes;
+}