calvin.erfmann/pangolin
1
0
Fork 0
mirror of https://github.com/fosrl/pangolin.git synced 2026-07-10 16:02:08 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into feat/paginate-user-roles-table

Browse Source
This commit is contained in:
miloschwartz
2026-04-20 21:27:51 -07:00
parent 543542713b 66c0ed5bf0
commit e3aabc6b2d
362 changed files with 13561 additions and 4286 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/routers/auth/deleteMyAccount.ts
+23 -13
View File
@@ -1,7 +1,7 @@
import { Request, Response, NextFunction } from "express";
import { z } from "zod";
import { db, orgs, userOrgs, users } from "@server/db";
import { eq, and, inArray } from "drizzle-orm";
import { eq, and, inArray, not } from "drizzle-orm";
import response from "@server/lib/response";
import HttpCode from "@server/types/HttpCode";
import createHttpError from "http-errors";
@@ -17,11 +17,10 @@ import { verifyTotpCode } from "@server/auth/totp";
import { calculateUserClientsForOrgs } from "@server/lib/calculateUserClientsForOrgs";
import { build } from "@server/build";
import { getOrgTierData } from "#dynamic/lib/billing";
import {
deleteOrgById,
sendTerminationMessages
} from "@server/lib/deleteOrg";
import { deleteOrgById, sendTerminationMessages } from "@server/lib/deleteOrg";
import { UserType } from "@server/types/UserTypes";
import { usageService } from "@server/lib/billing/usageService";
import { FeatureId } from "@server/lib/billing";
const deleteMyAccountBody = z.strictObject({
password: z.string().optional(),
@@ -98,9 +97,9 @@ export async function deleteMyAccount(
and(eq(userOrgs.userId, userId), eq(userOrgs.isOwner, true))
);
const orgIds = ownedOrgsRows.map((r) => r.orgId);
const ownedOrgIds = ownedOrgsRows.map((r) => r.orgId);
if (build === "saas" && orgIds.length > 0) {
if (build === "saas" && ownedOrgIds.length > 0) {
const primaryOrgId = ownedOrgsRows.find(
(r) => r.isBillingOrg && r.isOwner
)?.orgId;
@@ -119,14 +118,14 @@ export async function deleteMyAccount(
if (!password) {
const orgsWithNames =
orgIds.length > 0
ownedOrgIds.length > 0
? await db
.select({
orgId: orgs.orgId,
name: orgs.name
})
.from(orgs)
.where(inArray(orgs.orgId, orgIds))
.where(inArray(orgs.orgId, ownedOrgIds))
: [];
return response<DeleteMyAccountPreviewResponse>(res, {
data: {
@@ -206,9 +205,23 @@ export async function deleteMyAccount(
olmsToTerminate: allOlmsToTerminate
});
const otherOrgsTheUserWasIn = await db
.select()
.from(userOrgs)
.where(
and(
eq(userOrgs.userId, userId),
not(inArray(userOrgs.orgId, ownedOrgIds))
)
);
await db.transaction(async (trx) => {
await trx.delete(users).where(eq(users.userId, userId));
await calculateUserClientsForOrgs(userId, trx);
// loop through the other orgs and decrement the count
for (const userOrg of otherOrgsTheUserWasIn) {
await usageService.add(userOrg.orgId, FeatureId.USERS, -1, trx);
}
});
try {
@@ -233,10 +246,7 @@ export async function deleteMyAccount(
} catch (error) {
logger.error(error);
return next(
createHttpError(
HttpCode.INTERNAL_SERVER_ERROR,
"An error occurred"
)
createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred")
);
}
}
server/routers/client/getClient.ts
+1 -1
View File
@@ -243,7 +243,7 @@ registry.registerPath({
path: "/org/{orgId}/client/{niceId}",
description:
"Get a client by orgId and niceId. NiceId is a readable ID for the site and unique on a per org basis.",
tags: [OpenAPITags.Site],
tags: [OpenAPITags.Client],
request: {
params: z.object({
orgId: z.string(),
server/routers/client/listClients.ts
+32 -87
View File
@@ -29,65 +29,9 @@ import {
} from "drizzle-orm";
import { NextFunction, Request, Response } from "express";
import createHttpError from "http-errors";
import NodeCache from "node-cache";
import semver from "semver";
import { z } from "zod";
import { fromError } from "zod-validation-error";
const olmVersionCache = new NodeCache({ stdTTL: 3600 });
async function getLatestOlmVersion(): Promise<string | null> {
try {
const cachedVersion = olmVersionCache.get<string>("latestOlmVersion");
if (cachedVersion) {
return cachedVersion;
}
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 1500);
const response = await fetch(
"https://api.github.com/repos/fosrl/olm/tags",
{
signal: controller.signal
}
);
clearTimeout(timeoutId);
if (!response.ok) {
logger.warn(
`Failed to fetch latest Olm version from GitHub: ${response.status} ${response.statusText}`
);
return null;
}
let tags = await response.json();
if (!Array.isArray(tags) || tags.length === 0) {
logger.warn("No tags found for Olm repository");
return null;
}
tags = tags.filter((version) => !version.name.includes("rc"));
const latestVersion = tags[0].name;
olmVersionCache.set("latestOlmVersion", latestVersion, 3600);
return latestVersion;
} catch (error: any) {
if (error.name === "AbortError") {
logger.warn("Request to fetch latest Olm version timed out (1.5s)");
} else if (error.cause?.code === "UND_ERR_CONNECT_TIMEOUT") {
logger.warn("Connection timeout while fetching latest Olm version");
} else {
logger.warn(
"Error fetching latest Olm version:",
error.message || error
);
}
return null;
}
}
const listClientsParamsSchema = z.strictObject({
orgId: z.string()
});
@@ -413,44 +357,45 @@ export async function listClients(
};
});
const latestOlVersionPromise = getLatestOlmVersion();
// REMOVING THIS BECAUSE WE HAVE DIFFERENT TYPES OF CLIENTS NOW
// const latestOlmVersionPromise = getLatestOlmVersion();
const olmsWithUpdates: OlmWithUpdateAvailable[] = clientsWithSites.map(
(client) => {
const OlmWithUpdate: OlmWithUpdateAvailable = { ...client };
// Initially set to false, will be updated if version check succeeds
OlmWithUpdate.olmUpdateAvailable = false;
return OlmWithUpdate;
}
);
// const olmsWithUpdates: OlmWithUpdateAvailable[] = clientsWithSites.map(
// (client) => {
// const OlmWithUpdate: OlmWithUpdateAvailable = { ...client };
// // Initially set to false, will be updated if version check succeeds
// OlmWithUpdate.olmUpdateAvailable = false;
// return OlmWithUpdate;
// }
// );
// Try to get the latest version, but don't block if it fails
try {
const latestOlVersion = await latestOlVersionPromise;
// try {
// const latestOlmVersion = await latestOlVersionPromise;
if (latestOlVersion) {
olmsWithUpdates.forEach((client) => {
try {
client.olmUpdateAvailable = semver.lt(
client.olmVersion ? client.olmVersion : "",
latestOlVersion
);
} catch (error) {
client.olmUpdateAvailable = false;
}
});
}
} catch (error) {
// Log the error but don't let it block the response
logger.warn(
"Failed to check for OLM updates, continuing without update info:",
error
);
}
// if (latestOlVersion) {
// olmsWithUpdates.forEach((client) => {
// try {
// client.olmUpdateAvailable = semver.lt(
// client.olmVersion ? client.olmVersion : "",
// latestOlVersion
// );
// } catch (error) {
// client.olmUpdateAvailable = false;
// }
// });
// }
// } catch (error) {
// // Log the error but don't let it block the response
// logger.warn(
// "Failed to check for OLM updates, continuing without update info:",
// error
// );
// }
return response<ListClientsResponse>(res, {
data: {
clients: olmsWithUpdates,
clients: clientsWithSites,
pagination: {
total: totalCount,
page,
server/routers/client/listUserDevices.ts
+23 -77
View File
@@ -30,65 +30,10 @@ import {
} from "drizzle-orm";
import { NextFunction, Request, Response } from "express";
import createHttpError from "http-errors";
import NodeCache from "node-cache";
import semver from "semver";
import { z } from "zod";
import { fromError } from "zod-validation-error";
const olmVersionCache = new NodeCache({ stdTTL: 3600 });
async function getLatestOlmVersion(): Promise<string | null> {
try {
const cachedVersion = olmVersionCache.get<string>("latestOlmVersion");
if (cachedVersion) {
return cachedVersion;
}
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 1500);
const response = await fetch(
"https://api.github.com/repos/fosrl/olm/tags",
{
signal: controller.signal
}
);
clearTimeout(timeoutId);
if (!response.ok) {
logger.warn(
`Failed to fetch latest Olm version from GitHub: ${response.status} ${response.statusText}`
);
return null;
}
let tags = await response.json();
if (!Array.isArray(tags) || tags.length === 0) {
logger.warn("No tags found for Olm repository");
return null;
}
tags = tags.filter((version) => !version.name.includes("rc"));
const latestVersion = tags[0].name;
olmVersionCache.set("latestOlmVersion", latestVersion, 3600);
return latestVersion;
} catch (error: any) {
if (error.name === "AbortError") {
logger.warn("Request to fetch latest Olm version timed out (1.5s)");
} else if (error.cause?.code === "UND_ERR_CONNECT_TIMEOUT") {
logger.warn("Connection timeout while fetching latest Olm version");
} else {
logger.warn(
"Error fetching latest Olm version:",
error.message || error
);
}
return null;
}
}
const listUserDevicesParamsSchema = z.strictObject({
orgId: z.string()
});
@@ -453,29 +398,30 @@ export async function listUserDevices(
}
);
// Try to get the latest version, but don't block if it fails
try {
const latestOlmVersion = await getLatestOlmVersion();
// REMOVING THIS BECAUSE WE HAVE DIFFERENT TYPES OF CLIENTS NOW
// // Try to get the latest version, but don't block if it fails
// try {
// const latestOlmVersion = await getLatestOlmVersion();
if (latestOlmVersion) {
olmsWithUpdates.forEach((client) => {
try {
client.olmUpdateAvailable = semver.lt(
client.olmVersion ? client.olmVersion : "",
latestOlmVersion
);
} catch (error) {
client.olmUpdateAvailable = false;
}
});
}
} catch (error) {
// Log the error but don't let it block the response
logger.warn(
"Failed to check for OLM updates, continuing without update info:",
error
);
}
// if (latestOlmVersion) {
// olmsWithUpdates.forEach((client) => {
// try {
// client.olmUpdateAvailable = semver.lt(
// client.olmVersion ? client.olmVersion : "",
// latestOlmVersion
// );
// } catch (error) {
// client.olmUpdateAvailable = false;
// }
// });
// }
// } catch (error) {
// // Log the error but don't let it block the response
// logger.warn(
// "Failed to check for OLM updates, continuing without update info:",
// error
// );
// }
return response<ListUserDevicesResponse>(res, {
data: {
server/routers/domain/listDomains.ts
+2 -1
View File
@@ -103,7 +103,8 @@ export async function listDomains(
const [{ count }] = await db
.select({ count: sql<number>`count(*)` })
.from(domains);
.from(orgDomains)
.where(eq(orgDomains.orgId, orgId));
return response<ListDomainsResponse>(res, {
data: {
server/routers/external.ts
+6
View File
@@ -440,6 +440,12 @@ authenticated.get(
resource.getUserResources
);
authenticated.get(
"/org/:orgId/user-resource-aliases",
verifyOrgAccess,
resource.listUserResourceAliases
);
authenticated.get(
"/org/:orgId/domains",
verifyOrgAccess,
server/routers/gerbil/receiveBandwidth.ts
+4 -9
View File
@@ -1,6 +1,6 @@
import { Request, Response, NextFunction } from "express";
import { sql } from "drizzle-orm";
import { db } from "@server/db";
import { db, DB_TYPE } from "@server/db";
import logger from "@server/logger";
import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
@@ -96,12 +96,8 @@ async function dbQueryRows<T extends Record<string, unknown>>(
return (await anyDb.all(query)) as T[];
}
/**
* Returns true when the active database driver is SQLite (better-sqlite3).
* Used to select the appropriate bulk-update strategy.
*/
function isSQLite(): boolean {
return typeof (db as any).execute !== "function";
return DB_TYPE == "sqlite";
}
/**
@@ -175,9 +171,8 @@ export async function flushSiteBandwidthToDb(): Promise<void> {
}
// PostgreSQL: batch UPDATE … FROM (VALUES …) — single round-trip per chunk.
const valuesList = chunk.map(
([publicKey, { bytesIn, bytesOut }]) =>
sql`(${publicKey}, ${bytesIn}, ${bytesOut})`
const valuesList = chunk.map(([publicKey, { bytesIn, bytesOut }]) =>
sql`(${publicKey}::text, ${bytesIn}::real, ${bytesOut}::real)`
);
const valuesClause = sql.join(valuesList, sql`, `);
return dbQueryRows<{ orgId: string; pubKey: string }>(sql`
server/routers/newt/buildConfiguration.ts
+3 -3
View File
@@ -168,13 +168,13 @@ export async function buildClientConfigurationForNewtClient(
)
);
const resourceTarget = generateSubnetProxyTargetV2(
const resourceTargets = generateSubnetProxyTargetV2(
resource,
resourceClients
);
if (resourceTarget) {
targetsToSend.push(resourceTarget);
if (resourceTargets) {
targetsToSend.push(...resourceTargets);
}
}
server/routers/newt/handleGetConfigMessage.ts
+2 -1
View File
@@ -8,6 +8,7 @@ import { sendToExitNode } from "#dynamic/lib/exitNodes";
import { buildClientConfigurationForNewtClient } from "./buildConfiguration";
import { convertTargetsIfNessicary } from "../client/targets";
import { canCompress } from "@server/lib/clientVersionChecks";
import config from "@server/lib/config";
export const handleGetConfigMessage: MessageHandler = async (context) => {
const { message, client, sendToClient } = context;
@@ -55,7 +56,7 @@ export const handleGetConfigMessage: MessageHandler = async (context) => {
if (existingSite.lastHolePunch && now - existingSite.lastHolePunch > 5) {
logger.warn(
`handleGetConfigMessage: Site ${existingSite.siteId} last hole punch is too old, skipping`
`Site last hole punch is too old; skipping this register. The site is failing to hole punch and identify its network address with the server. Can the client reach the server on UDP port ${config.getRawConfig().gerbil.clients_start_port}?`
);
return;
}
server/routers/newt/handleNewtPingMessage.ts
+87 -4
View File
@@ -1,8 +1,11 @@
import { db, newts, sites } from "@server/db";
import { hasActiveConnections, getClientConfigVersion } from "#dynamic/routers/ws";
import { db, newts, sites, targetHealthCheck, targets } from "@server/db";
import {
hasActiveConnections,
getClientConfigVersion
} from "#dynamic/routers/ws";
import { MessageHandler } from "@server/routers/ws";
import { Newt } from "@server/db";
import { eq, lt, isNull, and, or } from "drizzle-orm";
import { eq, lt, isNull, and, or, ne, not } from "drizzle-orm";
import logger from "@server/logger";
import { sendNewtSyncMessage } from "./sync";
import { recordPing } from "./pingAccumulator";
@@ -11,6 +14,7 @@ import { recordPing } from "./pingAccumulator";
let offlineCheckerInterval: NodeJS.Timeout | null = null;
const OFFLINE_CHECK_INTERVAL = 30 * 1000; // Check every 30 seconds
const OFFLINE_THRESHOLD_MS = 2 * 60 * 1000; // 2 minutes
const OFFLINE_THRESHOLD_BANDWIDTH_MS = 8 * 60 * 1000; // 8 minutes
/**
* Starts the background interval that checks for newt sites that haven't
@@ -56,7 +60,9 @@ export const startNewtOfflineChecker = (): void => {
// Backward-compatibility check: if the newt still has an
// active WebSocket connection (older clients that don't send
// pings), keep the site online.
const isConnected = await hasActiveConnections(staleSite.newtId);
const isConnected = await hasActiveConnections(
staleSite.newtId
);
if (isConnected) {
logger.debug(
`Newt ${staleSite.newtId} has not pinged recently but is still connected via WebSocket — keeping site ${staleSite.siteId} online`
@@ -72,6 +78,83 @@ export const startNewtOfflineChecker = (): void => {
.update(sites)
.set({ online: false })
.where(eq(sites.siteId, staleSite.siteId));
const healthChecksOnSite = await db
.select()
.from(targetHealthCheck)
.innerJoin(
targets,
eq(targets.targetId, targetHealthCheck.targetId)
)
.innerJoin(sites, eq(sites.siteId, targets.siteId))
.where(eq(sites.siteId, staleSite.siteId));
for (const healthCheck of healthChecksOnSite) {
logger.info(
`Marking health check ${healthCheck.targetHealthCheck.targetHealthCheckId} offline due to site ${staleSite.siteId} being marked offline`
);
await db
.update(targetHealthCheck)
.set({ hcHealth: "unknown" })
.where(
eq(
targetHealthCheck.targetHealthCheckId,
healthCheck.targetHealthCheck
.targetHealthCheckId
)
);
}
}
// this part only effects self hosted. Its not efficient but we dont expect people to have very many wireguard sites
// select all of the wireguard sites to evaluate if they need to be offline due to the last bandwidth update
const allWireguardSites = await db
.select({
siteId: sites.siteId,
online: sites.online,
lastBandwidthUpdate: sites.lastBandwidthUpdate
})
.from(sites)
.where(
and(
eq(sites.type, "wireguard"),
not(isNull(sites.lastBandwidthUpdate))
)
);
const wireguardOfflineThreshold = Math.floor(
(Date.now() - OFFLINE_THRESHOLD_BANDWIDTH_MS) / 1000
);
// loop over each one. If its offline and there is a new update then mark it online. If its online and there is no update then mark it offline
for (const site of allWireguardSites) {
const lastBandwidthUpdate =
new Date(site.lastBandwidthUpdate!).getTime() / 1000;
if (
lastBandwidthUpdate < wireguardOfflineThreshold &&
site.online
) {
logger.info(
`Marking wireguard site ${site.siteId} offline: no bandwidth update in over ${OFFLINE_THRESHOLD_BANDWIDTH_MS / 60000} minutes`
);
await db
.update(sites)
.set({ online: false })
.where(eq(sites.siteId, site.siteId));
} else if (
lastBandwidthUpdate >= wireguardOfflineThreshold &&
!site.online
) {
logger.info(
`Marking wireguard site ${site.siteId} online: recent bandwidth update`
);
await db
.update(sites)
.set({ online: true })
.where(eq(sites.siteId, site.siteId));
}
}
} catch (error) {
logger.error("Error in newt offline checker interval", { error });
server/routers/newt/pingAccumulator.ts
+98 -88
View File
@@ -1,6 +1,6 @@
import { db } from "@server/db";
import { sites, clients, olms } from "@server/db";
import { eq, inArray } from "drizzle-orm";
import { inArray } from "drizzle-orm";
import logger from "@server/logger";
/**
@@ -21,7 +21,7 @@ import logger from "@server/logger";
*/
const FLUSH_INTERVAL_MS = 10_000; // Flush every 10 seconds
const MAX_RETRIES = 2;
const MAX_RETRIES = 5;
const BASE_DELAY_MS = 50;
// ── Site (newt) pings ──────────────────────────────────────────────────
@@ -36,6 +36,14 @@ const pendingOlmArchiveResets: Set<string> = new Set();
let flushTimer: NodeJS.Timeout | null = null;
/**
* Guard that prevents two flush cycles from running concurrently.
* setInterval does not await async callbacks, so without this a slow flush
* (e.g. due to DB latency) would overlap with the next scheduled cycle and
* the two concurrent bulk UPDATEs would deadlock each other.
*/
let isFlushing = false;
// ── Public API ─────────────────────────────────────────────────────────
/**
@@ -72,6 +80,12 @@ export function recordClientPing(
/**
* Flush all accumulated site pings to the database.
*
* Each batch of up to BATCH_SIZE rows is written with a **single** UPDATE
* statement. We use the maximum timestamp across the batch so that `lastPing`
* reflects the most recent ping seen for any site in the group. This avoids
* the multi-statement transaction that previously created additional
* row-lock ordering hazards.
*/
async function flushSitePingsToDb(): Promise<void> {
if (pendingSitePings.size === 0) {
@@ -83,55 +97,35 @@ async function flushSitePingsToDb(): Promise<void> {
const pingsToFlush = new Map(pendingSitePings);
pendingSitePings.clear();
// Sort by siteId for consistent lock ordering (prevents deadlocks)
const sortedEntries = Array.from(pingsToFlush.entries()).sort(
([a], [b]) => a - b
);
const entries = Array.from(pingsToFlush.entries());
const BATCH_SIZE = 50;
for (let i = 0; i < sortedEntries.length; i += BATCH_SIZE) {
const batch = sortedEntries.slice(i, i + BATCH_SIZE);
for (let i = 0; i < entries.length; i += BATCH_SIZE) {
const batch = entries.slice(i, i + BATCH_SIZE);
// Use the latest timestamp in the batch so that `lastPing` always
// moves forward. Using a single timestamp for the whole batch means
// we only ever need one UPDATE statement (no transaction).
const maxTimestamp = Math.max(...batch.map(([, ts]) => ts));
const siteIds = batch.map(([id]) => id);
try {
await withRetry(async () => {
// Group by timestamp for efficient bulk updates
const byTimestamp = new Map<number, number[]>();
for (const [siteId, timestamp] of batch) {
const group = byTimestamp.get(timestamp) || [];
group.push(siteId);
byTimestamp.set(timestamp, group);
}
if (byTimestamp.size === 1) {
const [timestamp, siteIds] = Array.from(
byTimestamp.entries()
)[0];
await db
.update(sites)
.set({
online: true,
lastPing: timestamp
})
.where(inArray(sites.siteId, siteIds));
} else {
await db.transaction(async (tx) => {
for (const [timestamp, siteIds] of byTimestamp) {
await tx
.update(sites)
.set({
online: true,
lastPing: timestamp
})
.where(inArray(sites.siteId, siteIds));
}
});
}
await db
.update(sites)
.set({
online: true,
lastPing: maxTimestamp
})
.where(inArray(sites.siteId, siteIds));
}, "flushSitePingsToDb");
} catch (error) {
logger.error(
`Failed to flush site ping batch (${batch.length} sites), re-queuing for next cycle`,
{ error }
);
// Re-queue only if the preserved timestamp is newer than any
// update that may have landed since we snapshotted.
for (const [siteId, timestamp] of batch) {
const existing = pendingSitePings.get(siteId);
if (!existing || existing < timestamp) {
@@ -144,6 +138,8 @@ async function flushSitePingsToDb(): Promise<void> {
/**
* Flush all accumulated client (OLM) pings to the database.
*
* Same single-UPDATE-per-batch approach as `flushSitePingsToDb`.
*/
async function flushClientPingsToDb(): Promise<void> {
if (pendingClientPings.size === 0 && pendingOlmArchiveResets.size === 0) {
@@ -159,51 +155,25 @@ async function flushClientPingsToDb(): Promise<void> {
// ── Flush client pings ─────────────────────────────────────────────
if (pingsToFlush.size > 0) {
const sortedEntries = Array.from(pingsToFlush.entries()).sort(
([a], [b]) => a - b
);
const entries = Array.from(pingsToFlush.entries());
const BATCH_SIZE = 50;
for (let i = 0; i < sortedEntries.length; i += BATCH_SIZE) {
const batch = sortedEntries.slice(i, i + BATCH_SIZE);
for (let i = 0; i < entries.length; i += BATCH_SIZE) {
const batch = entries.slice(i, i + BATCH_SIZE);
const maxTimestamp = Math.max(...batch.map(([, ts]) => ts));
const clientIds = batch.map(([id]) => id);
try {
await withRetry(async () => {
const byTimestamp = new Map<number, number[]>();
for (const [clientId, timestamp] of batch) {
const group = byTimestamp.get(timestamp) || [];
group.push(clientId);
byTimestamp.set(timestamp, group);
}
if (byTimestamp.size === 1) {
const [timestamp, clientIds] = Array.from(
byTimestamp.entries()
)[0];
await db
.update(clients)
.set({
lastPing: timestamp,
online: true,
archived: false
})
.where(inArray(clients.clientId, clientIds));
} else {
await db.transaction(async (tx) => {
for (const [timestamp, clientIds] of byTimestamp) {
await tx
.update(clients)
.set({
lastPing: timestamp,
online: true,
archived: false
})
.where(
inArray(clients.clientId, clientIds)
);
}
});
}
await db
.update(clients)
.set({
lastPing: maxTimestamp,
online: true,
archived: false
})
.where(inArray(clients.clientId, clientIds));
}, "flushClientPingsToDb");
} catch (error) {
logger.error(
@@ -260,7 +230,12 @@ export async function flushPingsToDb(): Promise<void> {
/**
* Simple retry wrapper with exponential backoff for transient errors
* (connection timeouts, unexpected disconnects).
* (deadlocks, connection timeouts, unexpected disconnects).
*
* PostgreSQL deadlocks (40P01) are always safe to retry: the database
* guarantees exactly one winner per deadlock pair, so the loser just needs
* to try again. MAX_RETRIES is intentionally higher than typical connection
* retry budgets to give deadlock victims enough chances to succeed.
*/
async function withRetry<T>(
operation: () => Promise<T>,
@@ -277,7 +252,8 @@ async function withRetry<T>(
const jitter = Math.random() * baseDelay;
const delay = baseDelay + jitter;
logger.warn(
`Transient DB error in ${context}, retrying attempt ${attempt}/${MAX_RETRIES} after ${delay.toFixed(0)}ms`
`Transient DB error in ${context}, retrying attempt ${attempt}/${MAX_RETRIES} after ${delay.toFixed(0)}ms`,
{ code: error?.code ?? error?.cause?.code }
);
await new Promise((resolve) => setTimeout(resolve, delay));
continue;
@@ -288,14 +264,14 @@ async function withRetry<T>(
}
/**
* Detect transient connection errors that are safe to retry.
* Detect transient errors that are safe to retry.
*/
function isTransientError(error: any): boolean {
if (!error) return false;
const message = (error.message || "").toLowerCase();
const causeMessage = (error.cause?.message || "").toLowerCase();
const code = error.code || "";
const code = error.code || error.cause?.code || "";
// Connection timeout / terminated
if (
@@ -308,12 +284,17 @@ function isTransientError(error: any): boolean {
return true;
}
// PostgreSQL deadlock
// PostgreSQL deadlock detected — always safe to retry (one winner guaranteed)
if (code === "40P01" || message.includes("deadlock")) {
return true;
}
// ECONNRESET, ECONNREFUSED, EPIPE
// PostgreSQL serialization failure
if (code === "40001") {
return true;
}
// ECONNRESET, ECONNREFUSED, EPIPE, ETIMEDOUT
if (
code === "ECONNRESET" ||
code === "ECONNREFUSED" ||
@@ -337,12 +318,26 @@ export function startPingAccumulator(): void {
}
flushTimer = setInterval(async () => {
// Skip this tick if the previous flush is still in progress.
// setInterval does not await async callbacks, so without this guard
// two flush cycles can run concurrently and deadlock each other on
// overlapping bulk UPDATE statements.
if (isFlushing) {
logger.debug(
"Ping accumulator: previous flush still in progress, skipping cycle"
);
return;
}
isFlushing = true;
try {
await flushPingsToDb();
} catch (error) {
logger.error("Unhandled error in ping accumulator flush", {
error
});
} finally {
isFlushing = false;
}
}, FLUSH_INTERVAL_MS);
@@ -364,7 +359,22 @@ export async function stopPingAccumulator(): Promise<void> {
flushTimer = null;
}
// Final flush to persist any remaining pings
// Final flush to persist any remaining pings.
// Wait for any in-progress flush to finish first so we don't race.
if (isFlushing) {
logger.debug(
"Ping accumulator: waiting for in-progress flush before stopping…"
);
await new Promise<void>((resolve) => {
const poll = setInterval(() => {
if (!isFlushing) {
clearInterval(poll);
resolve();
}
}, 50);
});
}
try {
await flushPingsToDb();
} catch (error) {
@@ -379,4 +389,4 @@ export async function stopPingAccumulator(): Promise<void> {
*/
export function getPendingPingCount(): number {
return pendingSitePings.size + pendingClientPings.size;
}
}
server/routers/newt/registerNewt.ts
+21 -1
View File
@@ -27,7 +27,7 @@ import { build } from "@server/build";
import { usageService } from "@server/lib/billing/usageService";
import { FeatureId } from "@server/lib/billing";
import { INSPECT_MAX_BYTES } from "buffer";
import { v } from "@faker-js/faker/dist/airline-Dz1uGqgJ";
import { getNextAvailableClientSubnet } from "@server/lib/ip";
const bodySchema = z.object({
provisioningKey: z.string().nonempty(),
@@ -152,6 +152,11 @@ export async function registerNewt(
createHttpError(HttpCode.NOT_FOUND, "Organization not found")
);
}
if (!org.subnet) {
return next(
createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "Organization subnet not found")
);
}
// SaaS billing check
if (build == "saas") {
@@ -190,6 +195,20 @@ export async function registerNewt(
let newSiteId: number | undefined;
await db.transaction(async (trx) => {
const newClientAddress = await getNextAvailableClientSubnet(orgId);
if (!newClientAddress) {
return next(
createHttpError(
HttpCode.INTERNAL_SERVER_ERROR,
"No available subnet found"
)
);
}
let clientAddress = newClientAddress.split("/")[0];
clientAddress = `${clientAddress}/${org.subnet!.split("/")[1]}`; // we want the block size of the whole org
// Create the site (type "newt", name = niceId)
const [newSite] = await trx
.insert(sites)
@@ -197,6 +216,7 @@ export async function registerNewt(
orgId,
name: name || niceId,
niceId,
address: clientAddress,
type: "newt",
dockerSocketEnabled: true,
status: keyRecord.approveNewSites ? "approved" : "pending",
server/routers/olm/handleOlmRegisterMessage.ts
+2 -1
View File
@@ -20,6 +20,7 @@ import { handleFingerprintInsertion } from "./fingerprintingUtils";
import { Alias } from "@server/lib/ip";
import { build } from "@server/build";
import { canCompress } from "@server/lib/clientVersionChecks";
import config from "@server/lib/config";
export const handleOlmRegisterMessage: MessageHandler = async (context) => {
logger.info("Handling register olm message!");
@@ -274,7 +275,7 @@ export const handleOlmRegisterMessage: MessageHandler = async (context) => {
// TODO: I still think there is a better way to do this rather than locking it out here but ???
if (now - (client.lastHolePunch || 0) > 5 && sitesCount > 0) {
logger.warn(
"Client last hole punch is too old and we have sites to send; skipping this register"
`Client last hole punch is too old and we have sites to send; skipping this register. The client is failing to hole punch and identify its network address with the server. Can the client reach the server on UDP port ${config.getRawConfig().gerbil.clients_start_port}?`
);
return;
}
server/routers/orgIdp/types.ts
+19
View File
@@ -25,3 +25,22 @@ export type ListOrgIdpsResponse = {
offset: number;
};
};
export type ListUserAdminOrgIdpsEntry = {
idpId: number;
orgId: string;
orgName: string;
name: string;
type: string;
variant: string;
tags: string | null;
};
export type ListUserAdminOrgIdpsResponse = {
idps: ListUserAdminOrgIdpsEntry[];
pagination: {
total: number;
limit: number;
offset: number;
};
};
server/routers/resource/createResource.ts
+30 -2
View File
@@ -1,6 +1,6 @@
import { Request, Response, NextFunction } from "express";
import { z } from "zod";
import { db, loginPage } from "@server/db";
import { db, domainNamespaces, loginPage } from "@server/db";
import {
domains,
orgDomains,
@@ -24,6 +24,8 @@ import { build } from "@server/build";
import { createCertificate } from "#dynamic/routers/certificates/createCertificate";
import { getUniqueResourceName } from "@server/db/names";
import { validateAndConstructDomain } from "@server/lib/domainUtils";
import { isSubscribed } from "#dynamic/lib/isSubscribed";
import { tierMatrix } from "@server/lib/billing/tierMatrix";
const createResourceParamsSchema = z.strictObject({
orgId: z.string()
@@ -112,7 +114,10 @@ export async function createResource(
const { orgId } = parsedParams.data;
if (req.user && (!req.userOrgRoleIds || req.userOrgRoleIds.length === 0)) {
if (
req.user &&
(!req.userOrgRoleIds || req.userOrgRoleIds.length === 0)
) {
return next(
createHttpError(HttpCode.FORBIDDEN, "User does not have a role")
);
@@ -193,6 +198,29 @@ async function createHttpResource(
const subdomain = parsedBody.data.subdomain;
const stickySession = parsedBody.data.stickySession;
if (build == "saas" && !isSubscribed(orgId!, tierMatrix.domainNamespaces)) {
// grandfather in existing users
const lastAllowedDate = new Date("2026-04-13");
const userCreatedDate = new Date(req.user?.dateCreated || new Date());
if (userCreatedDate > lastAllowedDate) {
// check if this domain id is a namespace domain and if so, reject
const domain = await db
.select()
.from(domainNamespaces)
.where(eq(domainNamespaces.domainId, domainId))
.limit(1);
if (domain.length > 0) {
return next(
createHttpError(
HttpCode.BAD_REQUEST,
"Your current subscription does not support custom domain namespaces. Please upgrade to access this feature."
)
);
}
}
}
// Validate domain and construct full domain
const domainResult = await validateAndConstructDomain(
domainId,
server/routers/resource/getUserResources.ts
+3 -1
View File
@@ -142,6 +142,7 @@ export async function getUserResources(
let siteResourcesData: Array<{
siteResourceId: number;
name: string;
niceId: string;
destination: string;
mode: string;
protocol: string | null;
@@ -154,6 +155,7 @@ export async function getUserResources(
.select({
siteResourceId: siteResources.siteResourceId,
name: siteResources.name,
niceId: siteResources.niceId,
destination: siteResources.destination,
mode: siteResources.mode,
protocol: siteResources.protocol,
@@ -249,7 +251,7 @@ export async function getUserResources(
});
return response(res, {
data: {
data: {
resources: resourcesWithAuth,
siteResources: siteResourcesFormatted
},
server/routers/resource/index.ts
+1
View File
@@ -22,6 +22,7 @@ export * from "./deleteResourceRule";
export * from "./listResourceRules";
export * from "./updateResourceRule";
export * from "./getUserResources";
export * from "./listUserResourceAliases";
export * from "./setResourceHeaderAuth";
export * from "./addEmailToResourceWhitelist";
export * from "./removeEmailFromResourceWhitelist";
server/routers/resource/listResources.ts
+6 -2
View File
@@ -6,6 +6,7 @@ import {
resourcePincode,
resources,
roleResources,
sites,
targetHealthCheck,
targets,
userResources
@@ -138,6 +139,7 @@ export type ResourceWithTargets = {
port: number;
enabled: boolean;
healthStatus: "healthy" | "unhealthy" | "unknown" | null;
siteName: string | null;
}>;
};
@@ -446,14 +448,16 @@ export async function listResources(
port: targets.port,
enabled: targets.enabled,
healthStatus: targetHealthCheck.hcHealth,
hcEnabled: targetHealthCheck.hcEnabled
hcEnabled: targetHealthCheck.hcEnabled,
siteName: sites.name
})
.from(targets)
.where(inArray(targets.resourceId, resourceIdList))
.leftJoin(
targetHealthCheck,
eq(targetHealthCheck.targetId, targets.targetId)
);
)
.leftJoin(sites, eq(targets.siteId, sites.siteId));
// avoids TS issues with reduce/never[]
const map = new Map<number, ResourceWithTargets>();
server/routers/resource/listUserResourceAliases.ts
+262
View File
@@ -0,0 +1,262 @@
import { Request, Response, NextFunction } from "express";
import {
db,
siteResources,
userSiteResources,
roleSiteResources,
userOrgRoles,
userOrgs
} from "@server/db";
import { and, eq, inArray, asc, isNotNull, ne } from "drizzle-orm";
import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
import response from "@server/lib/response";
import logger from "@server/logger";
import { z } from "zod";
import { fromZodError } from "zod-validation-error";
import type { PaginatedResponse } from "@server/types/Pagination";
import { OpenAPITags, registry } from "@server/openApi";
import { localCache } from "#dynamic/lib/cache";
const USER_RESOURCE_ALIASES_CACHE_TTL_SEC = 60;
function userResourceAliasesCacheKey(
orgId: string,
userId: string,
page: number,
pageSize: number
) {
return `userResourceAliases:${orgId}:${userId}:${page}:${pageSize}`;
}
const listUserResourceAliasesParamsSchema = z.strictObject({
orgId: z.string()
});
const listUserResourceAliasesQuerySchema = z.object({
pageSize: z.coerce
.number<string>()
.int()
.positive()
.optional()
.catch(20)
.default(20)
.openapi({
type: "integer",
default: 20,
description: "Number of items per page"
}),
page: z.coerce
.number<string>()
.int()
.min(0)
.optional()
.catch(1)
.default(1)
.openapi({
type: "integer",
default: 1,
description: "Page number to retrieve"
})
});
export type ListUserResourceAliasesResponse = PaginatedResponse<{
aliases: string[];
}>;
// registry.registerPath({
// method: "get",
// path: "/org/{orgId}/user-resource-aliases",
// description:
// "List private (host-mode) site resource aliases the authenticated user can access in the organization, paginated.",
// tags: [OpenAPITags.PrivateResource],
// request: {
// params: z.object({
// orgId: z.string()
// }),
// query: listUserResourceAliasesQuerySchema
// },
// responses: {}
// });
export async function listUserResourceAliases(
req: Request,
res: Response,
next: NextFunction
): Promise<any> {
try {
const parsedQuery = listUserResourceAliasesQuerySchema.safeParse(
req.query
);
if (!parsedQuery.success) {
return next(
createHttpError(
HttpCode.BAD_REQUEST,
fromZodError(parsedQuery.error)
)
);
}
const { page, pageSize } = parsedQuery.data;
const parsedParams = listUserResourceAliasesParamsSchema.safeParse(
req.params
);
if (!parsedParams.success) {
return next(
createHttpError(
HttpCode.BAD_REQUEST,
fromZodError(parsedParams.error)
)
);
}
const { orgId } = parsedParams.data;
const userId = req.user?.userId;
if (!userId) {
return next(
createHttpError(HttpCode.UNAUTHORIZED, "User not authenticated")
);
}
const [userOrg] = await db
.select()
.from(userOrgs)
.where(and(eq(userOrgs.userId, userId), eq(userOrgs.orgId, orgId)))
.limit(1);
if (!userOrg) {
return next(
createHttpError(HttpCode.FORBIDDEN, "User not in organization")
);
}
const cacheKey = userResourceAliasesCacheKey(
orgId,
userId,
page,
pageSize
);
const cachedData: ListUserResourceAliasesResponse | undefined =
localCache.get(cacheKey);
if (cachedData) {
return response<ListUserResourceAliasesResponse>(res, {
data: cachedData,
success: true,
error: false,
message: "User resource aliases retrieved successfully",
status: HttpCode.OK
});
}
const userRoleIds = await db
.select({ roleId: userOrgRoles.roleId })
.from(userOrgRoles)
.where(
and(
eq(userOrgRoles.userId, userId),
eq(userOrgRoles.orgId, orgId)
)
)
.then((rows) => rows.map((r) => r.roleId));
const directSiteResourcesQuery = db
.select({ siteResourceId: userSiteResources.siteResourceId })
.from(userSiteResources)
.where(eq(userSiteResources.userId, userId));
const roleSiteResourcesQuery =
userRoleIds.length > 0
? db
.select({
siteResourceId: roleSiteResources.siteResourceId
})
.from(roleSiteResources)
.where(inArray(roleSiteResources.roleId, userRoleIds))
: Promise.resolve([]);
const [directSiteResourceResults, roleSiteResourceResults] =
await Promise.all([
directSiteResourcesQuery,
roleSiteResourcesQuery
]);
const accessibleSiteResourceIds = [
...directSiteResourceResults.map((r) => r.siteResourceId),
...roleSiteResourceResults.map((r) => r.siteResourceId)
];
if (accessibleSiteResourceIds.length === 0) {
const data: ListUserResourceAliasesResponse = {
aliases: [],
pagination: {
total: 0,
pageSize,
page
}
};
localCache.set(cacheKey, data, USER_RESOURCE_ALIASES_CACHE_TTL_SEC);
return response<ListUserResourceAliasesResponse>(res, {
data,
success: true,
error: false,
message: "User resource aliases retrieved successfully",
status: HttpCode.OK
});
}
const whereClause = and(
eq(siteResources.orgId, orgId),
eq(siteResources.enabled, true),
eq(siteResources.mode, "host"),
isNotNull(siteResources.alias),
ne(siteResources.alias, ""),
inArray(siteResources.siteResourceId, accessibleSiteResourceIds)
);
const baseSelect = () =>
db
.select({ alias: siteResources.alias })
.from(siteResources)
.where(whereClause);
const countQuery = db.$count(baseSelect().as("filtered_aliases"));
const [rows, totalCount] = await Promise.all([
baseSelect()
.orderBy(asc(siteResources.alias))
.limit(pageSize)
.offset(pageSize * (page - 1)),
countQuery
]);
const aliases = rows.map((r) => r.alias as string);
const data: ListUserResourceAliasesResponse = {
aliases,
pagination: {
total: totalCount,
pageSize,
page
}
};
localCache.set(cacheKey, data, USER_RESOURCE_ALIASES_CACHE_TTL_SEC);
return response<ListUserResourceAliasesResponse>(res, {
data,
success: true,
error: false,
message: "User resource aliases retrieved successfully",
status: HttpCode.OK
});
} catch (error) {
logger.error(error);
return next(
createHttpError(
HttpCode.INTERNAL_SERVER_ERROR,
"Internal server error"
)
);
}
}
server/routers/resource/updateResource.ts
+34 -3
View File
@@ -1,6 +1,6 @@
import { Request, Response, NextFunction } from "express";
import { z } from "zod";
import { db, loginPage } from "@server/db";
import { db, domainNamespaces, loginPage } from "@server/db";
import {
domains,
Org,
@@ -25,6 +25,7 @@ import { validateAndConstructDomain } from "@server/lib/domainUtils";
import { build } from "@server/build";
import { isLicensedOrSubscribed } from "#dynamic/lib/isLicencedOrSubscribed";
import { tierMatrix } from "@server/lib/billing/tierMatrix";
import { isSubscribed } from "#dynamic/lib/isSubscribed";
const updateResourceParamsSchema = z.strictObject({
resourceId: z.string().transform(Number).pipe(z.int().positive())
@@ -120,7 +121,9 @@ const updateHttpResourceBodySchema = z
if (data.headers) {
// HTTP header values must be visible ASCII or horizontal whitespace, no control chars (RFC 7230)
const validHeaderValue = /^[\t\x20-\x7E]*$/;
return data.headers.every((h) => validHeaderValue.test(h.value));
return data.headers.every((h) =>
validHeaderValue.test(h.value)
);
}
return true;
},
@@ -318,6 +321,34 @@ async function updateHttpResource(
if (updateData.domainId) {
const domainId = updateData.domainId;
if (
build == "saas" &&
!isSubscribed(resource.orgId, tierMatrix.domainNamespaces)
) {
// grandfather in existing users
const lastAllowedDate = new Date("2026-04-13");
const userCreatedDate = new Date(
req.user?.dateCreated || new Date()
);
if (userCreatedDate > lastAllowedDate) {
// check if this domain id is a namespace domain and if so, reject
const domain = await db
.select()
.from(domainNamespaces)
.where(eq(domainNamespaces.domainId, domainId))
.limit(1);
if (domain.length > 0) {
return next(
createHttpError(
HttpCode.BAD_REQUEST,
"Your current subscription does not support custom domain namespaces. Please upgrade to access this feature."
)
);
}
}
}
// Validate domain and construct full domain
const domainResult = await validateAndConstructDomain(
domainId,
@@ -366,7 +397,7 @@ async function updateHttpResource(
);
}
}
if (build != "oss") {
const existingLoginPages = await db
.select()
server/routers/site/createSite.ts
+72 -14
View File
@@ -15,11 +15,12 @@ import moment from "moment";
import { OpenAPITags, registry } from "@server/openApi";
import { hashPassword } from "@server/auth/password";
import { isValidIP } from "@server/lib/validators";
import { isIpInCidr } from "@server/lib/ip";
import { getNextAvailableClientSubnet, isIpInCidr } from "@server/lib/ip";
import { verifyExitNodeOrgAccess } from "#dynamic/lib/exitNodes";
import { build } from "@server/build";
import { usageService } from "@server/lib/billing/usageService";
import { FeatureId } from "@server/lib/billing";
import { generateId } from "@server/auth/sessions/app";
const createSiteParamsSchema = z.strictObject({
orgId: z.string()
@@ -28,6 +29,7 @@ const createSiteParamsSchema = z.strictObject({
const createSiteSchema = z.strictObject({
name: z.string().min(1).max(255),
exitNodeId: z.int().positive().optional(),
niceId: z.string().min(1).max(255).optional(),
// subdomain: z
// .string()
// .min(1)
@@ -52,7 +54,10 @@ const createSiteSchema = z.strictObject({
export type CreateSiteBody = z.infer<typeof createSiteSchema>;
export type CreateSiteResponse = Site;
export type CreateSiteResponse = Site & {
newtId?: string;
secret?: string;
};
registry.registerPath({
method: "put",
@@ -64,7 +69,11 @@ registry.registerPath({
body: {
content: {
"application/json": {
schema: createSiteSchema
schema: createSiteSchema,
example: {
name: "My Site",
type: "newt"
}
}
}
}
@@ -96,9 +105,13 @@ export async function createSite(
subnet,
newtId,
secret,
address
address,
niceId
} = parsedBody.data;
const updatedNewtSecret = secret || generateId(48);
const updatedNewtId = newtId || generateId(15);
const parsedParams = createSiteParamsSchema.safeParse(req.params);
if (!parsedParams.success) {
return next(
@@ -111,7 +124,10 @@ export async function createSite(
const { orgId } = parsedParams.data;
if (req.user && (!req.userOrgRoleIds || req.userOrgRoleIds.length === 0)) {
if (
req.user &&
(!req.userOrgRoleIds || req.userOrgRoleIds.length === 0)
) {
return next(
createHttpError(HttpCode.FORBIDDEN, "User does not have a role")
);
@@ -227,6 +243,18 @@ export async function createSite(
)
);
}
} else {
const newClientAddress = await getNextAvailableClientSubnet(orgId);
if (!newClientAddress) {
return next(
createHttpError(
HttpCode.INTERNAL_SERVER_ERROR,
"No available address found"
)
);
}
updatedAddress = newClientAddress.split("/")[0];
}
if (subnet && exitNodeId) {
@@ -285,7 +313,31 @@ export async function createSite(
}
}
const niceId = await getUniqueSiteName(orgId);
let updatedNiceId = niceId;
if (!niceId) {
updatedNiceId = await getUniqueSiteName(orgId);
} else {
// make sure the niceId is unique
const existingSite = await db
.select()
.from(sites)
.where(
and(
eq(sites.niceId, niceId),
eq(sites.orgId, orgId)
)
)
.limit(1);
if (existingSite.length > 0) {
return next(
createHttpError(
HttpCode.CONFLICT,
`Nice ID ${niceId} already exists. Please choose a different one.`
)
);
}
}
let newSite: Site | undefined;
await db.transaction(async (trx) => {
@@ -295,7 +347,7 @@ export async function createSite(
.values({ // NOTE: NO SUBNET OR EXIT NODE ID PASSED IN HERE BECAUSE ITS NOW CHOSEN ON CONNECT
orgId,
name,
niceId,
niceId: updatedNiceId!,
address: updatedAddress || null,
type,
dockerSocketEnabled: true,
@@ -353,7 +405,7 @@ export async function createSite(
orgId,
exitNodeId,
name,
niceId,
niceId: updatedNiceId!,
subnet,
type,
pubKey: pubKey || null,
@@ -367,8 +419,7 @@ export async function createSite(
exitNodeId: exitNodeId || null,
orgId,
name,
niceId,
address: updatedAddress || null,
niceId: updatedNiceId!,
type,
dockerSocketEnabled: false,
online: true,
@@ -402,7 +453,10 @@ export async function createSite(
siteId: newSite.siteId
});
if (req.user && !req.userOrgRoleIds?.includes(adminRole[0].roleId)) {
if (
req.user &&
!req.userOrgRoleIds?.includes(adminRole[0].roleId)
) {
// make sure the user can access the site
trx.insert(userSites).values({
userId: req.user?.userId!,
@@ -412,10 +466,10 @@ export async function createSite(
// add the peer to the exit node
if (type == "newt") {
const secretHash = await hashPassword(secret!);
const secretHash = await hashPassword(updatedNewtSecret);
await trx.insert(newts).values({
newtId: newtId!,
newtId: updatedNewtId,
secretHash,
siteId: newSite.siteId,
dateCreated: moment().toISOString()
@@ -458,7 +512,11 @@ export async function createSite(
}
return response<CreateSiteResponse>(res, {
data: newSite,
data: {
...newSite,
newtId: type == "newt" ? updatedNewtId : undefined,
secret: type == "newt" ? updatedNewtSecret : undefined
},
success: true,
error: false,
message: "Site created successfully",
server/routers/site/listSites.ts
+39 -6
View File
@@ -21,6 +21,11 @@ import semver from "semver";
import { z } from "zod";
import { fromError } from "zod-validation-error";
// Stale-while-revalidate: keeps the last successfully fetched version so that
// a transient network failure / timeout does not flip every site back to
// newtUpdateAvailable: false.
let staleNewtVersion: string | null = null;
async function getLatestNewtVersion(): Promise<string | null> {
try {
const cachedVersion = await cache.get<string>("latestNewtVersion");
@@ -29,7 +34,7 @@ async function getLatestNewtVersion(): Promise<string | null> {
}
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 1500); // Reduced timeout to 1.5 seconds
const timeoutId = setTimeout(() => controller.abort(), 1500);
const response = await fetch(
"https://api.github.com/repos/fosrl/newt/tags",
@@ -44,18 +49,46 @@ async function getLatestNewtVersion(): Promise<string | null> {
logger.warn(
`Failed to fetch latest Newt version from GitHub: ${response.status} ${response.statusText}`
);
return null;
return staleNewtVersion;
}
let tags = await response.json();
if (!Array.isArray(tags) || tags.length === 0) {
logger.warn("No tags found for Newt repository");
return null;
return staleNewtVersion;
}
tags = tags.filter((version) => !version.name.includes("rc"));
// Remove release-candidates, then sort descending by semver so that
// duplicate tags (e.g. "1.10.3" and "v1.10.3") and any ordering quirks
// from the GitHub API do not cause an older tag to be selected.
tags = tags.filter((tag: any) => !tag.name.includes("rc"));
tags.sort((a: any, b: any) => {
const va = semver.coerce(a.name);
const vb = semver.coerce(b.name);
if (!va && !vb) return 0;
if (!va) return 1;
if (!vb) return -1;
return semver.rcompare(va, vb);
});
// Deduplicate: keep only the first (highest) entry per normalised version
const seen = new Set<string>();
tags = tags.filter((tag: any) => {
const normalised = semver.coerce(tag.name)?.version;
if (!normalised || seen.has(normalised)) return false;
seen.add(normalised);
return true;
});
if (tags.length === 0) {
logger.warn("No valid semver tags found for Newt repository");
return staleNewtVersion;
}
const latestVersion = tags[0].name;
await cache.set("latestNewtVersion", latestVersion, 3600);
staleNewtVersion = latestVersion;
await cache.set("cache:latestNewtVersion", latestVersion, 3600);
return latestVersion;
} catch (error: any) {
@@ -73,7 +106,7 @@ async function getLatestNewtVersion(): Promise<string | null> {
error.message || error
);
}
return null;
return staleNewtVersion;
}
}
server/routers/site/pickSiteDefaults.ts
+1 -1
View File
@@ -124,7 +124,7 @@ export async function pickSiteDefaults(
return next(
createHttpError(
HttpCode.INTERNAL_SERVER_ERROR,
"No available subnet found"
"No available address"
)
);
}
server/routers/siteResource/updateSiteResource.ts
+4 -4
View File
@@ -618,11 +618,11 @@ export async function handleMessagingForUpdatedSiteResource(
// Only update targets on newt if destination changed
if (destinationChanged || portRangesChanged) {
const oldTarget = generateSubnetProxyTargetV2(
const oldTargets = generateSubnetProxyTargetV2(
existingSiteResource,
mergedAllClients
);
const newTarget = generateSubnetProxyTargetV2(
const newTargets = generateSubnetProxyTargetV2(
updatedSiteResource,
mergedAllClients
);
@@ -630,8 +630,8 @@ export async function handleMessagingForUpdatedSiteResource(
await updateTargets(
newt.newtId,
{
oldTargets: oldTarget ? [oldTarget] : [],
newTargets: newTarget ? [newTarget] : []
oldTargets: oldTargets ? oldTargets : [],
newTargets: newTargets ? newTargets : []
},
newt.version
);
server/routers/target/handleHealthcheckStatusMessage.ts
+11 -1
View File
@@ -77,7 +77,8 @@ export const handleHealthcheckStatusMessage: MessageHandler = async (
const [targetCheck] = await db
.select({
targetId: targets.targetId,
siteId: targets.siteId
siteId: targets.siteId,
hcStatus: targetHealthCheck.hcHealth
})
.from(targets)
.innerJoin(
@@ -85,6 +86,7 @@ export const handleHealthcheckStatusMessage: MessageHandler = async (
eq(targets.resourceId, resources.resourceId)
)
.innerJoin(sites, eq(targets.siteId, sites.siteId))
.innerJoin(targetHealthCheck, eq(targets.targetId, targetHealthCheck.targetId))
.where(
and(
eq(targets.targetId, targetIdNum),
@@ -101,6 +103,14 @@ export const handleHealthcheckStatusMessage: MessageHandler = async (
continue;
}
// check if the status has changed
if (targetCheck.hcStatus === healthStatus.status) {
logger.debug(
`Health status for target ${targetId} is already ${healthStatus.status}, skipping update`
);
continue;
}
// Update the target's health status in the database
await db
.update(targetHealthCheck)
server/routers/user/getUser.ts
+2 -1
View File
@@ -21,7 +21,8 @@ async function queryUser(userId: string) {
serverAdmin: users.serverAdmin,
idpName: idp.name,
idpId: users.idpId,
locale: users.locale
locale: users.locale,
dateCreated: users.dateCreated
})
.from(users)
.leftJoin(idp, eq(users.idpId, idp.idpId))
server/routers/user/inviteUser.ts
+14 -8
View File
@@ -1,7 +1,14 @@
import { Request, Response, NextFunction } from "express";
import { z } from "zod";
import { db } from "@server/db";
import { orgs, roles, userInviteRoles, userInvites, userOrgs, users } from "@server/db";
import {
orgs,
roles,
userInviteRoles,
userInvites,
userOrgs,
users
} from "@server/db";
import { and, eq, inArray } from "drizzle-orm";
import response from "@server/lib/response";
import HttpCode from "@server/types/HttpCode";
@@ -37,8 +44,7 @@ const inviteUserBodySchema = z
regenerate: z.boolean().optional()
})
.refine(
(d) =>
(d.roleIds != null && d.roleIds.length > 0) || d.roleId != null,
(d) => (d.roleIds != null && d.roleIds.length > 0) || d.roleId != null,
{ message: "roleIds or roleId is required", path: ["roleIds"] }
)
.transform((data) => ({
@@ -265,7 +271,7 @@ export async function inviteUser(
)
);
const inviteLink = `${config.getRawConfig().app.dashboard_url}/invite?token=${inviteId}-${token}&email=${encodeURIComponent(email)}`;
const inviteLink = `${config.getRawConfig().app.dashboard_url}/invite?token=${inviteId}-${token}&email=${email}`;
if (doEmail) {
await sendEmail(
@@ -314,12 +320,12 @@ export async function inviteUser(
expiresAt,
tokenHash
});
await trx.insert(userInviteRoles).values(
uniqueRoleIds.map((roleId) => ({ inviteId, roleId }))
);
await trx
.insert(userInviteRoles)
.values(uniqueRoleIds.map((roleId) => ({ inviteId, roleId })));
});
const inviteLink = `${config.getRawConfig().app.dashboard_url}/invite?token=${inviteId}-${token}&email=${encodeURIComponent(email)}`;
const inviteLink = `${config.getRawConfig().app.dashboard_url}/invite?token=${inviteId}-${token}&email=${email}`;
if (doEmail) {
await sendEmail(
server/routers/user/myDevice.ts
+2 -1
View File
@@ -64,7 +64,8 @@ export async function myDevice(
serverAdmin: users.serverAdmin,
idpName: idp.name,
idpId: users.idpId,
locale: users.locale
locale: users.locale,
dateCreated: users.dateCreated
})
.from(users)
.leftJoin(idp, eq(users.idpId, idp.idpId))