Compare commits

...

5 Commits

Author SHA1 Message Date
miloschwartz
6c2c620c99 set cache ttl and default ttl 2026-03-20 17:52:07 -07:00
miloschwartz
f643abf19a dont show create org for oidc users 2026-03-20 16:04:00 -07:00
Owen
7311766512 Fix offline issue 2026-03-20 15:30:41 -07:00
Owen
edcfbd26e4 Merge branch 'dev' of github.com:fosrl/pangolin into dev 2026-03-20 14:31:27 -07:00
Owen
0c4d9ea164 Extend santize into hybrid 2026-03-20 14:31:12 -07:00
11 changed files with 108 additions and 73 deletions

40
server/lib/sanitize.ts Normal file
View File

@@ -0,0 +1,40 @@
/**
* Sanitize a string field before inserting into a database TEXT column.
*
* Two passes are applied:
*
* 1. Lone UTF-16 surrogates JavaScript strings can hold unpaired surrogates
* (e.g. \uD800 without a following \uDC00-\uDFFF codepoint). These are
* valid in JS but cannot be encoded as UTF-8, triggering
* `report_invalid_encoding` in SQLite / Postgres. They are replaced with
* the Unicode replacement character U+FFFD so the data is preserved as a
* visible signal that something was malformed.
*
* 2. Null bytes and C0 control characters SQLite stores TEXT as
* null-terminated C strings, so \x00 in a value causes
* `report_invalid_encoding`. Bots and scanners routinely inject null bytes
* into URLs (e.g. `/path\u0000.jpg`). All C0 control characters in the
* range \x00-\x1F are stripped except for the three that are legitimate in
* text payloads: HT (\x09), LF (\x0A), and CR (\x0D). DEL (\x7F) is also
* stripped.
*/
export function sanitizeString(value: string): string;
export function sanitizeString(
value: string | null | undefined
): string | undefined;
export function sanitizeString(
value: string | null | undefined
): string | undefined {
if (value == null) return undefined;
return (
value
// Replace lone high surrogates (not followed by a low surrogate)
// and lone low surrogates (not preceded by a high surrogate).
.replace(
/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g,
"\uFFFD"
)
// Strip null bytes, C0 control chars (except HT/LF/CR), and DEL.
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "")
);
}

View File

@@ -24,23 +24,31 @@ setInterval(() => {
*/ */
class AdaptiveCache { class AdaptiveCache {
private useRedis(): boolean { private useRedis(): boolean {
return redisManager.isRedisEnabled() && redisManager.getHealthStatus().isHealthy; return (
redisManager.isRedisEnabled() &&
redisManager.getHealthStatus().isHealthy
);
} }
/** /**
* Set a value in the cache * Set a value in the cache
* @param key - Cache key * @param key - Cache key
* @param value - Value to cache (will be JSON stringified for Redis) * @param value - Value to cache (will be JSON stringified for Redis)
* @param ttl - Time to live in seconds (0 = no expiration) * @param ttl - Time to live in seconds (0 = no expiration; omit = 3600s for Redis)
* @returns boolean indicating success * @returns boolean indicating success
*/ */
async set(key: string, value: any, ttl?: number): Promise<boolean> { async set(key: string, value: any, ttl?: number): Promise<boolean> {
const effectiveTtl = ttl === 0 ? undefined : ttl; const effectiveTtl = ttl === 0 ? undefined : ttl;
const redisTtl = ttl === 0 ? undefined : (ttl ?? 3600);
if (this.useRedis()) { if (this.useRedis()) {
try { try {
const serialized = JSON.stringify(value); const serialized = JSON.stringify(value);
const success = await redisManager.set(key, serialized, effectiveTtl); const success = await redisManager.set(
key,
serialized,
redisTtl
);
if (success) { if (success) {
logger.debug(`Set key in Redis: ${key}`); logger.debug(`Set key in Redis: ${key}`);
@@ -48,7 +56,9 @@ class AdaptiveCache {
} }
// Redis failed, fall through to local cache // Redis failed, fall through to local cache
logger.debug(`Redis set failed for key ${key}, falling back to local cache`); logger.debug(
`Redis set failed for key ${key}, falling back to local cache`
);
} catch (error) { } catch (error) {
logger.error(`Redis set error for key ${key}:`, error); logger.error(`Redis set error for key ${key}:`, error);
// Fall through to local cache // Fall through to local cache
@@ -120,9 +130,14 @@ class AdaptiveCache {
} }
// Some Redis deletes failed, fall through to local cache // Some Redis deletes failed, fall through to local cache
logger.debug(`Some Redis deletes failed, falling back to local cache`); logger.debug(
`Some Redis deletes failed, falling back to local cache`
);
} catch (error) { } catch (error) {
logger.error(`Redis del error for keys ${keys.join(", ")}:`, error); logger.error(
`Redis del error for keys ${keys.join(", ")}:`,
error
);
// Fall through to local cache // Fall through to local cache
deletedCount = 0; deletedCount = 0;
} }
@@ -195,7 +210,9 @@ class AdaptiveCache {
*/ */
async flushAll(): Promise<void> { async flushAll(): Promise<void> {
if (this.useRedis()) { if (this.useRedis()) {
logger.warn("Adaptive cache flushAll called - Redis flush not implemented, only local cache will be flushed"); logger.warn(
"Adaptive cache flushAll called - Redis flush not implemented, only local cache will be flushed"
);
} }
localCache.flushAll(); localCache.flushAll();
@@ -239,7 +256,9 @@ class AdaptiveCache {
getTtl(key: string): number { getTtl(key: string): number {
// Note: This only works for local cache, Redis TTL is not supported // Note: This only works for local cache, Redis TTL is not supported
if (this.useRedis()) { if (this.useRedis()) {
logger.warn(`getTtl called for key ${key} but Redis TTL lookup is not implemented`); logger.warn(
`getTtl called for key ${key} but Redis TTL lookup is not implemented`
);
} }
const ttl = localCache.getTtl(key); const ttl = localCache.getTtl(key);
@@ -255,7 +274,9 @@ class AdaptiveCache {
*/ */
keys(): string[] { keys(): string[] {
if (this.useRedis()) { if (this.useRedis()) {
logger.warn("keys() called but Redis keys are not included, only local cache keys returned"); logger.warn(
"keys() called but Redis keys are not included, only local cache keys returned"
);
} }
return localCache.keys(); return localCache.keys();
} }

View File

@@ -81,6 +81,7 @@ import { verifyResourceAccessToken } from "@server/auth/verifyResourceAccessToke
import semver from "semver"; import semver from "semver";
import { maxmindAsnLookup } from "@server/db/maxmindAsn"; import { maxmindAsnLookup } from "@server/db/maxmindAsn";
import { checkOrgAccessPolicy } from "@server/lib/checkOrgAccessPolicy"; import { checkOrgAccessPolicy } from "@server/lib/checkOrgAccessPolicy";
import { sanitizeString } from "@server/lib/sanitize";
// Zod schemas for request validation // Zod schemas for request validation
const getResourceByDomainParamsSchema = z.strictObject({ const getResourceByDomainParamsSchema = z.strictObject({
@@ -1859,24 +1860,24 @@ hybridRouter.post(
}) })
.map((logEntry) => ({ .map((logEntry) => ({
timestamp: logEntry.timestamp, timestamp: logEntry.timestamp,
orgId: logEntry.orgId, orgId: sanitizeString(logEntry.orgId),
actorType: logEntry.actorType, actorType: sanitizeString(logEntry.actorType),
actor: logEntry.actor, actor: sanitizeString(logEntry.actor),
actorId: logEntry.actorId, actorId: sanitizeString(logEntry.actorId),
metadata: logEntry.metadata, metadata: sanitizeString(logEntry.metadata),
action: logEntry.action, action: logEntry.action,
resourceId: logEntry.resourceId, resourceId: logEntry.resourceId,
reason: logEntry.reason, reason: logEntry.reason,
location: logEntry.location, location: sanitizeString(logEntry.location),
// userAgent: data.userAgent, // TODO: add this // userAgent: data.userAgent, // TODO: add this
// headers: data.body.headers, // headers: data.body.headers,
// query: data.body.query, // query: data.body.query,
originalRequestURL: logEntry.originalRequestURL, originalRequestURL: sanitizeString(logEntry.originalRequestURL) ?? "",
scheme: logEntry.scheme, scheme: sanitizeString(logEntry.scheme) ?? "",
host: logEntry.host, host: sanitizeString(logEntry.host) ?? "",
path: logEntry.path, path: sanitizeString(logEntry.path) ?? "",
method: logEntry.method, method: sanitizeString(logEntry.method) ?? "",
ip: logEntry.ip, ip: sanitizeString(logEntry.ip),
tls: logEntry.tls tls: logEntry.tls
})); }));

View File

@@ -38,7 +38,7 @@ export const startRemoteExitNodeOfflineChecker = (): void => {
); );
// Find clients that haven't pinged in the last 2 minutes and mark them as offline // Find clients that haven't pinged in the last 2 minutes and mark them as offline
const newlyOfflineNodes = await db const offlineNodes = await db
.update(exitNodes) .update(exitNodes)
.set({ online: false }) .set({ online: false })
.where( .where(
@@ -53,32 +53,15 @@ export const startRemoteExitNodeOfflineChecker = (): void => {
) )
.returning(); .returning();
// Update the sites to offline if they have not pinged either if (offlineNodes.length > 0) {
const exitNodeIds = newlyOfflineNodes.map( logger.info(
(node) => node.exitNodeId `checkRemoteExitNodeOffline: Marked ${offlineNodes.length} remoteExitNode client(s) offline due to inactivity`
); );
const sitesOnNode = await db for (const offlineClient of offlineNodes) {
.select() logger.debug(
.from(sites) `checkRemoteExitNodeOffline: Client ${offlineClient.exitNodeId} marked offline (lastPing: ${offlineClient.lastPing})`
.where(
and(
eq(sites.online, true),
inArray(sites.exitNodeId, exitNodeIds)
)
); );
// loop through the sites and process their lastBandwidthUpdate as an iso string and if its more than 1 minute old then mark the site offline
for (const site of sitesOnNode) {
if (!site.lastBandwidthUpdate) {
continue;
}
const lastBandwidthUpdate = new Date(site.lastBandwidthUpdate);
if (Date.now() - lastBandwidthUpdate.getTime() > 60 * 1000) {
await db
.update(sites)
.set({ online: false })
.where(eq(sites.siteId, site.siteId));
} }
} }
} catch (error) { } catch (error) {

View File

@@ -5,25 +5,7 @@ import cache from "#dynamic/lib/cache";
import { calculateCutoffTimestamp } from "@server/lib/cleanupLogs"; import { calculateCutoffTimestamp } from "@server/lib/cleanupLogs";
import { stripPortFromHost } from "@server/lib/ip"; import { stripPortFromHost } from "@server/lib/ip";
/** import { sanitizeString } from "@server/lib/sanitize";
* Sanitize a string field by replacing lone UTF-16 surrogates (which cannot
* be encoded as valid UTF-8) with the Unicode replacement character, and
* stripping ASCII control characters that are invalid in most text columns.
*/
function sanitizeString(value: string | undefined | null): string | undefined {
if (value == null) return undefined;
return (
value
// Replace lone high surrogates (not followed by a low surrogate)
// and lone low surrogates (not preceded by a high surrogate)
.replace(
/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g,
"\uFFFD"
)
// Strip C0 control characters except HT (\x09), LF (\x0A), CR (\x0D)
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "")
);
}
/** /**

View File

@@ -70,7 +70,7 @@ async function getLatestOlmVersion(): Promise<string | null> {
tags = tags.filter((version) => !version.name.includes("rc")); tags = tags.filter((version) => !version.name.includes("rc"));
const latestVersion = tags[0].name; const latestVersion = tags[0].name;
olmVersionCache.set("latestOlmVersion", latestVersion); olmVersionCache.set("latestOlmVersion", latestVersion, 3600);
return latestVersion; return latestVersion;
} catch (error: any) { } catch (error: any) {

View File

@@ -71,7 +71,7 @@ async function getLatestOlmVersion(): Promise<string | null> {
tags = tags.filter((version) => !version.name.includes("rc")); tags = tags.filter((version) => !version.name.includes("rc"));
const latestVersion = tags[0].name; const latestVersion = tags[0].name;
olmVersionCache.set("latestOlmVersion", latestVersion); olmVersionCache.set("latestOlmVersion", latestVersion, 3600);
return latestVersion; return latestVersion;
} catch (error: any) { } catch (error: any) {

View File

@@ -6,7 +6,9 @@ import logger from "@server/logger";
/** /**
* Handles disconnecting messages from sites to show disconnected in the ui * Handles disconnecting messages from sites to show disconnected in the ui
*/ */
export const handleNewtDisconnectingMessage: MessageHandler = async (context) => { export const handleNewtDisconnectingMessage: MessageHandler = async (
context
) => {
const { message, client: c, sendToClient } = context; const { message, client: c, sendToClient } = context;
const newt = c as Newt; const newt = c as Newt;
@@ -27,7 +29,7 @@ export const handleNewtDisconnectingMessage: MessageHandler = async (context) =>
.set({ .set({
online: false online: false
}) })
.where(eq(sites.siteId, sites.siteId)); .where(eq(sites.siteId, newt.siteId));
} catch (error) { } catch (error) {
logger.error("Error handling disconnecting message", { error }); logger.error("Error handling disconnecting message", { error });
} }

View File

@@ -55,7 +55,7 @@ async function getLatestNewtVersion(): Promise<string | null> {
tags = tags.filter((version) => !version.name.includes("rc")); tags = tags.filter((version) => !version.name.includes("rc"));
const latestVersion = tags[0].name; const latestVersion = tags[0].name;
await cache.set("latestNewtVersion", latestVersion); await cache.set("latestNewtVersion", latestVersion, 3600);
return latestVersion; return latestVersion;
} catch (error: any) { } catch (error: any) {
@@ -180,7 +180,7 @@ registry.registerPath({
method: "get", method: "get",
path: "/org/{orgId}/sites", path: "/org/{orgId}/sites",
description: "List all sites in an organization", description: "List all sites in an organization",
tags: [OpenAPITags.Site], tags: [OpenAPITags.Org, OpenAPITags.Site],
request: { request: {
params: listSitesParamsSchema, params: listSitesParamsSchema,
query: listSitesSchema query: listSitesSchema

View File

@@ -201,7 +201,7 @@ export async function inviteUser(
); );
} }
await cache.set(email, attempts + 1); await cache.set("regenerateInvite:" + email, attempts + 1, 3600);
const inviteId = existingInvite[0].inviteId; // Retrieve the original inviteId const inviteId = existingInvite[0].inviteId; // Retrieve the original inviteId
const token = generateRandomString( const token = generateRandomString(

View File

@@ -29,6 +29,7 @@ import { usePathname, useRouter } from "next/navigation";
import { useMemo, useState } from "react"; import { useMemo, useState } from "react";
import { useUserContext } from "@app/hooks/useUserContext"; import { useUserContext } from "@app/hooks/useUserContext";
import { useTranslations } from "next-intl"; import { useTranslations } from "next-intl";
import { build } from "@server/build";
interface OrgSelectorProps { interface OrgSelectorProps {
orgId?: string; orgId?: string;
@@ -50,6 +51,11 @@ export function OrgSelector({
const selectedOrg = orgs?.find((org) => org.orgId === orgId); const selectedOrg = orgs?.find((org) => org.orgId === orgId);
let canCreateOrg = !env.flags.disableUserCreateOrg || user.serverAdmin;
if (build === "saas" && user.type !== "internal") {
canCreateOrg = false;
}
const sortedOrgs = useMemo(() => { const sortedOrgs = useMemo(() => {
if (!orgs?.length) return orgs ?? []; if (!orgs?.length) return orgs ?? [];
return [...orgs].sort((a, b) => { return [...orgs].sort((a, b) => {
@@ -161,7 +167,7 @@ export function OrgSelector({
</CommandGroup> </CommandGroup>
</CommandList> </CommandList>
</Command> </Command>
{(!env.flags.disableUserCreateOrg || user.serverAdmin) && ( {canCreateOrg && (
<div className="p-2 border-t border-border"> <div className="p-2 border-t border-border">
<Button <Button
variant="ghost" variant="ghost"