Merge pull request #3267 from Josh-Voyles/memfix1-1.18.4

fix(sqlite): revert no-op finalize wrapper and aggressive PRAGMAs (#2120)

.github/dependabot.yml

@@ -1,42 +1,52 @@
 version: 2
-
 updates:
   - package-ecosystem: "npm"
     directory: "/"
     schedule:
       interval: "daily"
-    open-pull-requests-limit: 1
     groups:
-      npm-dependencies:
-        patterns:
-          - "*"
-
+      dev-patch-updates:
+        dependency-type: "development"
+        update-types:
+          - "patch"
+      dev-minor-updates:
+        dependency-type: "development"
+        update-types:
+          - "minor"
+      prod-patch-updates:
+        dependency-type: "production"
+        update-types:
+          - "patch"
+      prod-minor-updates:
+        dependency-type: "production"
+        update-types:
+          - "minor"
+          
   - package-ecosystem: "docker"
     directory: "/"
     schedule:
       interval: "daily"
-    open-pull-requests-limit: 1
     groups:
-      docker-dependencies:
-        patterns:
-          - "*"
+      patch-updates:
+        update-types:
+          - "patch"
+      minor-updates:
+        update-types:
+          - "minor"
 
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "weekly"
-    open-pull-requests-limit: 1
-    groups:
-      github-actions-dependencies:
-        patterns:
-          - "*"
 
   - package-ecosystem: "gomod"
     directory: "/install"
     schedule:
       interval: "daily"
-    open-pull-requests-limit: 1
     groups:
-      go-install-dependencies:
-        patterns:
-          - "*"
+      patch-updates:
+        update-types:
+          - "patch"
+      minor-updates:
+        update-types:
+          - "minor"

install/go.mod

@@ -5,7 +5,7 @@ go 1.25.0
 require (
 	github.com/charmbracelet/huh v1.0.0
 	github.com/charmbracelet/lipgloss v1.1.0
-	golang.org/x/term v0.44.0
+	golang.org/x/term v0.43.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -33,6 +33,6 @@ require (
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	golang.org/x/sync v0.15.0 // indirect
-	golang.org/x/sys v0.46.0 // indirect
+	golang.org/x/sys v0.44.0 // indirect
 	golang.org/x/text v0.23.0 // indirect
 )

install/go.sum

@@ -69,10 +69,10 @@ golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
 golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
-golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
-golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
+golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
+golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4=
+golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk=
 golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
 golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

server/db/sqlite/driver.ts

@@ -1,6 +1,5 @@
 import { drizzle as DrizzleSqlite } from "drizzle-orm/better-sqlite3";
 import Database from "better-sqlite3";
-import type BetterSqlite3 from "better-sqlite3";
 import * as schema from "./schema/schema";
 import path from "path";
 import fs from "fs";
@@ -12,68 +11,31 @@ export const exists = checkFileExists(location);
 
 bootstrapVolume();
 
-/**
- * Wraps better-sqlite3 Statement to call `finalize()` immediately after
- * execution, freeing native sqlite3_stmt memory deterministically instead
- * of waiting for GC. Fixes steady off-heap growth under load (#2120).
- * WARNING: Finalizes after first execution — incompatible with drizzle's
- * reusable .prepare() builders. No such usage exists in this codebase.
- */
-function autoFinalizeStatement(
-    stmt: BetterSqlite3.Statement
-): BetterSqlite3.Statement {
-    const wrapExec = <T extends (...args: any[]) => any>(fn: T): T => {
-        return function (this: any, ...args: any[]) {
-            try {
-                return fn.apply(this, args);
-            } finally {
-                try {
-                    // finalize() exists on the native Statement at runtime but
-                    // is missing from @types/better-sqlite3.
-                    (stmt as any).finalize();
-                } catch {
-                    // Already finalized — harmless
-                }
-            }
-        } as unknown as T;
-    };
-
-    stmt.run = wrapExec(stmt.run);
-    stmt.get = wrapExec(stmt.get);
-    stmt.all = wrapExec(stmt.all);
-
-    return stmt;
-}
-
 function createDb() {
     const sqlite = new Database(location);
 
     if (process.env.ENABLE_SQLITE_WAL_MODE == "true") {
         // Enable WAL mode — allows concurrent readers + single writer, preventing
         // contention across subsystems (verifySession, Traefik, audit, ping).
+        // NOTE: journal_mode persists in the DB file once set; unsetting this
+        // env var does NOT revert an existing WAL database.
         sqlite.pragma("journal_mode = WAL");
         // NORMAL sync mode: safe with WAL, reduces write lock hold time.
         sqlite.pragma("synchronous = NORMAL");
     }
 
-    // Wait up to 5s on SQLITE_BUSY instead of failing — prevents audit log
-    // retry loops that accumulate memory.
-    sqlite.pragma("busy_timeout = 5000");
+    // No busy_timeout pragma: better-sqlite3 already arms
+    // sqlite3_busy_timeout(db, 5000) via its default `timeout` option
+    // (lib/database.js), so an explicit pragma is redundant.
 
-    // 64 MB page cache (default 2 MB) — reduces I/O round-trips on large
-    // TraefikConfigManager JOINs that block the event loop.
-    sqlite.pragma("cache_size = -65536");
+    // Intentionally NOT setting cache_size or mmap_size: a large page cache plus
+    // a multi-hundred-MB mmap region inflate RSS and cause page-cache thrashing
+    // on small (~1 GB) instances. Leave SQLite on its conservative defaults.
 
-    // 256 MB memory-mapped I/O — OS serves reads from page cache directly,
-    // reducing event-loop blocking.
-    sqlite.pragma("mmap_size = 268435456");
-
-    // Wrap prepare() so every drizzle-orm statement is auto-finalized after
-    // first use, preventing sqlite3_stmt accumulation between GC cycles.
-    const originalPrepare = sqlite.prepare.bind(sqlite);
-    (sqlite as any).prepare = function autoFinalizePrepare(source: string) {
-        return autoFinalizeStatement(originalPrepare(source));
-    };
+    // Intentionally NOT wrapping prepare()/statements: better-sqlite3 finalizes
+    // sqlite3_stmt in the Statement destructor at GC, and drizzle-orm prepares a
+    // fresh statement per query (no statement cache), so statements cannot
+    // accumulate. better-sqlite3 11.x exposes no Statement.finalize() at all.
 
     return DrizzleSqlite(sqlite, {
         schema

server/lib/billing/usageService.ts

@@ -12,7 +12,7 @@ import {
 import { FeatureId, getFeatureMeterId } from "./features";
 import logger from "@server/logger";
 import { build } from "@server/build";
-import cache from "#dynamic/lib/cache";
+import { regionalCache as cache } from "#dynamic/lib/cache";
 
 export function noop() {
     if (build !== "saas") {
@@ -22,7 +22,6 @@ export function noop() {
 }
 
 export class UsageService {
-
     constructor() {
         if (noop()) {
             return;
@@ -57,7 +56,10 @@ export class UsageService {
             try {
                 let usage;
                 if (transaction) {
-                    const orgIdToUse = await this.getBillingOrg(orgId, transaction);
+                    const orgIdToUse = await this.getBillingOrg(
+                        orgId,
+                        transaction
+                    );
                     usage = await this.internalAddUsage(
                         orgIdToUse,
                         featureId,

server/lib/blueprints/applyBlueprint.ts

@@ -48,18 +48,18 @@ export async function applyBlueprint({
     name,
     source = "API"
 }: ApplyBlueprintArgs): Promise<Blueprint> {
-    // Validate the input data
-    const validationResult = ConfigSchema.safeParse(configData);
-    if (!validationResult.success) {
-        throw new Error(fromError(validationResult.error).toString());
-    }
-
-    const config: Config = validationResult.data;
     let blueprintSucceeded: boolean = false;
-    let blueprintMessage: string;
+    let blueprintMessage = "";
     let error: any | null = null;
 
     try {
+        const validationResult = ConfigSchema.safeParse(configData);
+        if (!validationResult.success) {
+            throw new Error(fromError(validationResult.error).toString());
+        }
+
+        const config: Config = validationResult.data;
+
         let proxyResourcesResults: PublicResourcesResults = [];
         let clientResourcesResults: ClientResourcesResults = [];
         await db.transaction(async (trx) => {

server/lib/validators.test.ts

@@ -1,4 +1,7 @@
-import { isValidUrlGlobPattern } from "./validators";
+import {
+    getResourceRuleValueValidationError,
+    isValidUrlGlobPattern
+} from "./validators";
 import { assertEquals } from "@test/assert";
 
 function runTests() {
@@ -236,6 +239,43 @@ function runTests() {
         "Path with isolated percent sign should be invalid"
     );
 
+    // ASN validation tests
+    assertEquals(
+        getResourceRuleValueValidationError("ASN", "AS15169"),
+        null,
+        "Standard ASN should be valid"
+    );
+    assertEquals(
+        getResourceRuleValueValidationError("ASN", "  As15169  "),
+        null,
+        "Standard ASN should be valid with mixed case and whitespace"
+    );
+    assertEquals(
+        getResourceRuleValueValidationError("ASN", "ALL"),
+        null,
+        "ALL ASN selector should be valid"
+    );
+    assertEquals(
+        getResourceRuleValueValidationError("ASN", " all "),
+        null,
+        "ALL ASN selector should be valid with mixed case and whitespace"
+    );
+    assertEquals(
+        getResourceRuleValueValidationError("ASN", "AS0"),
+        null,
+        "AS0 alias should be valid"
+    );
+    assertEquals(
+        getResourceRuleValueValidationError("ASN", " as0 "),
+        null,
+        "AS0 alias should be valid with mixed case and whitespace"
+    );
+    assertEquals(
+        getResourceRuleValueValidationError("ASN", "not-an-asn"),
+        "Invalid ASN provided",
+        "Invalid ASN should return an error"
+    );
+
     console.log("All tests passed!");
 }
 

server/lib/validators.ts

@@ -100,7 +100,10 @@ export function getResourceRuleValueValidationError(
                 ? null
                 : "Invalid country code provided";
         case "ASN":
-            return /^AS\d+$/i.test(value.trim())
+            const normalizedValue = value.trim().toUpperCase();
+            return /^AS\d+$/.test(normalizedValue) ||
+                normalizedValue === "ALL" ||
+                normalizedValue === "AS0"
                 ? null
                 : "Invalid ASN provided";
         default:

server/private/lib/certificates.ts

@@ -17,7 +17,7 @@ import { certificates, db } from "@server/db";
 import { and, eq, isNotNull, or, inArray, sql } from "drizzle-orm";
 import { decrypt } from "@server/lib/crypto";
 import logger from "@server/logger";
-import cache from "#private/lib/cache";
+import { regionalCache as cache } from "#private/lib/cache";
 import { build } from "@server/build";
 
 // Define the return type for clarity and type safety

server/private/routers/remoteExitNode/listRemoteExitNodes.ts

@@ -22,7 +22,7 @@ import createHttpError from "http-errors";
 import logger from "@server/logger";
 import { fromError } from "zod-validation-error";
 import { ListRemoteExitNodesResponse } from "@server/routers/remoteExitNode/types";
-import cache from "#private/lib/cache";
+import { regionalCache as cache } from "#private/lib/cache";
 import semver from "semver";
 
 let stalePangolinNodeVersion: string | null = null;

server/routers/newt/getNewtVersion.ts

@@ -10,7 +10,7 @@ import { verifyPassword } from "@server/auth/password";
 import response from "@server/lib/response";
 import HttpCode from "@server/types/HttpCode";
 import logger from "@server/logger";
-import cache from "#dynamic/lib/cache";
+import { regionalCache as cache } from "#dynamic/lib/cache";
 import config from "@server/lib/config";
 
 // Stale-while-revalidate in-memory fallback for the releases API.

server/routers/newt/handleSocketMessages.ts

@@ -2,7 +2,7 @@ import { MessageHandler } from "@server/routers/ws";
 import logger from "@server/logger";
 import { Newt } from "@server/db";
 import { applyNewtDockerBlueprint } from "@server/lib/blueprints/applyNewtDockerBlueprint";
-import cache from "#dynamic/lib/cache";
+import cache from "#dynamic/lib/cache"; // not using regional here because we dont know where the site is
 
 export const handleDockerStatusMessage: MessageHandler = async (context) => {
     const { message, client, sendToClient } = context;

server/routers/olm/handleOlmRegisterMessage.ts

@@ -20,7 +20,7 @@ import { handleFingerprintInsertion } from "./fingerprintingUtils";
 import { build } from "@server/build";
 import { canCompress } from "@server/lib/clientVersionChecks";
 import config from "@server/lib/config";
-import cache from "#dynamic/lib/cache";
+import cache from "#dynamic/lib/cache"; // not using regional here because we need this in the register message handler before we know where the client is
 
 const HOLEPUNCH_STALE_CHAIN_THRESHOLD = 18;
 const HOLEPUNCH_STALE_CHAIN_TTL_SECONDS = 1800;

server/routers/resource/listUserResourceAliases.ts

@@ -15,8 +15,7 @@ import logger from "@server/logger";
 import { z } from "zod";
 import { fromZodError } from "zod-validation-error";
 import type { PaginatedResponse } from "@server/types/Pagination";
-import { OpenAPITags, registry } from "@server/openApi";
-import { localCache } from "#dynamic/lib/cache";
+import { regionalCache as cache } from "#dynamic/lib/cache";
 
 const USER_RESOURCE_ALIASES_CACHE_TTL_SEC = 60;
 
@@ -153,7 +152,7 @@ export async function listUserResourceAliases(
             pageSize
         );
         const cachedData: ListUserResourceAliasesResponse | undefined =
-            localCache.get(cacheKey);
+            await cache.get(cacheKey);
 
         if (cachedData) {
             return response<ListUserResourceAliasesResponse>(res, {
@@ -211,7 +210,11 @@ export async function listUserResourceAliases(
                     page
                 }
             };
-            localCache.set(cacheKey, data, USER_RESOURCE_ALIASES_CACHE_TTL_SEC);
+            await cache.set(
+                cacheKey,
+                data,
+                USER_RESOURCE_ALIASES_CACHE_TTL_SEC
+            );
             return response<ListUserResourceAliasesResponse>(res, {
                 data,
                 success: true,
@@ -256,7 +259,7 @@ export async function listUserResourceAliases(
                 page
             }
         };
-        localCache.set(cacheKey, data, USER_RESOURCE_ALIASES_CACHE_TTL_SEC);
+        await cache.set(cacheKey, data, USER_RESOURCE_ALIASES_CACHE_TTL_SEC);
 
         return response<ListUserResourceAliasesResponse>(res, {
             data,

server/routers/site/listSites.ts

@@ -14,7 +14,7 @@ import {
     siteLabels,
     type Label
 } from "@server/db";
-import cache from "#dynamic/lib/cache";
+import { regionalCache as cache } from "#dynamic/lib/cache";
 import response from "@server/lib/response";
 import logger from "@server/logger";
 import { OpenAPITags, registry } from "@server/openApi";

src/components/newt-install-commands.tsx

@@ -139,7 +139,6 @@ Restart=always
 RestartSec=2
 UMask=0077
 
-NoNewPrivileges=true
 PrivateTmp=true
 
 [Install]

src/components/resource-policy/policy-access-rule-validation.ts

@@ -83,9 +83,19 @@ export function createPolicyRuleValueSchema(t: TranslateFn, match: string) {
                 { message: t("rulesErrorInvalidCountryDescription") }
             );
         case "ASN":
-            return required.refine((value) => /^AS\d+$/i.test(value.trim()), {
-                message: t("rulesErrorInvalidAsnDescription")
-            });
+            return required.refine(
+                (value) => {
+                    const normalizedValue = value.trim().toUpperCase();
+                    return (
+                        /^AS\d+$/.test(normalizedValue) ||
+                        normalizedValue === "ALL" ||
+                        normalizedValue === "AS0"
+                    );
+                },
+                {
+                    message: t("rulesErrorInvalidAsnDescription")
+                }
+            );
         default:
             return required;
     }