From d6abe83fdcabc1c22ddf9c211cb82bf33815413c Mon Sep 17 00:00:00 2001 From: Josh Voyles <120749218+Josh-Voyles@users.noreply.github.com> Date: Sat, 2 May 2026 07:37:18 -0400 Subject: [PATCH 1/5] fix: memory improvements - SQLite: enable WAL mode and PRAGMA performance settings - ws.ts (public + private): fix clientConfigVersions memory leak - internal server: add rate limiting and request timeouts - audit log: fix flush re-queue feedback loop - memory: add monitoring instrumentation - security: remove debug log of full request body --- server/cleanup.ts | 2 ++ server/db/sqlite/driver.ts | 30 ++++++++++++++++++++++++ server/index.ts | 26 ++++++++++++++++++++ server/internalServer.ts | 21 +++++++++++++++++ server/private/routers/ws/ws.ts | 13 ++++++++++ server/routers/badger/exchangeSession.ts | 2 +- server/routers/badger/logRequestAudit.ts | 16 ++++++------- server/routers/badger/verifySession.ts | 2 +- server/routers/ws/ws.ts | 10 ++++++++ 9 files changed, 112 insertions(+), 10 deletions(-) diff --git a/server/cleanup.ts b/server/cleanup.ts index 10e9f4cc3..a1c7fec59 100644 --- a/server/cleanup.ts +++ b/server/cleanup.ts @@ -3,9 +3,11 @@ import { flushConnectionLogToDb } from "#dynamic/routers/newt"; import { flushSiteBandwidthToDb } from "@server/routers/gerbil/receiveBandwidth"; import { stopPingAccumulator } from "@server/routers/newt/pingAccumulator"; import { cleanup as wsCleanup } from "#dynamic/routers/ws"; +import { shutdownAuditLogger } from "@server/routers/badger/logRequestAudit"; async function cleanup() { await stopPingAccumulator(); + await shutdownAuditLogger(); await flushBandwidthToDb(); await flushConnectionLogToDb(); await flushSiteBandwidthToDb(); diff --git a/server/db/sqlite/driver.ts b/server/db/sqlite/driver.ts index 832ff16f9..1b0b0b526 100644 --- a/server/db/sqlite/driver.ts +++ b/server/db/sqlite/driver.ts @@ -13,6 +13,36 @@ bootstrapVolume(); function createDb() { const sqlite = new Database(location); + + // Enable WAL mode for dramatically better concurrent read/write + // performance. Without this, readers block writers and vice versa, + // causing severe contention when multiple subsystems (verifySession, + // TraefikConfigManager, audit log flushes, ping flushes) all share + // this single connection. WAL mode allows concurrent readers with a + // single writer, which is the typical access pattern. + sqlite.pragma("journal_mode = WAL"); + + // Wait up to 5 seconds when the database is locked instead of + // failing immediately with SQLITE_BUSY. This prevents transient + // write failures from causing audit log buffer re-queues and retry + // loops that accumulate memory. + sqlite.pragma("busy_timeout = 5000"); + + // NORMAL synchronous mode is safe with WAL and significantly reduces + // the time each write holds the database lock. + sqlite.pragma("synchronous = NORMAL"); + + // Increase the page cache to 64 MB (negative value = KB). The + // default (2 MB) causes frequent I/O round-trips on the large JOIN + // queries used by TraefikConfigManager, which block the event loop + // for longer than necessary. + sqlite.pragma("cache_size = -65536"); + + // Enable memory-mapped I/O for reads (256 MB). This allows the OS + // to serve read queries from the page cache without going through + // SQLite's own cache, reducing event-loop blocking time. + sqlite.pragma("mmap_size = 268435456"); + return DrizzleSqlite(sqlite, { schema }); diff --git a/server/index.ts b/server/index.ts index e3a6ba049..f3274d2fb 100644 --- a/server/index.ts +++ b/server/index.ts @@ -24,6 +24,29 @@ import license from "#dynamic/license/license"; import { initLogCleanupInterval } from "@server/lib/cleanupLogs"; import { initAcmeCertSync } from "#dynamic/lib/acmeCertSync"; import { fetchServerIp } from "@server/lib/serverIpService"; +import logger from "@server/logger"; + +/** + * Periodic memory usage logging for monitoring and leak detection. + * Logs heap usage, external (native) memory, and RSS every 60 seconds. + * This is lightweight (single process.memoryUsage() call) and provides + * the data needed to detect slow memory growth over hours/days. + */ +function startMemoryMonitor(): void { + const INTERVAL_MS = 60_000; // every 60 seconds + const timer = setInterval(() => { + const mem = process.memoryUsage(); + logger.info( + `Memory usage - ` + + `heapUsed: ${(mem.heapUsed / 1024 / 1024).toFixed(1)}MB, ` + + `heapTotal: ${(mem.heapTotal / 1024 / 1024).toFixed(1)}MB, ` + + `rss: ${(mem.rss / 1024 / 1024).toFixed(1)}MB, ` + + `external: ${(mem.external / 1024 / 1024).toFixed(1)}MB, ` + + `arrayBuffers: ${(mem.arrayBuffers / 1024 / 1024).toFixed(1)}MB` + ); + }, INTERVAL_MS); + timer.unref(); +} async function startServers() { await setHostMeta(); @@ -42,6 +65,9 @@ async function startServers() { initLogCleanupInterval(); initAcmeCertSync(); + // Start memory monitoring for leak detection + startMemoryMonitor(); + // Start all servers const apiServer = createApiServer(); const internalServer = createInternalServer(); diff --git a/server/internalServer.ts b/server/internalServer.ts index 83872e7f9..6eb207121 100644 --- a/server/internalServer.ts +++ b/server/internalServer.ts @@ -10,6 +10,8 @@ import { } from "@server/middlewares"; import { internalRouter } from "#dynamic/routers/internal"; import { stripDuplicateSesions } from "./middlewares/stripDuplicateSessions"; +import { requestTimeoutMiddleware } from "./middlewares/requestTimeout"; +import rateLimit from "express-rate-limit"; const internalPort = config.getRawConfig().server.internal_port; @@ -27,6 +29,25 @@ export function createInternalServer() { internalServer.use(cookieParser()); internalServer.use(express.json()); + // Prevent requests from hanging indefinitely. Without this, if a + // database query blocks (especially on SQLite), pending requests + // accumulate in memory with no upper bound on lifetime. + internalServer.use(requestTimeoutMiddleware(30000)); // 30 second timeout + + // Rate-limit the internal verify-session endpoint. This server + // handles forward-auth requests from Traefik/Badger. Under heavy + // monitoring (e.g. Uptime Kuma), requests can arrive faster than + // SQLite can serve them, causing unbounded request queuing and + // memory growth. + internalServer.use( + rateLimit({ + windowMs: 60 * 1000, // 1 minute window + max: 1000, // generous limit: ~17 req/s + standardHeaders: true, + legacyHeaders: false + }) + ); + const prefix = `/api/v1`; internalServer.use(prefix, internalRouter); diff --git a/server/private/routers/ws/ws.ts b/server/private/routers/ws/ws.ts index 0970735e0..83bae0162 100644 --- a/server/private/routers/ws/ws.ts +++ b/server/private/routers/ws/ws.ts @@ -406,6 +406,10 @@ const removeClient = async ( const updatedClients = existingClients.filter((client) => client !== ws); if (updatedClients.length === 0) { connectedClients.delete(mapKey); + // Clean up config version tracking to prevent unbounded memory + // growth. Without this, every unique clientId that ever connects + // leaves a permanent entry in clientConfigVersions. + clientConfigVersions.delete(clientId); if (redisManager.isRedisEnabled()) { try { @@ -413,6 +417,9 @@ const removeClient = async ( await redisManager.del( getNodeConnectionsKey(NODE_ID, clientId) ); + // Also clean up the Redis config version key so it doesn't + // accumulate indefinitely in Redis either. + await redisManager.del(getConfigVersionKey(clientId)); } catch (error) { logger.error( "Failed to remove client from Redis tracking (cleanup will occur on recovery):", @@ -1097,6 +1104,12 @@ const disconnectClient = async (clientId: string): Promise => { } }); + // Eagerly clean up tracking maps. The close event handlers will also + // call removeClient, but if the socket is already in CLOSING state + // the close event may never fire, leaving zombie entries. + connectedClients.delete(mapKey); + clientConfigVersions.delete(clientId); + return true; }; diff --git a/server/routers/badger/exchangeSession.ts b/server/routers/badger/exchangeSession.ts index 08987961d..1fc52cea7 100644 --- a/server/routers/badger/exchangeSession.ts +++ b/server/routers/badger/exchangeSession.ts @@ -41,7 +41,7 @@ export async function exchangeSession( res: Response, next: NextFunction ): Promise { - logger.debug("Exchange session: Badger sent", req.body); + logger.debug("Exchange session: Badger request received"); const parsedBody = exchangeSessionBodySchema.safeParse(req.body); diff --git a/server/routers/badger/logRequestAudit.ts b/server/routers/badger/logRequestAudit.ts index 884fb7ae4..39f0eb3ca 100644 --- a/server/routers/badger/logRequestAudit.ts +++ b/server/routers/badger/logRequestAudit.ts @@ -84,14 +84,14 @@ async function flushAuditLogs() { logger.debug(`Flushed ${logsToWrite.length} audit logs to database`); } catch (error) { logger.error("Error flushing audit logs:", error); - // On transaction error, put logs back at the front of the buffer to retry - // but only if buffer isn't too large - if (auditLogBuffer.length < MAX_BUFFER_SIZE - logsToWrite.length) { - auditLogBuffer.unshift(...logsToWrite); - logger.info(`Re-queued ${logsToWrite.length} audit logs for retry`); - } else { - logger.error(`Buffer full, dropped ${logsToWrite.length} audit logs`); - } + // On transaction error, drop the logs rather than re-queuing them. + // The previous re-queue approach created a positive feedback loop: + // failed flush → re-queue → larger next flush → longer DB lock → + // higher chance of next failure → repeat. This caused unbounded + // memory growth on SQLite where write contention is common. + // Audit logs are best-effort telemetry — losing a batch on error + // is acceptable; leaking memory until the process crashes is not. + logger.warn(`Dropped ${logsToWrite.length} audit logs after flush failure`); } finally { isFlushInProgress = false; // If buffer filled up while we were flushing, flush again diff --git a/server/routers/badger/verifySession.ts b/server/routers/badger/verifySession.ts index e2e5f6766..b80a1ada0 100644 --- a/server/routers/badger/verifySession.ts +++ b/server/routers/badger/verifySession.ts @@ -80,7 +80,7 @@ export async function verifyResourceSession( res: Response, next: NextFunction ): Promise { - logger.debug("Verify session: Badger sent", req.body); // remove when done testing + logger.debug("Verify session: Badger request received"); const parsedBody = verifyResourceSessionSchema.safeParse(req.body); diff --git a/server/routers/ws/ws.ts b/server/routers/ws/ws.ts index 6e6312715..51274cc20 100644 --- a/server/routers/ws/ws.ts +++ b/server/routers/ws/ws.ts @@ -80,6 +80,10 @@ const removeClient = async ( const updatedClients = existingClients.filter((client) => client !== ws); if (updatedClients.length === 0) { connectedClients.delete(mapKey); + // Clean up config version tracking to prevent unbounded memory + // growth. Without this, every unique clientId that ever connects + // leaves a permanent entry in clientConfigVersions. + clientConfigVersions.delete(clientId); logger.info( `All connections removed for ${clientType.toUpperCase()} ID: ${clientId}` @@ -507,6 +511,12 @@ const disconnectClient = async (clientId: string): Promise => { } }); + // Eagerly clean up tracking maps. The close event handlers will also + // call removeClient, but if the socket is already in CLOSING state + // the close event may never fire, leaving zombie entries. + connectedClients.delete(mapKey); + clientConfigVersions.delete(clientId); + return true; }; From 2c85bcd06b3f9caf44ddf1e03e3a0d30f8d57d24 Mon Sep 17 00:00:00 2001 From: Josh Voyles <120749218+Josh-Voyles@users.noreply.github.com> Date: Sat, 2 May 2026 15:50:54 -0400 Subject: [PATCH 2/5] fix(db): deterministically finalize prepared statements after execution Wrap Statement .all()/.get()/.run() via autoFinalizeStatement() with try/finally calling stmt.finalize() post-execution, releasing native sqlite3_stmt memory immediately instead of waiting for GC. Safe because: - Drizzle one-time queries invoke each statement once only - Drizzle does not access statement after .all()/.get()/.run() returns - Migration scripts use isolated new Database() instances (unpatched) - No app code holds persistent .prepare() refs on main db --- package-lock.json | 59 ++++---------------------------------- server/db/sqlite/driver.ts | 50 ++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 53 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8c241554a..1676e6f25 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1058,7 +1058,6 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -2354,7 +2353,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2377,7 +2375,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2400,7 +2397,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2417,7 +2413,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2434,7 +2429,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2451,7 +2445,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2468,7 +2461,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2485,7 +2477,6 @@ "cpu": [ "s390x" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2502,7 +2493,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2519,7 +2509,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2536,7 +2525,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2553,7 +2541,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2576,7 +2563,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2599,7 +2585,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2622,7 +2607,6 @@ "cpu": [ "s390x" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2645,7 +2629,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2668,7 +2651,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2691,7 +2673,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2714,7 +2695,6 @@ "cpu": [ "wasm32" ], - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", "optional": true, "dependencies": { @@ -2734,7 +2714,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -2754,7 +2733,6 @@ "cpu": [ "ia32" ], - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -2774,7 +2752,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -3034,7 +3011,6 @@ "integrity": "sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": "^14.21.3 || >=16" }, @@ -6981,7 +6957,6 @@ "resolved": "https://registry.npmjs.org/@react-email/text/-/text-0.1.6.tgz", "integrity": "sha512-TYqkioRS45wTR5il3dYk/SbUjjEdhSwh9BtRNB99qNH1pXAwA45H7rAuxehiu8iJQJH0IyIr+6n62gBz9ezmsw==", "license": "MIT", - "peer": true, "engines": { "node": ">=20.0.0" }, @@ -8442,7 +8417,6 @@ "version": "5.90.21", "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.90.21.tgz", "integrity": "sha512-0Lu6y5t+tvlTJMTO7oh5NSpJfpg/5D41LlThfepTixPYkJ0sE2Jj0m0f6yYqujBwIXlId87e234+MxG3D3g7kg==", - "peer": true, "dependencies": { "@tanstack/query-core": "5.90.20" }, @@ -8558,7 +8532,6 @@ "integrity": "sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "@types/node": "*" } @@ -8906,7 +8879,6 @@ "integrity": "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^5.0.0", @@ -9002,7 +8974,6 @@ "integrity": "sha512-oX8xrhvpiyRCQkG1MFchB09f+cXftgIXb3a7UUa4Y3wpmZPw5tyZGTLWhlESOLq1Rq6oDlc8npVU2/9xiCuXMA==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~7.18.0" } @@ -9030,7 +9001,6 @@ "integrity": "sha512-gT+oueVQkqnj6ajGJXblFR4iavIXWsGAFCk3dP4Kki5+a9R4NMt0JARdk6s8cUKcfUoqP5dAtDSLU8xYUTFV+Q==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "@types/node": "*", "pg-protocol": "*", @@ -9056,7 +9026,6 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", "devOptional": true, - "peer": true, "dependencies": { "csstype": "^3.2.2" } @@ -9067,7 +9036,6 @@ "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", "devOptional": true, "license": "MIT", - "peer": true, "peerDependencies": { "@types/react": "^19.2.0" } @@ -9154,7 +9122,8 @@ "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", "license": "MIT", - "optional": true + "optional": true, + "peer": true }, "node_modules/@types/ws": { "version": "8.18.1", @@ -9228,7 +9197,6 @@ "integrity": "sha512-klQbnPAAiGYFyI02+znpBRLyjL4/BrBd0nyWkdC0s/6xFLkXYQ8OoRrSkqacS1ddVxf/LDyODIKbQ5TgKAf/Fg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.56.1", "@typescript-eslint/types": "8.56.1", @@ -9702,7 +9670,6 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -10152,7 +10119,6 @@ "integrity": "sha512-Ixm8tFfoKKIPYdCCKYTsqv+Fd4IJ0DQqMyEimo+pxUOMUR9cVPlwTrFt9Avu+3cb6Zp3mAzl+t1MrG2fxxKsxw==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/types": "^7.26.0" } @@ -10224,7 +10190,6 @@ "integrity": "sha512-Ba0KR+Fzxh2jDRhdg6TSH0SJGzb8C0aBY4hR8w8madIdIzzC6Y1+kx5qR6eS1Z+Gy20h6ZU28aeyg0z1VIrShQ==", "hasInstallScript": true, "license": "MIT", - "peer": true, "dependencies": { "bindings": "^1.5.0", "prebuild-install": "^7.1.1" @@ -10353,7 +10318,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -11260,7 +11224,6 @@ "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "license": "ISC", - "peer": true, "engines": { "node": ">=12" } @@ -11701,6 +11664,7 @@ "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.3.2.tgz", "integrity": "sha512-6obghkliLdmKa56xdbLOpUZ43pAR6xFy1uOrxBaIDjT+yaRuuybLjGS9eVBoSR/UPU5fq3OXClEHLJNGvbxKpQ==", "license": "(MPL-2.0 OR Apache-2.0)", + "peer": true, "engines": { "node": ">=20" }, @@ -12335,7 +12299,6 @@ "dev": true, "hasInstallScript": true, "license": "MIT", - "peer": true, "bin": { "esbuild": "bin/esbuild" }, @@ -12421,7 +12384,6 @@ "integrity": "sha512-COV33RzXZkqhG9P2rZCFl9ZmJ7WL+gQSCRzE7RhkbclbQPtLAWReL7ysA0Sh4c8Im2U9ynybdR56PV0XcKvqaQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", @@ -12558,7 +12520,6 @@ "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -12952,7 +12913,6 @@ "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", - "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -15370,6 +15330,7 @@ "resolved": "https://registry.npmjs.org/monaco-editor/-/monaco-editor-0.55.1.tgz", "integrity": "sha512-jz4x+TJNFHwHtwuV9vA9rMujcZRb0CEilTEwG2rRSpe/A7Jdkuj8xPKttCgOh+v/lkHy7HsZ64oj+q3xoAFl9A==", "license": "MIT", + "peer": true, "dependencies": { "dompurify": "3.2.7", "marked": "14.0.0" @@ -15380,6 +15341,7 @@ "resolved": "https://registry.npmjs.org/marked/-/marked-14.0.0.tgz", "integrity": "sha512-uIj4+faQ+MgHgwUW1l2PsPglZLOLOT1uErt06dAPtx2kjteLAkbsd/0FiYg/MGS+i7ZKLb7w2WClxHkzOOuryQ==", "license": "MIT", + "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -15468,7 +15430,6 @@ "resolved": "https://registry.npmjs.org/next/-/next-15.5.15.tgz", "integrity": "sha512-VSqCrJwtLVGwAVE0Sb/yikrQfkwkZW9p+lL/J4+xe+G3ZA+QnWPqgcfH1tDUEuk9y+pthzzVFp4L/U8JerMfMQ==", "license": "MIT", - "peer": true, "dependencies": { "@next/env": "15.5.15", "@swc/helpers": "0.5.15", @@ -16428,7 +16389,6 @@ "resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz", "integrity": "sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==", "license": "MIT", - "peer": true, "dependencies": { "pg-connection-string": "^2.12.0", "pg-pool": "^3.13.0", @@ -16936,7 +16896,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.4.tgz", "integrity": "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ==", "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -16968,7 +16927,6 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.4.tgz", "integrity": "sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ==", "license": "MIT", - "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -17261,7 +17219,6 @@ "resolved": "https://registry.npmjs.org/react-hook-form/-/react-hook-form-7.71.2.tgz", "integrity": "sha512-1CHvcDYzuRUNOflt4MOq3ZM46AronNJtQ1S7tnX6YN4y72qhgiUItpacZUAQ0TyWYci3yz1X+rXaSxiuEm86PA==", "license": "MIT", - "peer": true, "engines": { "node": ">=18.0.0" }, @@ -18723,8 +18680,7 @@ "version": "4.2.2", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.2.2.tgz", "integrity": "sha512-KWBIxs1Xb6NoLdMVqhbhgwZf2PGBpPEiwOqgI4pFIYbNTfBXiKYyWoTsXgBQ9WFg/OlhnvHaY+AEpW7wSmFo2Q==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/tapable": { "version": "2.3.2", @@ -19199,7 +19155,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "devOptional": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -19627,7 +19582,6 @@ "resolved": "https://registry.npmjs.org/winston/-/winston-3.19.0.tgz", "integrity": "sha512-LZNJgPzfKR+/J3cHkxcpHKpKKvGfDZVPS4hfJCc4cCG0CgYzvlD6yE/S3CIL/Yt91ak327YCpiF/0MyeZHEHKA==", "license": "MIT", - "peer": true, "dependencies": { "@colors/colors": "^1.6.0", "@dabh/diagnostics": "^2.0.8", @@ -19834,7 +19788,6 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/server/db/sqlite/driver.ts b/server/db/sqlite/driver.ts index 1b0b0b526..93188c929 100644 --- a/server/db/sqlite/driver.ts +++ b/server/db/sqlite/driver.ts @@ -1,5 +1,6 @@ import { drizzle as DrizzleSqlite } from "drizzle-orm/better-sqlite3"; import Database from "better-sqlite3"; +import type BetterSqlite3 from "better-sqlite3"; import * as schema from "./schema/schema"; import path from "path"; import fs from "fs"; @@ -11,6 +12,46 @@ export const exists = checkFileExists(location); bootstrapVolume(); +/** + * Wrap a better-sqlite3 Statement so that the native sqlite3_stmt handle + * is released immediately after the first execution instead of waiting + * for V8 garbage collection. + * + * Background: drizzle-orm creates a **new** native prepared statement for + * every query execution (`session.prepareQuery` → `this.client.prepare`). + * Each native `sqlite3_stmt` consumes 8-32 KB of off-heap memory and is + * only freed when V8's GC collects the JS wrapper. Under sustained load + * (e.g. Uptime Kuma polling verify-session), statement creation outpaces + * GC, causing steady native memory growth — the root cause of #2120. + * + * By calling `stmt.finalize()` right after `.all()` / `.get()` / `.run()` + * returns, the native memory is freed deterministically. This is safe + * because drizzle's one-time queries only invoke each statement once. + */ +function autoFinalizeStatement(stmt: BetterSqlite3.Statement): BetterSqlite3.Statement { + const wrapExec = any>(fn: T): T => { + return function (this: any, ...args: any[]) { + try { + return fn.apply(this, args); + } finally { + try { + // finalize() exists on the native Statement at runtime but + // is missing from @types/better-sqlite3. + (stmt as any).finalize(); + } catch { + // Already finalized — harmless + } + } + } as unknown as T; + }; + + stmt.run = wrapExec(stmt.run); + stmt.get = wrapExec(stmt.get); + stmt.all = wrapExec(stmt.all); + + return stmt; +} + function createDb() { const sqlite = new Database(location); @@ -43,6 +84,15 @@ function createDb() { // SQLite's own cache, reducing event-loop blocking time. sqlite.pragma("mmap_size = 268435456"); + // Intercept prepare() so every statement produced by drizzle-orm is + // automatically finalized after its first (and only) execution. + // This prevents the native sqlite3_stmt objects from accumulating + // until the next GC cycle. + const originalPrepare = sqlite.prepare.bind(sqlite); + (sqlite as any).prepare = function autoFinalizePrepare(source: string) { + return autoFinalizeStatement(originalPrepare(source)); + }; + return DrizzleSqlite(sqlite, { schema }); From 0655ba9423c3a1501e907e5781542a7aeb9097e6 Mon Sep 17 00:00:00 2001 From: Josh Voyles <120749218+Josh-Voyles@users.noreply.github.com> Date: Sat, 2 May 2026 16:33:13 -0400 Subject: [PATCH 3/5] fix: revert investigative changes, keep root cause fixes only Reverts diagnostic instrumentation and defensive hardening added during memory leak investigation. Only root cause fixes survive. Root causes fixed: - SQLite driver: auto-finalize wrapper + PRAGMAs - WS routers: delete clientConfigVersions on disconnect (unbounded Map leak) - WS private router: same + Redis key cleanup Reverted: - Memory monitor, rate limiting, request timeouts (diagnostic/hardening) - shutdownAuditLogger wiring, audit re-queue change, debug logs (cleanup/secondary) - package-lock.json drift --- package-lock.json | 59 +++++++++++++++++++++--- server/cleanup.ts | 2 - server/index.ts | 26 ----------- server/internalServer.ts | 21 --------- server/routers/badger/exchangeSession.ts | 2 +- server/routers/badger/logRequestAudit.ts | 16 +++---- server/routers/badger/verifySession.ts | 2 +- 7 files changed, 63 insertions(+), 65 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1676e6f25..8c241554a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1058,6 +1058,7 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -2353,6 +2354,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2375,6 +2377,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2397,6 +2400,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2413,6 +2417,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2429,6 +2434,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2445,6 +2451,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2461,6 +2468,7 @@ "cpu": [ "ppc64" ], + "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2477,6 +2485,7 @@ "cpu": [ "s390x" ], + "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2493,6 +2502,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2509,6 +2519,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2525,6 +2536,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2541,6 +2553,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2563,6 +2576,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2585,6 +2599,7 @@ "cpu": [ "ppc64" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2607,6 +2622,7 @@ "cpu": [ "s390x" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2629,6 +2645,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2651,6 +2668,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2673,6 +2691,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -2695,6 +2714,7 @@ "cpu": [ "wasm32" ], + "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", "optional": true, "dependencies": { @@ -2714,6 +2734,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -2733,6 +2754,7 @@ "cpu": [ "ia32" ], + "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -2752,6 +2774,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ @@ -3011,6 +3034,7 @@ "integrity": "sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": "^14.21.3 || >=16" }, @@ -6957,6 +6981,7 @@ "resolved": "https://registry.npmjs.org/@react-email/text/-/text-0.1.6.tgz", "integrity": "sha512-TYqkioRS45wTR5il3dYk/SbUjjEdhSwh9BtRNB99qNH1pXAwA45H7rAuxehiu8iJQJH0IyIr+6n62gBz9ezmsw==", "license": "MIT", + "peer": true, "engines": { "node": ">=20.0.0" }, @@ -8417,6 +8442,7 @@ "version": "5.90.21", "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.90.21.tgz", "integrity": "sha512-0Lu6y5t+tvlTJMTO7oh5NSpJfpg/5D41LlThfepTixPYkJ0sE2Jj0m0f6yYqujBwIXlId87e234+MxG3D3g7kg==", + "peer": true, "dependencies": { "@tanstack/query-core": "5.90.20" }, @@ -8532,6 +8558,7 @@ "integrity": "sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@types/node": "*" } @@ -8879,6 +8906,7 @@ "integrity": "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^5.0.0", @@ -8974,6 +9002,7 @@ "integrity": "sha512-oX8xrhvpiyRCQkG1MFchB09f+cXftgIXb3a7UUa4Y3wpmZPw5tyZGTLWhlESOLq1Rq6oDlc8npVU2/9xiCuXMA==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.18.0" } @@ -9001,6 +9030,7 @@ "integrity": "sha512-gT+oueVQkqnj6ajGJXblFR4iavIXWsGAFCk3dP4Kki5+a9R4NMt0JARdk6s8cUKcfUoqP5dAtDSLU8xYUTFV+Q==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@types/node": "*", "pg-protocol": "*", @@ -9026,6 +9056,7 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", "devOptional": true, + "peer": true, "dependencies": { "csstype": "^3.2.2" } @@ -9036,6 +9067,7 @@ "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", "devOptional": true, "license": "MIT", + "peer": true, "peerDependencies": { "@types/react": "^19.2.0" } @@ -9122,8 +9154,7 @@ "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", "license": "MIT", - "optional": true, - "peer": true + "optional": true }, "node_modules/@types/ws": { "version": "8.18.1", @@ -9197,6 +9228,7 @@ "integrity": "sha512-klQbnPAAiGYFyI02+znpBRLyjL4/BrBd0nyWkdC0s/6xFLkXYQ8OoRrSkqacS1ddVxf/LDyODIKbQ5TgKAf/Fg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.56.1", "@typescript-eslint/types": "8.56.1", @@ -9670,6 +9702,7 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -10119,6 +10152,7 @@ "integrity": "sha512-Ixm8tFfoKKIPYdCCKYTsqv+Fd4IJ0DQqMyEimo+pxUOMUR9cVPlwTrFt9Avu+3cb6Zp3mAzl+t1MrG2fxxKsxw==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/types": "^7.26.0" } @@ -10190,6 +10224,7 @@ "integrity": "sha512-Ba0KR+Fzxh2jDRhdg6TSH0SJGzb8C0aBY4hR8w8madIdIzzC6Y1+kx5qR6eS1Z+Gy20h6ZU28aeyg0z1VIrShQ==", "hasInstallScript": true, "license": "MIT", + "peer": true, "dependencies": { "bindings": "^1.5.0", "prebuild-install": "^7.1.1" @@ -10318,6 +10353,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -11224,6 +11260,7 @@ "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "license": "ISC", + "peer": true, "engines": { "node": ">=12" } @@ -11664,7 +11701,6 @@ "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.3.2.tgz", "integrity": "sha512-6obghkliLdmKa56xdbLOpUZ43pAR6xFy1uOrxBaIDjT+yaRuuybLjGS9eVBoSR/UPU5fq3OXClEHLJNGvbxKpQ==", "license": "(MPL-2.0 OR Apache-2.0)", - "peer": true, "engines": { "node": ">=20" }, @@ -12299,6 +12335,7 @@ "dev": true, "hasInstallScript": true, "license": "MIT", + "peer": true, "bin": { "esbuild": "bin/esbuild" }, @@ -12384,6 +12421,7 @@ "integrity": "sha512-COV33RzXZkqhG9P2rZCFl9ZmJ7WL+gQSCRzE7RhkbclbQPtLAWReL7ysA0Sh4c8Im2U9ynybdR56PV0XcKvqaQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", @@ -12520,6 +12558,7 @@ "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -12913,6 +12952,7 @@ "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", + "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -15330,7 +15370,6 @@ "resolved": "https://registry.npmjs.org/monaco-editor/-/monaco-editor-0.55.1.tgz", "integrity": "sha512-jz4x+TJNFHwHtwuV9vA9rMujcZRb0CEilTEwG2rRSpe/A7Jdkuj8xPKttCgOh+v/lkHy7HsZ64oj+q3xoAFl9A==", "license": "MIT", - "peer": true, "dependencies": { "dompurify": "3.2.7", "marked": "14.0.0" @@ -15341,7 +15380,6 @@ "resolved": "https://registry.npmjs.org/marked/-/marked-14.0.0.tgz", "integrity": "sha512-uIj4+faQ+MgHgwUW1l2PsPglZLOLOT1uErt06dAPtx2kjteLAkbsd/0FiYg/MGS+i7ZKLb7w2WClxHkzOOuryQ==", "license": "MIT", - "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -15430,6 +15468,7 @@ "resolved": "https://registry.npmjs.org/next/-/next-15.5.15.tgz", "integrity": "sha512-VSqCrJwtLVGwAVE0Sb/yikrQfkwkZW9p+lL/J4+xe+G3ZA+QnWPqgcfH1tDUEuk9y+pthzzVFp4L/U8JerMfMQ==", "license": "MIT", + "peer": true, "dependencies": { "@next/env": "15.5.15", "@swc/helpers": "0.5.15", @@ -16389,6 +16428,7 @@ "resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz", "integrity": "sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==", "license": "MIT", + "peer": true, "dependencies": { "pg-connection-string": "^2.12.0", "pg-pool": "^3.13.0", @@ -16896,6 +16936,7 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.4.tgz", "integrity": "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -16927,6 +16968,7 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.4.tgz", "integrity": "sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ==", "license": "MIT", + "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -17219,6 +17261,7 @@ "resolved": "https://registry.npmjs.org/react-hook-form/-/react-hook-form-7.71.2.tgz", "integrity": "sha512-1CHvcDYzuRUNOflt4MOq3ZM46AronNJtQ1S7tnX6YN4y72qhgiUItpacZUAQ0TyWYci3yz1X+rXaSxiuEm86PA==", "license": "MIT", + "peer": true, "engines": { "node": ">=18.0.0" }, @@ -18680,7 +18723,8 @@ "version": "4.2.2", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.2.2.tgz", "integrity": "sha512-KWBIxs1Xb6NoLdMVqhbhgwZf2PGBpPEiwOqgI4pFIYbNTfBXiKYyWoTsXgBQ9WFg/OlhnvHaY+AEpW7wSmFo2Q==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/tapable": { "version": "2.3.2", @@ -19155,6 +19199,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "devOptional": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -19582,6 +19627,7 @@ "resolved": "https://registry.npmjs.org/winston/-/winston-3.19.0.tgz", "integrity": "sha512-LZNJgPzfKR+/J3cHkxcpHKpKKvGfDZVPS4hfJCc4cCG0CgYzvlD6yE/S3CIL/Yt91ak327YCpiF/0MyeZHEHKA==", "license": "MIT", + "peer": true, "dependencies": { "@colors/colors": "^1.6.0", "@dabh/diagnostics": "^2.0.8", @@ -19788,6 +19834,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/server/cleanup.ts b/server/cleanup.ts index a1c7fec59..10e9f4cc3 100644 --- a/server/cleanup.ts +++ b/server/cleanup.ts @@ -3,11 +3,9 @@ import { flushConnectionLogToDb } from "#dynamic/routers/newt"; import { flushSiteBandwidthToDb } from "@server/routers/gerbil/receiveBandwidth"; import { stopPingAccumulator } from "@server/routers/newt/pingAccumulator"; import { cleanup as wsCleanup } from "#dynamic/routers/ws"; -import { shutdownAuditLogger } from "@server/routers/badger/logRequestAudit"; async function cleanup() { await stopPingAccumulator(); - await shutdownAuditLogger(); await flushBandwidthToDb(); await flushConnectionLogToDb(); await flushSiteBandwidthToDb(); diff --git a/server/index.ts b/server/index.ts index f3274d2fb..e3a6ba049 100644 --- a/server/index.ts +++ b/server/index.ts @@ -24,29 +24,6 @@ import license from "#dynamic/license/license"; import { initLogCleanupInterval } from "@server/lib/cleanupLogs"; import { initAcmeCertSync } from "#dynamic/lib/acmeCertSync"; import { fetchServerIp } from "@server/lib/serverIpService"; -import logger from "@server/logger"; - -/** - * Periodic memory usage logging for monitoring and leak detection. - * Logs heap usage, external (native) memory, and RSS every 60 seconds. - * This is lightweight (single process.memoryUsage() call) and provides - * the data needed to detect slow memory growth over hours/days. - */ -function startMemoryMonitor(): void { - const INTERVAL_MS = 60_000; // every 60 seconds - const timer = setInterval(() => { - const mem = process.memoryUsage(); - logger.info( - `Memory usage - ` + - `heapUsed: ${(mem.heapUsed / 1024 / 1024).toFixed(1)}MB, ` + - `heapTotal: ${(mem.heapTotal / 1024 / 1024).toFixed(1)}MB, ` + - `rss: ${(mem.rss / 1024 / 1024).toFixed(1)}MB, ` + - `external: ${(mem.external / 1024 / 1024).toFixed(1)}MB, ` + - `arrayBuffers: ${(mem.arrayBuffers / 1024 / 1024).toFixed(1)}MB` - ); - }, INTERVAL_MS); - timer.unref(); -} async function startServers() { await setHostMeta(); @@ -65,9 +42,6 @@ async function startServers() { initLogCleanupInterval(); initAcmeCertSync(); - // Start memory monitoring for leak detection - startMemoryMonitor(); - // Start all servers const apiServer = createApiServer(); const internalServer = createInternalServer(); diff --git a/server/internalServer.ts b/server/internalServer.ts index 6eb207121..83872e7f9 100644 --- a/server/internalServer.ts +++ b/server/internalServer.ts @@ -10,8 +10,6 @@ import { } from "@server/middlewares"; import { internalRouter } from "#dynamic/routers/internal"; import { stripDuplicateSesions } from "./middlewares/stripDuplicateSessions"; -import { requestTimeoutMiddleware } from "./middlewares/requestTimeout"; -import rateLimit from "express-rate-limit"; const internalPort = config.getRawConfig().server.internal_port; @@ -29,25 +27,6 @@ export function createInternalServer() { internalServer.use(cookieParser()); internalServer.use(express.json()); - // Prevent requests from hanging indefinitely. Without this, if a - // database query blocks (especially on SQLite), pending requests - // accumulate in memory with no upper bound on lifetime. - internalServer.use(requestTimeoutMiddleware(30000)); // 30 second timeout - - // Rate-limit the internal verify-session endpoint. This server - // handles forward-auth requests from Traefik/Badger. Under heavy - // monitoring (e.g. Uptime Kuma), requests can arrive faster than - // SQLite can serve them, causing unbounded request queuing and - // memory growth. - internalServer.use( - rateLimit({ - windowMs: 60 * 1000, // 1 minute window - max: 1000, // generous limit: ~17 req/s - standardHeaders: true, - legacyHeaders: false - }) - ); - const prefix = `/api/v1`; internalServer.use(prefix, internalRouter); diff --git a/server/routers/badger/exchangeSession.ts b/server/routers/badger/exchangeSession.ts index 1fc52cea7..08987961d 100644 --- a/server/routers/badger/exchangeSession.ts +++ b/server/routers/badger/exchangeSession.ts @@ -41,7 +41,7 @@ export async function exchangeSession( res: Response, next: NextFunction ): Promise { - logger.debug("Exchange session: Badger request received"); + logger.debug("Exchange session: Badger sent", req.body); const parsedBody = exchangeSessionBodySchema.safeParse(req.body); diff --git a/server/routers/badger/logRequestAudit.ts b/server/routers/badger/logRequestAudit.ts index 39f0eb3ca..884fb7ae4 100644 --- a/server/routers/badger/logRequestAudit.ts +++ b/server/routers/badger/logRequestAudit.ts @@ -84,14 +84,14 @@ async function flushAuditLogs() { logger.debug(`Flushed ${logsToWrite.length} audit logs to database`); } catch (error) { logger.error("Error flushing audit logs:", error); - // On transaction error, drop the logs rather than re-queuing them. - // The previous re-queue approach created a positive feedback loop: - // failed flush → re-queue → larger next flush → longer DB lock → - // higher chance of next failure → repeat. This caused unbounded - // memory growth on SQLite where write contention is common. - // Audit logs are best-effort telemetry — losing a batch on error - // is acceptable; leaking memory until the process crashes is not. - logger.warn(`Dropped ${logsToWrite.length} audit logs after flush failure`); + // On transaction error, put logs back at the front of the buffer to retry + // but only if buffer isn't too large + if (auditLogBuffer.length < MAX_BUFFER_SIZE - logsToWrite.length) { + auditLogBuffer.unshift(...logsToWrite); + logger.info(`Re-queued ${logsToWrite.length} audit logs for retry`); + } else { + logger.error(`Buffer full, dropped ${logsToWrite.length} audit logs`); + } } finally { isFlushInProgress = false; // If buffer filled up while we were flushing, flush again diff --git a/server/routers/badger/verifySession.ts b/server/routers/badger/verifySession.ts index b80a1ada0..e2e5f6766 100644 --- a/server/routers/badger/verifySession.ts +++ b/server/routers/badger/verifySession.ts @@ -80,7 +80,7 @@ export async function verifyResourceSession( res: Response, next: NextFunction ): Promise { - logger.debug("Verify session: Badger request received"); + logger.debug("Verify session: Badger sent", req.body); // remove when done testing const parsedBody = verifyResourceSessionSchema.safeParse(req.body); From 9bd33072f477a1fa14b3ac010c9813207f61d279 Mon Sep 17 00:00:00 2001 From: Josh Voyles <120749218+Josh-Voyles@users.noreply.github.com> Date: Sun, 3 May 2026 00:00:11 -0400 Subject: [PATCH 4/5] cleaned comments - more concise --- server/db/sqlite/driver.ts | 57 +++++++++++---------------------- server/private/routers/ws/ws.ts | 17 ++++------ server/routers/ws/ws.ts | 28 ++++++++++------ 3 files changed, 43 insertions(+), 59 deletions(-) diff --git a/server/db/sqlite/driver.ts b/server/db/sqlite/driver.ts index 93188c929..af60ce331 100644 --- a/server/db/sqlite/driver.ts +++ b/server/db/sqlite/driver.ts @@ -13,22 +13,13 @@ export const exists = checkFileExists(location); bootstrapVolume(); /** - * Wrap a better-sqlite3 Statement so that the native sqlite3_stmt handle - * is released immediately after the first execution instead of waiting - * for V8 garbage collection. - * - * Background: drizzle-orm creates a **new** native prepared statement for - * every query execution (`session.prepareQuery` → `this.client.prepare`). - * Each native `sqlite3_stmt` consumes 8-32 KB of off-heap memory and is - * only freed when V8's GC collects the JS wrapper. Under sustained load - * (e.g. Uptime Kuma polling verify-session), statement creation outpaces - * GC, causing steady native memory growth — the root cause of #2120. - * - * By calling `stmt.finalize()` right after `.all()` / `.get()` / `.run()` - * returns, the native memory is freed deterministically. This is safe - * because drizzle's one-time queries only invoke each statement once. + * Wraps better-sqlite3 Statement to call `finalize()` immediately after + * execution, freeing native sqlite3_stmt memory deterministically instead + * of waiting for GC. Fixes steady off-heap growth under load (#2120). */ -function autoFinalizeStatement(stmt: BetterSqlite3.Statement): BetterSqlite3.Statement { +function autoFinalizeStatement( + stmt: BetterSqlite3.Statement +): BetterSqlite3.Statement { const wrapExec = any>(fn: T): T => { return function (this: any, ...args: any[]) { try { @@ -55,39 +46,27 @@ function autoFinalizeStatement(stmt: BetterSqlite3.Statement): BetterSqlite3.Sta function createDb() { const sqlite = new Database(location); - // Enable WAL mode for dramatically better concurrent read/write - // performance. Without this, readers block writers and vice versa, - // causing severe contention when multiple subsystems (verifySession, - // TraefikConfigManager, audit log flushes, ping flushes) all share - // this single connection. WAL mode allows concurrent readers with a - // single writer, which is the typical access pattern. + // Enable WAL mode — allows concurrent readers + single writer, preventing + // contention across subsystems (verifySession, Traefik, audit, ping). sqlite.pragma("journal_mode = WAL"); - // Wait up to 5 seconds when the database is locked instead of - // failing immediately with SQLITE_BUSY. This prevents transient - // write failures from causing audit log buffer re-queues and retry - // loops that accumulate memory. + // Wait up to 5s on SQLITE_BUSY instead of failing — prevents audit log + // retry loops that accumulate memory. sqlite.pragma("busy_timeout = 5000"); - // NORMAL synchronous mode is safe with WAL and significantly reduces - // the time each write holds the database lock. + // NORMAL sync mode: safe with WAL, reduces write lock hold time. sqlite.pragma("synchronous = NORMAL"); - // Increase the page cache to 64 MB (negative value = KB). The - // default (2 MB) causes frequent I/O round-trips on the large JOIN - // queries used by TraefikConfigManager, which block the event loop - // for longer than necessary. + // 64 MB page cache (default 2 MB) — reduces I/O round-trips on large + // TraefikConfigManager JOINs that block the event loop. sqlite.pragma("cache_size = -65536"); - // Enable memory-mapped I/O for reads (256 MB). This allows the OS - // to serve read queries from the page cache without going through - // SQLite's own cache, reducing event-loop blocking time. + // 256 MB memory-mapped I/O — OS serves reads from page cache directly, + // reducing event-loop blocking. sqlite.pragma("mmap_size = 268435456"); - // Intercept prepare() so every statement produced by drizzle-orm is - // automatically finalized after its first (and only) execution. - // This prevents the native sqlite3_stmt objects from accumulating - // until the next GC cycle. + // Wrap prepare() so every drizzle-orm statement is auto-finalized after + // first use, preventing sqlite3_stmt accumulation between GC cycles. const originalPrepare = sqlite.prepare.bind(sqlite); (sqlite as any).prepare = function autoFinalizePrepare(source: string) { return autoFinalizeStatement(originalPrepare(source)); @@ -103,7 +82,7 @@ export default db; export const primaryDb = db; export type Transaction = Parameters< Parameters<(typeof db)["transaction"]>[0] - >[0]; +>[0]; export const DB_TYPE: "pg" | "sqlite" = "sqlite"; function checkFileExists(filePath: string): boolean { diff --git a/server/private/routers/ws/ws.ts b/server/private/routers/ws/ws.ts index 83bae0162..764127df4 100644 --- a/server/private/routers/ws/ws.ts +++ b/server/private/routers/ws/ws.ts @@ -22,7 +22,7 @@ import { Olm, olms, RemoteExitNode, - remoteExitNodes, + remoteExitNodes } from "@server/db"; import { eq } from "drizzle-orm"; import { db } from "@server/db"; @@ -194,8 +194,6 @@ const connectedClients: Map = new Map(); // Config version tracking map (local to this node, resets on server restart) const clientConfigVersions: Map = new Map(); - - // Recovery tracking let isRedisRecoveryInProgress = false; @@ -406,9 +404,8 @@ const removeClient = async ( const updatedClients = existingClients.filter((client) => client !== ws); if (updatedClients.length === 0) { connectedClients.delete(mapKey); - // Clean up config version tracking to prevent unbounded memory - // growth. Without this, every unique clientId that ever connects - // leaves a permanent entry in clientConfigVersions. + // Remove clientId from clientConfigVersions on disconnect — prevents + // unbounded memory growth from stale entries. clientConfigVersions.delete(clientId); if (redisManager.isRedisEnabled()) { @@ -417,8 +414,7 @@ const removeClient = async ( await redisManager.del( getNodeConnectionsKey(NODE_ID, clientId) ); - // Also clean up the Redis config version key so it doesn't - // accumulate indefinitely in Redis either. + // Remove Redis config version key to prevent indefinite accumulation. await redisManager.del(getConfigVersionKey(clientId)); } catch (error) { logger.error( @@ -1104,9 +1100,8 @@ const disconnectClient = async (clientId: string): Promise => { } }); - // Eagerly clean up tracking maps. The close event handlers will also - // call removeClient, but if the socket is already in CLOSING state - // the close event may never fire, leaving zombie entries. + // Eagerly remove client — close event may not fire if socket is already + // CLOSING, leaving zombie entries. connectedClients.delete(mapKey); clientConfigVersions.delete(clientId); diff --git a/server/routers/ws/ws.ts b/server/routers/ws/ws.ts index 51274cc20..e7dcfe9cb 100644 --- a/server/routers/ws/ws.ts +++ b/server/routers/ws/ws.ts @@ -3,7 +3,15 @@ import zlib from "zlib"; import { Server as HttpServer } from "http"; import { WebSocket, WebSocketServer } from "ws"; import { Socket } from "net"; -import { Newt, newts, NewtSession, olms, Olm, OlmSession, sites } from "@server/db"; +import { + Newt, + newts, + NewtSession, + olms, + Olm, + OlmSession, + sites +} from "@server/db"; import { eq } from "drizzle-orm"; import { db } from "@server/db"; import { recordPing } from "@server/routers/newt/pingAccumulator"; @@ -80,9 +88,8 @@ const removeClient = async ( const updatedClients = existingClients.filter((client) => client !== ws); if (updatedClients.length === 0) { connectedClients.delete(mapKey); - // Clean up config version tracking to prevent unbounded memory - // growth. Without this, every unique clientId that ever connects - // leaves a permanent entry in clientConfigVersions. + // Remove clientId from clientConfigVersions — prevents unbounded growth + // from stale entries. clientConfigVersions.delete(clientId); logger.info( @@ -222,9 +229,13 @@ const hasActiveConnections = async (clientId: string): Promise => { }; // Get the current config version for a client -const getClientConfigVersion = async (clientId: string): Promise => { +const getClientConfigVersion = async ( + clientId: string +): Promise => { const version = clientConfigVersions.get(clientId); - logger.debug(`getClientConfigVersion called for clientId: ${clientId}, returning: ${version} (type: ${typeof version})`); + logger.debug( + `getClientConfigVersion called for clientId: ${clientId}, returning: ${version} (type: ${typeof version})` + ); return version; }; @@ -511,9 +522,8 @@ const disconnectClient = async (clientId: string): Promise => { } }); - // Eagerly clean up tracking maps. The close event handlers will also - // call removeClient, but if the socket is already in CLOSING state - // the close event may never fire, leaving zombie entries. + // Eagerly remove client — close event may not fire if socket already + // CLOSING, leaving zombie entries. connectedClients.delete(mapKey); clientConfigVersions.delete(clientId); From 2154811ffb140ae0b656f93d5e02775ae88d8915 Mon Sep 17 00:00:00 2001 From: Josh Voyles <120749218+Josh-Voyles@users.noreply.github.com> Date: Sun, 3 May 2026 09:39:27 -0400 Subject: [PATCH 5/5] removed possible introduced HA Redis bug; improved comment --- server/db/sqlite/driver.ts | 2 ++ server/private/routers/ws/ws.ts | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/db/sqlite/driver.ts b/server/db/sqlite/driver.ts index af60ce331..12681f521 100644 --- a/server/db/sqlite/driver.ts +++ b/server/db/sqlite/driver.ts @@ -16,6 +16,8 @@ bootstrapVolume(); * Wraps better-sqlite3 Statement to call `finalize()` immediately after * execution, freeing native sqlite3_stmt memory deterministically instead * of waiting for GC. Fixes steady off-heap growth under load (#2120). + * WARNING: Finalizes after first execution — incompatible with drizzle's + * reusable .prepare() builders. No such usage exists in this codebase. */ function autoFinalizeStatement( stmt: BetterSqlite3.Statement diff --git a/server/private/routers/ws/ws.ts b/server/private/routers/ws/ws.ts index 764127df4..c01ebc9eb 100644 --- a/server/private/routers/ws/ws.ts +++ b/server/private/routers/ws/ws.ts @@ -414,8 +414,6 @@ const removeClient = async ( await redisManager.del( getNodeConnectionsKey(NODE_ID, clientId) ); - // Remove Redis config version key to prevent indefinite accumulation. - await redisManager.del(getConfigVersionKey(clientId)); } catch (error) { logger.error( "Failed to remove client from Redis tracking (cleanup will occur on recovery):",