Batch deletes

fix: Remove return in installer which prevents showing token

install/main.go

@@ -238,7 +238,6 @@ func main() {
 				}
 
 				fmt.Println("CrowdSec installed successfully!")
-				return
 			}
 		}
 	}

server/apiServer.ts

@@ -79,6 +79,12 @@ export function createApiServer() {
     // Add request timeout middleware
     apiServer.use(requestTimeoutMiddleware(60000)); // 60 second timeout
 
+    apiServer.use(logIncomingMiddleware);
+
+    if (build !== "oss") {
+        apiServer.use(`${prefix}/hybrid`, hybridRouter); // put before rate limiting because we will rate limit there separately because some of the routes are heavily used
+    }
+
     if (!dev) {
         apiServer.use(
             rateLimit({
@@ -101,11 +107,7 @@ export function createApiServer() {
     }
 
     // API routes
-    apiServer.use(logIncomingMiddleware);
     apiServer.use(prefix, unauthenticated);
-    if (build !== "oss") {
-        apiServer.use(`${prefix}/hybrid`, hybridRouter);
-    }
     apiServer.use(prefix, authenticated);
 
     // WebSocket routes

server/private/lib/rateLimit.ts

@@ -72,6 +72,43 @@ export class RateLimitService {
         return `ratelimit:${clientId}:${messageType}`;
     }
 
+    // Helper function to clean up old timestamp fields from a Redis hash
+    private async cleanupOldTimestamps(key: string, windowStart: number): Promise<void> {
+        if (!redisManager.isRedisEnabled()) return;
+
+        try {
+            const client = redisManager.getClient();
+            if (!client) return;
+
+            // Get all fields in the hash
+            const allData = await redisManager.hgetall(key);
+            if (!allData || Object.keys(allData).length === 0) return;
+
+            // Find fields that are older than the window
+            const fieldsToDelete: string[] = [];
+            for (const timestamp of Object.keys(allData)) {
+                const time = parseInt(timestamp);
+                if (time < windowStart) {
+                    fieldsToDelete.push(timestamp);
+                }
+            }
+
+            // Delete old fields in batches to avoid call stack size exceeded errors
+            // The spread operator can cause issues with very large arrays
+            if (fieldsToDelete.length > 0) {
+                const batchSize = 1000; // Process 1000 fields at a time
+                for (let i = 0; i < fieldsToDelete.length; i += batchSize) {
+                    const batch = fieldsToDelete.slice(i, i + batchSize);
+                    await client.hdel(key, ...batch);
+                }
+                logger.debug(`Cleaned up ${fieldsToDelete.length} old timestamp fields from ${key}`);
+            }
+        } catch (error) {
+            logger.error(`Failed to cleanup old timestamps for key ${key}:`, error);
+            // Don't throw - cleanup failures shouldn't block rate limiting
+        }
+    }
+
     // Helper function to sync local rate limit data to Redis
     private async syncRateLimitToRedis(
         clientId: string,
@@ -81,8 +118,12 @@ export class RateLimitService {
 
         try {
             const currentTime = Math.floor(Date.now() / 1000);
+            const windowStart = currentTime - RATE_LIMIT_WINDOW;
             const globalKey = this.getRateLimitKey(clientId);
 
+            // Clean up old timestamp fields before writing
+            await this.cleanupOldTimestamps(globalKey, windowStart);
+
             // Get current value and add pending count
             const currentValue = await redisManager.hget(
                 globalKey,
@@ -93,7 +134,7 @@ export class RateLimitService {
             ).toString();
             await redisManager.hset(globalKey, currentTime.toString(), newValue);
 
-            // Set TTL using the client directly
+            // Set TTL using the client directly - this prevents the key from persisting forever
             if (redisManager.getClient()) {
                 await redisManager
                     .getClient()
@@ -119,8 +160,12 @@ export class RateLimitService {
 
         try {
             const currentTime = Math.floor(Date.now() / 1000);
+            const windowStart = currentTime - RATE_LIMIT_WINDOW;
             const messageTypeKey = this.getMessageTypeRateLimitKey(clientId, messageType);
 
+            // Clean up old timestamp fields before writing
+            await this.cleanupOldTimestamps(messageTypeKey, windowStart);
+
             // Get current value and add pending count
             const currentValue = await redisManager.hget(
                 messageTypeKey,
@@ -135,7 +180,7 @@ export class RateLimitService {
                 newValue
             );
 
-            // Set TTL using the client directly
+            // Set TTL using the client directly - this prevents the key from persisting forever
             if (redisManager.getClient()) {
                 await redisManager
                     .getClient()
@@ -170,6 +215,10 @@ export class RateLimitService {
 
         try {
             const globalKey = this.getRateLimitKey(clientId);
+            
+            // Clean up old timestamp fields before reading
+            await this.cleanupOldTimestamps(globalKey, windowStart);
+            
             const globalRateLimitData = await redisManager.hgetall(globalKey);
 
             let count = 0;
@@ -215,6 +264,10 @@ export class RateLimitService {
 
         try {
             const messageTypeKey = this.getMessageTypeRateLimitKey(clientId, messageType);
+            
+            // Clean up old timestamp fields before reading
+            await this.cleanupOldTimestamps(messageTypeKey, windowStart);
+            
             const messageTypeRateLimitData = await redisManager.hgetall(messageTypeKey);
 
             let count = 0;

server/private/routers/hybrid.ts

@@ -227,6 +227,8 @@ export type UserSessionWithUser = {
 export const hybridRouter = Router();
 hybridRouter.use(verifySessionRemoteExitNodeMiddleware);
 
+// TODO: ADD RATE LIMITING TO THESE ROUTES AS NEEDED BASED ON USAGE PATTERNS
+
 hybridRouter.get(
     "/general-config",
     async (req: Request, res: Response, next: NextFunction) => {

server/private/routers/misc/index.ts

@@ -1 +1,14 @@
+/*
+ * This file is part of a proprietary work.
+ *
+ * Copyright (c) 2025 Fossorial, Inc.
+ * All rights reserved.
+ *
+ * This file is licensed under the Fossorial Commercial License.
+ * You may not use this file except in compliance with the License.
+ * Unauthorized use, copying, modification, or distribution is strictly prohibited.
+ *
+ * This file is not licensed under the AGPLv3.
+ */
+
 export * from "./sendSupportEmail";

server/private/routers/misc/sendSupportEmail.ts

@@ -68,7 +68,7 @@ export async function sendSupportEmail(
                 {
                     name: req.user?.email || "Support User",
                     to: "support@pangolin.net",
-                    from: req.user?.email || config.getNoReplyEmail(),
+                    from: config.getNoReplyEmail(),
                     subject: `Support Request: ${subject}`
                 }
             );