Increase maxRetriesPerRequest for Redis connections to improve resilience

Fixes #3677

messages/en-US.json

@@ -1424,15 +1424,6 @@
     "logoutError": "Error logging out",
     "signingAs": "Signed in as",
     "serverAdmin": "Server Admin",
-    "promoteServerAdmin": "Promote to Server admin",
-    "promoteServerAdminTitle": "Promote to Server Admin",
-    "promoteServerAdminQuestion": "Are you sure you want to promote {selectedUser} to server admin?",
-    "promoteServerAdminMessage": "Server admins have full access to every organization, user, and setting on this instance.",
-    "promoteServerAdminWarning": "You cannot demote a server admin from this page.",
-    "promoteServerAdminConfirm": "Promote to server admin",
-    "promoteServerAdminSuccess": "User promoted",
-    "promoteServerAdminSuccessDescription": "{selectedUser} is now a server admin.",
-    "promoteServerAdminError": "Failed to promote user",
     "managedSelfhosted": "Managed Self-Hosted",
     "otpEnable": "Enable Two-factor",
     "otpDisable": "Disable Two-factor",
@@ -2726,6 +2717,7 @@
     "healthScheme": "Method",
     "healthSelectScheme": "Select Method",
     "healthCheckPortInvalid": "Port must be between 1 and 65535",
+    "healthCheckHostnameInvalid": "Hostname must not contain whitespace",
     "healthCheckPath": "Path",
     "healthHostname": "IP / Host",
     "healthPort": "Port",

server/db/pg/schema/privateSchema.ts

@@ -26,7 +26,9 @@ import {
     sites,
     clients,
     sessions,
-    labels
+    labels,
+    aiProviders,
+    virtualApiKeys
 } from "./schema";
 
 export const dnsChallenge = pgTable("dnsChallenges", {
@@ -614,6 +616,87 @@ export const trialNotifications = pgTable("trialNotifications", {
     sentAt: bigint("sentAt", { mode: "number" }).notNull()
 });
 
+// Logs the aggregated prompt + response for a single AI gateway request, for
+// session replay. One row per request (not per streaming chunk). `sessionId`
+// is a fresh random id per row for now - no cross-request correlation yet,
+// but the column exists so a future pass can link multiple rows into a real
+// multi-turn session.
+export const aiSessionLog = pgTable(
+    "aiSessionLog",
+    {
+        id: serial("id").primaryKey(),
+        sessionId: varchar("sessionId").notNull(),
+        orgId: varchar("orgId").references(() => orgs.orgId, {
+            onDelete: "cascade"
+        }),
+        providerId: integer("providerId").references(
+            () => aiProviders.providerId,
+            { onDelete: "set null" }
+        ),
+        capability: varchar("capability").notNull(),
+        resourceId: integer("resourceId").references(
+            () => resources.resourceId,
+            { onDelete: "set null" }
+        ),
+        siteResourceId: integer("siteResourceId").references(
+            () => siteResources.siteResourceId,
+            { onDelete: "set null" }
+        ),
+        userId: varchar("userId").references(() => users.userId, {
+            onDelete: "set null"
+        }),
+        virtualApiKeyId: varchar("virtualApiKeyId").references(
+            () => virtualApiKeys.virtualApiKeyId,
+            { onDelete: "set null" }
+        ),
+        requestedModel: varchar("requestedModel"),
+        isStream: boolean("isStream").notNull().default(false),
+        requestBody: text("requestBody"),
+        responseBody: text("responseBody"),
+        // Capability-agnostic message transcript (JSON-encoded
+        // NormalizedAiMessage[] from server/lib/aiMessageNormalization.ts),
+        // computed at write time so search/display never need per-capability
+        // parsing logic. Null when normalization couldn't recognize the
+        // shape - callers fall back to requestBody/responseBody.
+        normalizedRequest: text("normalizedRequest"),
+        normalizedResponse: text("normalizedResponse"),
+        // True if any of the request/response (raw or normalized) fields
+        // were cut short at AI_SESSION_LOG_MAX_BODY_CHARS before storage.
+        truncated: boolean("truncated").notNull().default(false),
+        statusCode: integer("statusCode"),
+        createdAt: bigint("createdAt", { mode: "number" }).notNull() // epoch seconds
+    },
+    (t) => [
+        index("idx_ai_session_log_org_created").on(t.orgId, t.createdAt),
+        index("idx_ai_session_log_org_provider_created").on(
+            t.orgId,
+            t.providerId,
+            t.createdAt
+        ),
+        index("idx_ai_session_log_org_resource_created").on(
+            t.orgId,
+            t.resourceId,
+            t.createdAt
+        ),
+        index("idx_ai_session_log_org_site_resource_created").on(
+            t.orgId,
+            t.siteResourceId,
+            t.createdAt
+        ),
+        index("idx_ai_session_log_org_user_created").on(
+            t.orgId,
+            t.userId,
+            t.createdAt
+        ),
+        index("idx_ai_session_log_org_virtual_api_key_created").on(
+            t.orgId,
+            t.virtualApiKeyId,
+            t.createdAt
+        ),
+        index("idx_ai_session_log_session").on(t.sessionId)
+    ]
+);
+
 export type Approval = InferSelectModel<typeof approvals>;
 export type Limit = InferSelectModel<typeof limits>;
 export type Account = InferSelectModel<typeof account>;
@@ -660,3 +743,4 @@ export type AlertEmailRecipients = InferSelectModel<
 >;
 export type AlertWebhookActions = InferSelectModel<typeof alertWebhookActions>;
 export type TrialNotification = InferSelectModel<typeof trialNotifications>;
+export type AiSessionLog = InferSelectModel<typeof aiSessionLog>;

server/db/pg/schema/schema.ts

@@ -1,3 +1,4 @@
+import { aiSessionLog } from "@server/db/sqlite";
 import { randomUUID } from "crypto";
 import { InferSelectModel, sql } from "drizzle-orm";
 import {
@@ -1958,87 +1959,6 @@ export const aiBudgetBreachEvents = pgTable(
     ]
 );
 
-// Logs the aggregated prompt + response for a single AI gateway request, for
-// session replay. One row per request (not per streaming chunk). `sessionId`
-// is a fresh random id per row for now - no cross-request correlation yet,
-// but the column exists so a future pass can link multiple rows into a real
-// multi-turn session.
-export const aiSessionLog = pgTable(
-    "aiSessionLog",
-    {
-        id: serial("id").primaryKey(),
-        sessionId: varchar("sessionId").notNull(),
-        orgId: varchar("orgId").references(() => orgs.orgId, {
-            onDelete: "cascade"
-        }),
-        providerId: integer("providerId").references(
-            () => aiProviders.providerId,
-            { onDelete: "set null" }
-        ),
-        capability: varchar("capability").notNull(),
-        resourceId: integer("resourceId").references(
-            () => resources.resourceId,
-            { onDelete: "set null" }
-        ),
-        siteResourceId: integer("siteResourceId").references(
-            () => siteResources.siteResourceId,
-            { onDelete: "set null" }
-        ),
-        userId: varchar("userId").references(() => users.userId, {
-            onDelete: "set null"
-        }),
-        virtualApiKeyId: varchar("virtualApiKeyId").references(
-            () => virtualApiKeys.virtualApiKeyId,
-            { onDelete: "set null" }
-        ),
-        requestedModel: varchar("requestedModel"),
-        isStream: boolean("isStream").notNull().default(false),
-        requestBody: text("requestBody"),
-        responseBody: text("responseBody"),
-        // Capability-agnostic message transcript (JSON-encoded
-        // NormalizedAiMessage[] from server/lib/aiMessageNormalization.ts),
-        // computed at write time so search/display never need per-capability
-        // parsing logic. Null when normalization couldn't recognize the
-        // shape - callers fall back to requestBody/responseBody.
-        normalizedRequest: text("normalizedRequest"),
-        normalizedResponse: text("normalizedResponse"),
-        // True if any of the request/response (raw or normalized) fields
-        // were cut short at AI_SESSION_LOG_MAX_BODY_CHARS before storage.
-        truncated: boolean("truncated").notNull().default(false),
-        statusCode: integer("statusCode"),
-        createdAt: bigint("createdAt", { mode: "number" }).notNull() // epoch seconds
-    },
-    (t) => [
-        index("idx_ai_session_log_org_created").on(t.orgId, t.createdAt),
-        index("idx_ai_session_log_org_provider_created").on(
-            t.orgId,
-            t.providerId,
-            t.createdAt
-        ),
-        index("idx_ai_session_log_org_resource_created").on(
-            t.orgId,
-            t.resourceId,
-            t.createdAt
-        ),
-        index("idx_ai_session_log_org_site_resource_created").on(
-            t.orgId,
-            t.siteResourceId,
-            t.createdAt
-        ),
-        index("idx_ai_session_log_org_user_created").on(
-            t.orgId,
-            t.userId,
-            t.createdAt
-        ),
-        index("idx_ai_session_log_org_virtual_api_key_created").on(
-            t.orgId,
-            t.virtualApiKeyId,
-            t.createdAt
-        ),
-        index("idx_ai_session_log_session").on(t.sessionId)
-    ]
-);
-
 export const certificates = pgTable("certificates", {
     certId: serial("certId").primaryKey(),
     domain: varchar("domain", { length: 255 }).notNull().unique(),

server/db/sqlite/schema/privateSchema.ts

@@ -9,6 +9,7 @@ import {
     uniqueIndex
 } from "drizzle-orm/sqlite-core";
 import {
+    aiProviders,
     clients,
     domains,
     exitNodes,
@@ -20,7 +21,8 @@ import {
     siteResources,
     sites,
     targetHealthCheck,
-    users
+    users,
+    virtualApiKeys
 } from "./schema";
 
 export const dnsChallenge = sqliteTable("dnsChallenges", {
@@ -609,6 +611,91 @@ export const trialNotifications = sqliteTable("trialNotifications", {
     sentAt: integer("sentAt").notNull()
 });
 
+// Logs the aggregated prompt + response for a single AI gateway request, for
+// session replay. One row per request (not per streaming chunk). `sessionId`
+// is a fresh random id per row for now - no cross-request correlation yet,
+// but the column exists so a future pass can link multiple rows into a real
+// multi-turn session.
+export const aiSessionLog = sqliteTable(
+    "aiSessionLog",
+    {
+        id: integer("id").primaryKey({ autoIncrement: true }),
+        sessionId: text("sessionId").notNull(),
+        orgId: text("orgId").references(() => orgs.orgId, {
+            onDelete: "cascade"
+        }),
+        providerId: integer("providerId").references(
+            () => aiProviders.providerId,
+            { onDelete: "set null" }
+        ),
+        capability: text("capability").notNull(),
+        resourceId: integer("resourceId").references(
+            () => resources.resourceId,
+            { onDelete: "set null" }
+        ),
+        siteResourceId: integer("siteResourceId").references(
+            () => siteResources.siteResourceId,
+            { onDelete: "set null" }
+        ),
+        userId: text("userId").references(() => users.userId, {
+            onDelete: "set null"
+        }),
+        virtualApiKeyId: text("virtualApiKeyId").references(
+            () => virtualApiKeys.virtualApiKeyId,
+            { onDelete: "set null" }
+        ),
+        requestedModel: text("requestedModel"),
+        isStream: integer("isStream", { mode: "boolean" })
+            .notNull()
+            .default(false),
+        requestBody: text("requestBody"),
+        responseBody: text("responseBody"),
+        // Capability-agnostic message transcript (JSON-encoded
+        // NormalizedAiMessage[] from server/lib/aiMessageNormalization.ts),
+        // computed at write time so search/display never need per-capability
+        // parsing logic. Null when normalization couldn't recognize the
+        // shape - callers fall back to requestBody/responseBody.
+        normalizedRequest: text("normalizedRequest"),
+        normalizedResponse: text("normalizedResponse"),
+        // True if any of the request/response (raw or normalized) fields
+        // were cut short at AI_SESSION_LOG_MAX_BODY_CHARS before storage.
+        truncated: integer("truncated", { mode: "boolean" })
+            .notNull()
+            .default(false),
+        statusCode: integer("statusCode"),
+        createdAt: integer("createdAt").notNull() // epoch seconds
+    },
+    (t) => [
+        index("idx_ai_session_log_org_created").on(t.orgId, t.createdAt),
+        index("idx_ai_session_log_org_provider_created").on(
+            t.orgId,
+            t.providerId,
+            t.createdAt
+        ),
+        index("idx_ai_session_log_org_resource_created").on(
+            t.orgId,
+            t.resourceId,
+            t.createdAt
+        ),
+        index("idx_ai_session_log_org_site_resource_created").on(
+            t.orgId,
+            t.siteResourceId,
+            t.createdAt
+        ),
+        index("idx_ai_session_log_org_user_created").on(
+            t.orgId,
+            t.userId,
+            t.createdAt
+        ),
+        index("idx_ai_session_log_org_virtual_api_key_created").on(
+            t.orgId,
+            t.virtualApiKeyId,
+            t.createdAt
+        ),
+        index("idx_ai_session_log_session").on(t.sessionId)
+    ]
+);
+
 export type Approval = InferSelectModel<typeof approvals>;
 export type Limit = InferSelectModel<typeof limits>;
 export type Account = InferSelectModel<typeof account>;
@@ -647,3 +734,4 @@ export type AlertEmailAction = InferSelectModel<typeof alertEmailActions>;
 export type AlertEmailRecipient = InferSelectModel<typeof alertEmailRecipients>;
 export type AlertWebhookAction = InferSelectModel<typeof alertWebhookActions>;
 export type TrialNotification = InferSelectModel<typeof trialNotifications>;
+export type AiSessionLog = InferSelectModel<typeof aiSessionLog>;

server/db/sqlite/schema/schema.ts

@@ -147,9 +147,7 @@ export const sites = sqliteTable(
             .$type<"pending" | "approved">()
             .default("approved")
     },
-    (table) => [
-        index("idx_sites_orgId").on(table.orgId)
-    ]
+    (table) => [index("idx_sites_orgId").on(table.orgId)]
 );
 
 export const resources = sqliteTable(
@@ -192,7 +190,9 @@ export const resources = sqliteTable(
             mode: "boolean"
         }),
         applyRules: integer("applyRules", { mode: "boolean" }),
-        enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
+        enabled: integer("enabled", { mode: "boolean" })
+            .notNull()
+            .default(true),
         stickySession: integer("stickySession", { mode: "boolean" })
             .notNull()
             .default(false),
@@ -220,10 +220,14 @@ export const resources = sqliteTable(
         maintenanceEstimatedTime: text("maintenanceEstimatedTime"),
         postAuthPath: text("postAuthPath"),
         health: text("health").default("unknown"), // "healthy", "unhealthy", "unknown"
-        wildcard: integer("wildcard", { mode: "boolean" }).notNull().default(false),
+        wildcard: integer("wildcard", { mode: "boolean" })
+            .notNull()
+            .default(false),
         mode: text("mode")
             .default("http")
-            .$type<"rdp" | "ssh" | "http" | "vnc" | "inference" | "tcp" | "udp">()
+            .$type<
+                "rdp" | "ssh" | "http" | "vnc" | "inference" | "tcp" | "udp"
+            >()
             .notNull(), // rdp, ssh, http, vnc, inference
         pamMode: text("pamMode")
             .$type<"passthrough" | "push">()
@@ -236,9 +240,7 @@ export const resources = sqliteTable(
             .$type<"pending" | "approved">()
             .default("approved")
     },
-    (table) => [
-        index("idx_resources_orgId").on(table.orgId)
-    ]
+    (table) => [index("idx_resources_orgId").on(table.orgId)]
 );
 
 export const resourceAiProviders = sqliteTable(
@@ -288,9 +290,7 @@ export const labels = sqliteTable(
             })
             .notNull()
     },
-    (table) => [
-        index("idx_labels_orgId").on(table.orgId)
-    ]
+    (table) => [index("idx_labels_orgId").on(table.orgId)]
 );
 
 export const launcherViews = sqliteTable("launcherViews", {
@@ -409,7 +409,9 @@ export const targets = sqliteTable(
         method: text("method"),
         port: integer("port").notNull(),
         internalPort: integer("internalPort"),
-        enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
+        enabled: integer("enabled", { mode: "boolean" })
+            .notNull()
+            .default(true),
         path: text("path"),
         pathMatchType: text("pathMatchType"), // exact, prefix, regex
         rewritePath: text("rewritePath"), // if set, rewrites the path to this value before sending to the target
@@ -705,9 +707,7 @@ export const newts = sqliteTable(
             onDelete: "cascade"
         })
     },
-    (table) => [
-        index("idx_newts_siteId").on(table.siteId)
-    ]
+    (table) => [index("idx_newts_siteId").on(table.siteId)]
 );
 
 export const clients = sqliteTable(
@@ -740,8 +740,12 @@ export const clients = sqliteTable(
         online: integer("online", { mode: "boolean" }).notNull().default(false),
         // endpoint: text("endpoint"),
         lastHolePunch: integer("lastHolePunch"),
-        archived: integer("archived", { mode: "boolean" }).notNull().default(false),
-        blocked: integer("blocked", { mode: "boolean" }).notNull().default(false),
+        archived: integer("archived", { mode: "boolean" })
+            .notNull()
+            .default(false),
+        blocked: integer("blocked", { mode: "boolean" })
+            .notNull()
+            .default(false),
         approvalState: text("approvalState").$type<
             "pending" | "approved" | "denied"
         >()
@@ -795,11 +799,11 @@ export const olms = sqliteTable(
             // optionally tied to a user and in this case delete when the user deletes
             onDelete: "cascade"
         }),
-        archived: integer("archived", { mode: "boolean" }).notNull().default(false)
+        archived: integer("archived", { mode: "boolean" })
+            .notNull()
+            .default(false)
     },
-    (table) => [
-        index("idx_olms_userId").on(table.userId)
-    ]
+    (table) => [index("idx_olms_userId").on(table.userId)]
 );
 
 export const currentFingerprint = sqliteTable("currentFingerprint", {
@@ -975,9 +979,7 @@ export const sessions = sqliteTable(
             .notNull()
             .default(false)
     },
-    (table) => [
-        index("idx_sessions_userId").on(table.userId)
-    ]
+    (table) => [index("idx_sessions_userId").on(table.userId)]
 );
 
 export const newtSessions = sqliteTable("newtSession", {
@@ -1007,7 +1009,9 @@ export const userOrgs = sqliteTable(
                 onDelete: "cascade"
             })
             .notNull(),
-        isOwner: integer("isOwner", { mode: "boolean" }).notNull().default(false),
+        isOwner: integer("isOwner", { mode: "boolean" })
+            .notNull()
+            .default(false),
         autoProvisioned: integer("autoProvisioned", {
             mode: "boolean"
         }).default(false),
@@ -1062,14 +1066,12 @@ export const roles = sqliteTable(
         }).default(false),
         sshSudoMode: text("sshSudoMode").default("full"), // "none" | "full" | "commands"
         sshSudoCommands: text("sshSudoCommands").default("[]"),
-        sshCreateHomeDir: integer("sshCreateHomeDir", { mode: "boolean" }).default(
-            true
-        ),
+        sshCreateHomeDir: integer("sshCreateHomeDir", {
+            mode: "boolean"
+        }).default(true),
         sshUnixGroups: text("sshUnixGroups").default("[]")
     },
-    (table) => [
-        index("idx_roles_orgId").on(table.orgId)
-    ]
+    (table) => [index("idx_roles_orgId").on(table.orgId)]
 );
 
 export const userOrgRoles = sqliteTable(
@@ -1997,91 +1999,6 @@ export const aiBudgetBreachEvents = sqliteTable(
     ]
 );
 
-// Logs the aggregated prompt + response for a single AI gateway request, for
-// session replay. One row per request (not per streaming chunk). `sessionId`
-// is a fresh random id per row for now - no cross-request correlation yet,
-// but the column exists so a future pass can link multiple rows into a real
-// multi-turn session.
-export const aiSessionLog = sqliteTable(
-    "aiSessionLog",
-    {
-        id: integer("id").primaryKey({ autoIncrement: true }),
-        sessionId: text("sessionId").notNull(),
-        orgId: text("orgId").references(() => orgs.orgId, {
-            onDelete: "cascade"
-        }),
-        providerId: integer("providerId").references(
-            () => aiProviders.providerId,
-            { onDelete: "set null" }
-        ),
-        capability: text("capability").notNull(),
-        resourceId: integer("resourceId").references(
-            () => resources.resourceId,
-            { onDelete: "set null" }
-        ),
-        siteResourceId: integer("siteResourceId").references(
-            () => siteResources.siteResourceId,
-            { onDelete: "set null" }
-        ),
-        userId: text("userId").references(() => users.userId, {
-            onDelete: "set null"
-        }),
-        virtualApiKeyId: text("virtualApiKeyId").references(
-            () => virtualApiKeys.virtualApiKeyId,
-            { onDelete: "set null" }
-        ),
-        requestedModel: text("requestedModel"),
-        isStream: integer("isStream", { mode: "boolean" })
-            .notNull()
-            .default(false),
-        requestBody: text("requestBody"),
-        responseBody: text("responseBody"),
-        // Capability-agnostic message transcript (JSON-encoded
-        // NormalizedAiMessage[] from server/lib/aiMessageNormalization.ts),
-        // computed at write time so search/display never need per-capability
-        // parsing logic. Null when normalization couldn't recognize the
-        // shape - callers fall back to requestBody/responseBody.
-        normalizedRequest: text("normalizedRequest"),
-        normalizedResponse: text("normalizedResponse"),
-        // True if any of the request/response (raw or normalized) fields
-        // were cut short at AI_SESSION_LOG_MAX_BODY_CHARS before storage.
-        truncated: integer("truncated", { mode: "boolean" })
-            .notNull()
-            .default(false),
-        statusCode: integer("statusCode"),
-        createdAt: integer("createdAt").notNull() // epoch seconds
-    },
-    (t) => [
-        index("idx_ai_session_log_org_created").on(t.orgId, t.createdAt),
-        index("idx_ai_session_log_org_provider_created").on(
-            t.orgId,
-            t.providerId,
-            t.createdAt
-        ),
-        index("idx_ai_session_log_org_resource_created").on(
-            t.orgId,
-            t.resourceId,
-            t.createdAt
-        ),
-        index("idx_ai_session_log_org_site_resource_created").on(
-            t.orgId,
-            t.siteResourceId,
-            t.createdAt
-        ),
-        index("idx_ai_session_log_org_user_created").on(
-            t.orgId,
-            t.userId,
-            t.createdAt
-        ),
-        index("idx_ai_session_log_org_virtual_api_key_created").on(
-            t.orgId,
-            t.virtualApiKeyId,
-            t.createdAt
-        ),
-        index("idx_ai_session_log_session").on(t.sessionId)
-    ]
-);
-
 export const certificates = sqliteTable("certificates", {
     certId: integer("certId").primaryKey({ autoIncrement: true }),
     domain: text("domain").notNull().unique(),
@@ -2192,7 +2109,6 @@ export type AiModel = InferSelectModel<typeof aiModels>;
 export type AiBudget = InferSelectModel<typeof aiBudgets>;
 export type AiUsageRecord = InferSelectModel<typeof aiUsageRecords>;
 export type AiBudgetBreachEvent = InferSelectModel<typeof aiBudgetBreachEvents>;
-export type AiSessionLog = InferSelectModel<typeof aiSessionLog>;
 export type ResourceAiProvider = InferSelectModel<typeof resourceAiProviders>;
 export type SiteResourceAiProvider = InferSelectModel<
     typeof siteResourceAiProviders

server/lib/blueprints/applyBlueprint.ts

@@ -121,6 +121,13 @@ export async function applyBlueprint({
                             (hc) => hc.targetId === target.targetId
                         );
 
+                    // The DB writes for all resources have already committed
+                    // by this point, so a push failure for one target (e.g.
+                    // a newt rejecting a malformed health check) must not
+                    // abort pushing the rest, and must not mark the whole
+                    // blueprint as failed when the config was actually
+                    // persisted successfully.
+                    try {
                         if (["http", "tcp", "udp"].includes(target.mode)) {
                             await addProxyTargets(
                                 site.newt.newtId,
@@ -142,6 +149,11 @@ export async function applyBlueprint({
                                 site.newt.version
                             );
                         }
+                    } catch (e) {
+                        logger.error(
+                            `Failed to push target ${target.targetId} to newt on site ${site.sites.siteId}. Error: ${e}`
+                        );
+                    }
                 }
             }
         }

server/lib/blueprints/types.ts

@@ -29,8 +29,34 @@ export const SiteSchema = z.object({
     "docker-socket-enabled": z.boolean().optional().default(true)
 });
 
+// A malformed hostname (e.g. stray whitespace) is silently accepted here but
+// fails to parse as a URL when newt builds the health check request, which
+// takes the target out of the routing pool and breaks the resource entirely
+// (see #3677). Validate eagerly so blueprints reject it up front instead.
+const healthCheckHostnameSchema = z
+    .string()
+    .trim()
+    .min(1)
+    .refine((val) => !/\s/.test(val), {
+        message: "Hostname must not contain whitespace"
+    })
+    .refine(
+        (val) => {
+            if (z.union([z.ipv4(), z.ipv6()]).safeParse(val).success) {
+                return true;
+            }
+            const hostnameRegex =
+                /^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)*[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$/;
+            return hostnameRegex.test(val);
+        },
+        {
+            message:
+                "Hostname must be a valid IP address or hostname (no spaces or invalid characters)"
+        }
+    );
+
 export const TargetHealthCheckSchema = z.object({
-    hostname: z.string(),
+    hostname: healthCheckHostnameSchema,
     port: z.int().min(1).max(65535),
     enabled: z.boolean().optional().default(true),
     path: z.string().optional().default("/"),

server/private/lib/redis.ts

@@ -158,7 +158,7 @@ class RedisManager {
             this.writeClient = new Redis({
                 ...masterConfig,
                 enableReadyCheck: false,
-                maxRetriesPerRequest: 3,
+                maxRetriesPerRequest: 50,
                 keepAlive: 30000,
                 connectTimeout: this.connectionTimeout,
                 commandTimeout: this.commandTimeout
@@ -169,7 +169,7 @@ class RedisManager {
                 this.readClient = new Redis({
                     ...replicaConfig!,
                     enableReadyCheck: false,
-                    maxRetriesPerRequest: 3,
+                    maxRetriesPerRequest: 50,
                     keepAlive: 30000,
                     connectTimeout: this.connectionTimeout,
                     commandTimeout: this.commandTimeout
@@ -186,7 +186,7 @@ class RedisManager {
             this.publisher = new Redis({
                 ...masterConfig,
                 enableReadyCheck: false,
-                maxRetriesPerRequest: 3,
+                maxRetriesPerRequest: 50,
                 keepAlive: 30000,
                 connectTimeout: this.connectionTimeout,
                 commandTimeout: this.commandTimeout
@@ -196,7 +196,7 @@ class RedisManager {
             this.subscriber = new Redis({
                 ...(this.hasReplicas ? replicaConfig! : masterConfig),
                 enableReadyCheck: false,
-                maxRetriesPerRequest: 3,
+                maxRetriesPerRequest: 50,
                 keepAlive: 30000,
                 connectTimeout: this.connectionTimeout,
                 commandTimeout: this.commandTimeout
@@ -901,7 +901,9 @@ class RegionalRedisManager {
     // if the configured host doesn't match that pattern (e.g. local dev),
     // in which case callers should fall back to the primary for reads.
     private getReplicaHost(primaryHost: string): string | null {
-        const match = primaryHost.match(/^redis\.([^.]+)\.svc\.cluster\.local$/);
+        const match = primaryHost.match(
+            /^redis\.([^.]+)\.svc\.cluster\.local$/
+        );
         if (!match) return null;
         const namespace = match[1];
         return `redis-1.redis-headless.${namespace}.svc.cluster.local`;
@@ -912,7 +914,7 @@ class RegionalRedisManager {
         const baseOpts = {
             ...cfg,
             enableReadyCheck: false,
-            maxRetriesPerRequest: 3,
+            maxRetriesPerRequest: 50,
             keepAlive: 10000,
             connectTimeout: this.connectionTimeout,
             commandTimeout: this.commandTimeout

server/routers/external.ts

@@ -1378,12 +1378,6 @@ if (build !== "saas") {
         user.adminGeneratePasswordResetCode
     );
 
-    authenticated.post(
-        "/user/:userId/promote-server-admin",
-        verifyUserIsServerAdmin,
-        user.adminPromoteServerAdmin
-    );
-
     authenticated.delete(
         "/user/:userId",
         verifyUserIsServerAdmin,

server/routers/user/adminListUsers.ts

@@ -4,7 +4,7 @@ import { db, idp, users } from "@server/db";
 import response from "@server/lib/response";
 import HttpCode from "@server/types/HttpCode";
 import createHttpError from "http-errors";
-import { and, asc, desc, eq, like, or, sql, type SQL } from "drizzle-orm";
+import { and, asc, desc, eq, like, or, sql } from "drizzle-orm";
 import logger from "@server/logger";
 import { fromZodError } from "zod-validation-error";
 import { OpenAPITags, registry } from "@server/openApi";
@@ -196,7 +196,7 @@ export async function adminListUsers(
             }
         }
 
-        const conditions: Array<SQL<unknown> | undefined> = [];
+        const conditions = [eq(users.serverAdmin, false)];
 
         if (query) {
             const q = "%" + query.toLowerCase() + "%";

server/routers/user/adminPromoteServerAdmin.ts

@@ -1,116 +0,0 @@
-import { Request, Response, NextFunction } from "express";
-import { z } from "zod";
-import { db, users } from "@server/db";
-import { eq } from "drizzle-orm";
-import response from "@server/lib/response";
-import HttpCode from "@server/types/HttpCode";
-import createHttpError from "http-errors";
-import logger from "@server/logger";
-import { fromError } from "zod-validation-error";
-import { OpenAPITags, registry } from "@server/openApi";
-import { createApiResponseSchema } from "@server/lib/openapi/createApiResponseSchema";
-
-const promoteServerAdminParamsSchema = z.strictObject({
-    userId: z.string()
-});
-
-export type AdminPromoteServerAdminResponse = {
-    userId: string;
-    serverAdmin: boolean;
-};
-
-const AdminPromoteServerAdminResponseDataSchema = z.object({
-    userId: z.string(),
-    serverAdmin: z.boolean()
-});
-
-registry.registerPath({
-    method: "post",
-    path: "/user/{userId}/promote-server-admin",
-    description: "Promote a user to server admin (server admin).",
-    tags: [OpenAPITags.User],
-    request: {
-        params: promoteServerAdminParamsSchema
-    },
-    responses: {
-        200: {
-            description: "Successful response",
-            content: {
-                "application/json": {
-                    schema: createApiResponseSchema(
-                        AdminPromoteServerAdminResponseDataSchema
-                    )
-                }
-            }
-        }
-    }
-});
-
-export async function adminPromoteServerAdmin(
-    req: Request,
-    res: Response,
-    next: NextFunction
-): Promise<any> {
-    try {
-        const parsedParams = promoteServerAdminParamsSchema.safeParse(
-            req.params
-        );
-        if (!parsedParams.success) {
-            return next(
-                createHttpError(
-                    HttpCode.BAD_REQUEST,
-                    fromError(parsedParams.error).toString()
-                )
-            );
-        }
-
-        const { userId } = parsedParams.data;
-
-        const [existingUser] = await db
-            .select({
-                userId: users.userId,
-                serverAdmin: users.serverAdmin
-            })
-            .from(users)
-            .where(eq(users.userId, userId))
-            .limit(1);
-
-        if (!existingUser) {
-            return next(createHttpError(HttpCode.NOT_FOUND, "User not found"));
-        }
-
-        if (existingUser.serverAdmin) {
-            return next(
-                createHttpError(
-                    HttpCode.BAD_REQUEST,
-                    "User is already a server admin"
-                )
-            );
-        }
-
-        logger.info(
-            `Promoting user ${userId} to server admin (by ${req.user?.userId})`
-        );
-
-        await db
-            .update(users)
-            .set({ serverAdmin: true })
-            .where(eq(users.userId, userId));
-
-        return response<AdminPromoteServerAdminResponse>(res, {
-            data: {
-                userId: existingUser.userId,
-                serverAdmin: true
-            },
-            success: true,
-            error: false,
-            message: "User promoted to server admin successfully",
-            status: HttpCode.OK
-        });
-    } catch (error) {
-        logger.error(error);
-        return next(
-            createHttpError(HttpCode.INTERNAL_SERVER_ERROR, "An error occurred")
-        );
-    }
-}

server/routers/user/index.ts

@@ -11,7 +11,6 @@ export * from "./adminListUsers";
 export * from "./adminRemoveUser";
 export * from "./adminGetUser";
 export * from "./adminGeneratePasswordResetCode";
-export * from "./adminPromoteServerAdmin";
 export * from "./listInvitations";
 export * from "./removeInvitation";
 export * from "./createOrgUser";

src/app/admin/users/page.tsx

@@ -81,9 +81,6 @@ export default async function UsersPage(props: AdminUsersPageProps) {
         };
     });
 
-    console.log({
-        userRows
-    });
     return (
         <>
             <SettingsSectionTitle

src/components/AdminUsersTable.tsx

@@ -19,8 +19,7 @@ import {
     ArrowRight,
     ArrowUp10Icon,
     ChevronsUpDownIcon,
-    MoreHorizontal,
-    ShieldUserIcon
+    MoreHorizontal
 } from "lucide-react";
 import { useTranslations } from "next-intl";
 import { useRouter } from "next/navigation";
@@ -44,14 +43,6 @@ import {
     CredenzaClose
 } from "@app/components/Credenza";
 import CopyToClipboard from "@app/components/CopyToClipboard";
-import { Badge } from "./ui/badge";
-import {
-    Tooltip,
-    TooltipContent,
-    TooltipProvider,
-    TooltipTrigger
-} from "./ui/tooltip";
-import { useUserContext } from "@app/hooks/useUserContext";
 
 export type GlobalUserRow = {
     id: string;
@@ -99,9 +90,6 @@ export default function UsersTable({
     const [passwordResetCodeData, setPasswordResetCodeData] =
         useState<AdminGeneratePasswordResetCodeResponse | null>(null);
     const [isGeneratingCode, setIsGeneratingCode] = useState(false);
-    const [isPromoteModalOpen, setIsPromoteModalOpen] = useState(false);
-    const [promoting, setPromoting] = useState<GlobalUserRow | null>(null);
-    const user = useUserContext();
 
     const [isRefreshing, startTransition] = useTransition();
     const {
@@ -196,37 +184,6 @@ export default function UsersTable({
         }
     };
 
-    const promoteToServerAdmin = async (user: GlobalUserRow) => {
-        try {
-            await api.post(`/user/${user.id}/promote-server-admin`);
-
-            toast({
-                title: t("promoteServerAdminSuccess"),
-                description: t("promoteServerAdminSuccessDescription", {
-                    selectedUser: getUserDisplayName({
-                        email: user.email,
-                        name: user.name,
-                        username: user.username
-                    })
-                })
-            });
-
-            startTransition(() => {
-                router.refresh();
-            });
-        } catch (e) {
-            console.error(t("promoteServerAdminError"), e);
-            toast({
-                variant: "destructive",
-                title: t("promoteServerAdminError"),
-                description: formatAxiosError(e, t("promoteServerAdminError"))
-            });
-        } finally {
-            setIsPromoteModalOpen(false);
-            setPromoting(null);
-        }
-    };
-
     function toggleSort(column: string) {
         const newSearch = getNextSortOrder(column, searchParams);
         filter({
@@ -278,35 +235,7 @@ export default function UsersTable({
                         <Icon className="ml-2 h-4 w-4" />
                     </Button>
                 );
-            },
-            cell: ({ row }) => (
-                <span className="inline-flex gap-1 items-center">
-                    {row.original.username}{" "}
-                    {row.original.id === user.user.userId && (
-                        <>
-                            <span className="text-muted-foreground">
-                                &middot;
-                            </span>{" "}
-                            <span className="text-primary">you</span>
-                        </>
-                    )}
-                    {row.original.serverAdmin && (
-                        <>
-                            <TooltipProvider>
-                                <Tooltip>
-                                    <TooltipTrigger asChild>
-                                        <ShieldUserIcon className="text-primary size-5 flex-none" />
-                                    </TooltipTrigger>
-                                    <TooltipContent>
-                                        {t("serverAdmin")}
-                                    </TooltipContent>
-                                </Tooltip>
-                            </TooltipProvider>
-                            {/*  <Badge>{t("serverAdmin")}</Badge> */}
-                        </>
-                    )}
-                </span>
-            )
+            }
         },
         {
             accessorKey: "email",
@@ -440,22 +369,11 @@ export default function UsersTable({
                                         {t("generatePasswordResetCode")}
                                     </DropdownMenuItem>
                                 )}
-                                {!r.serverAdmin && (
-                                    <DropdownMenuItem
-                                        onClick={() => {
-                                            setPromoting(r);
-                                            setIsPromoteModalOpen(true);
-                                        }}
-                                    >
-                                        {t("promoteServerAdmin")}
-                                    </DropdownMenuItem>
-                                )}
                                 <DropdownMenuItem
                                     onClick={() => {
                                         setSelected(r);
                                         setIsDeleteModalOpen(true);
                                     }}
-                                    className="text-red-400"
                                 >
                                     {t("delete")}
                                 </DropdownMenuItem>
@@ -517,42 +435,6 @@ export default function UsersTable({
                 />
             )}
 
-            {promoting && (
-                <ConfirmDeleteDialog
-                    open={isPromoteModalOpen}
-                    setOpen={(val) => {
-                        setIsPromoteModalOpen(val);
-                        if (!val) {
-                            setPromoting(null);
-                        }
-                    }}
-                    dialog={
-                        <div className="space-y-2">
-                            <p>
-                                {t("promoteServerAdminQuestion", {
-                                    selectedUser: getUserDisplayName({
-                                        email: promoting.email,
-                                        name: promoting.name,
-                                        username: promoting.username
-                                    })
-                                })}
-                            </p>
-
-                            <p>{t("promoteServerAdminMessage")}</p>
-                        </div>
-                    }
-                    buttonText={t("promoteServerAdminConfirm")}
-                    onConfirm={async () => promoteToServerAdmin(promoting)}
-                    string={getUserDisplayName({
-                        email: promoting.email,
-                        name: promoting.name,
-                        username: promoting.username
-                    })}
-                    warningText={t("promoteServerAdminWarning")}
-                    title={t("promoteServerAdminTitle")}
-                />
-            )}
-
             <ControlledDataTable
                 columns={columns}
                 rows={users}

src/components/HealthCheckCredenza.tsx

@@ -172,7 +172,6 @@ export function HealthCheckCredenza(props: HealthCheckCredenzaProps) {
                 .nullable()
                 .optional(),
             hcScheme: z.string().optional(),
-            hcHostname: z.string(),
             hcPort: z
                 .string()
                 .min(1, { message: t("healthCheckPortInvalid") })
@@ -184,6 +183,11 @@ export function HealthCheckCredenza(props: HealthCheckCredenzaProps) {
                     { message: t("healthCheckPortInvalid") }
                 ),
             hcFollowRedirects: z.boolean(),
+            hcHostname: z
+                .string()
+                .refine((val) => !/\s/.test(val), {
+                    message: t("healthCheckHostnameInvalid")
+                }),
             hcMode: z.string(),
             hcUnhealthyInterval: z.int().positive().min(5),
             hcTlsServerName: z.string(),

src/components/command-palette/CommandPalette.tsx

@@ -155,13 +155,13 @@ export function CommandPalette({ orgId, orgs, navItems }: CommandPaletteProps) {
                 }
             />
             <CommandList className="max-h-118 min-h-0  h-(--cmdk-list-height) scroll-pb-4 scroll-pt-2 transition-[height] duration-250 ease-in-out">
+                <CommandEmpty>{t("commandPaletteNoResults")}</CommandEmpty>
+
                 <CommandGroup
                     heading={t("commandActionModeInfo")}
                     className="[&_[cmdk-group-heading]]:text-sm"
                 />
 
-                <CommandEmpty>{t("commandPaletteNoResults")}</CommandEmpty>
-
                 {!isActionMode &&
                     navigationGroups.map((group, groupIndex) => (
                         <React.Fragment key={group.heading}>