Add resource degraded

messages/en-US.json

@@ -1432,6 +1432,7 @@
     "alertingTriggerHcToggle": "Health check status changes",
     "alertingTriggerResourceHealthy": "Resource healthy",
     "alertingTriggerResourceUnhealthy": "Resource unhealthy",
+    "alertingTriggerResourceDegraded": "Resource degraded",
     "alertingSearchHealthChecks": "Search health checks…",
     "alertingHealthChecksEmpty": "No health checks available.",
     "alertingTriggerResourceToggle": "Resource status changes",

server/db/pg/schema/privateSchema.ts

@@ -484,6 +484,7 @@ export const alertRules = pgTable("alertRules", {
             | "health_check_toggle"
             | "resource_healthy"
             | "resource_unhealthy"
+            | "resource_degraded"
             | "resource_toggle"
         >()
         .notNull(),

server/db/sqlite/schema/privateSchema.ts

@@ -425,10 +425,18 @@ export const eventStreamingDestinations = sqliteTable(
         orgId: text("orgId")
             .notNull()
             .references(() => orgs.orgId, { onDelete: "cascade" }),
-        sendConnectionLogs: integer("sendConnectionLogs", { mode: "boolean" }).notNull().default(false),
+        sendConnectionLogs: integer("sendConnectionLogs", { mode: "boolean" })
-        sendRequestLogs: integer("sendRequestLogs", { mode: "boolean" }).notNull().default(false),
+            .notNull()
-        sendActionLogs: integer("sendActionLogs", { mode: "boolean" }).notNull().default(false),
+            .default(false),
-        sendAccessLogs: integer("sendAccessLogs", { mode: "boolean" }).notNull().default(false),
+        sendRequestLogs: integer("sendRequestLogs", { mode: "boolean" })
+            .notNull()
+            .default(false),
+        sendActionLogs: integer("sendActionLogs", { mode: "boolean" })
+            .notNull()
+            .default(false),
+        sendAccessLogs: integer("sendAccessLogs", { mode: "boolean" })
+            .notNull()
+            .default(false),
         type: text("type").notNull(), // e.g. "http", "kafka", etc.
         config: text("config").notNull(), // JSON string with the configuration for the destination
         enabled: integer("enabled", { mode: "boolean" })
@@ -476,14 +484,19 @@ export const alertRules = sqliteTable("alertRules", {
             | "health_check_toggle"
             | "resource_healthy"
             | "resource_unhealthy"
+            | "resource_degraded"
             | "resource_toggle"
         >()
         .notNull(),
     enabled: integer("enabled", { mode: "boolean" }).notNull().default(true),
     cooldownSeconds: integer("cooldownSeconds").notNull().default(300),
     allSites: integer("allSites", { mode: "boolean" }).notNull().default(false),
-    allHealthChecks: integer("allHealthChecks", { mode: "boolean" }).notNull().default(false),
+    allHealthChecks: integer("allHealthChecks", { mode: "boolean" })
-    allResources: integer("allResources", { mode: "boolean" }).notNull().default(false),
+        .notNull()
+        .default(false),
+    allResources: integer("allResources", { mode: "boolean" })
+        .notNull()
+        .default(false),
     lastTriggeredAt: integer("lastTriggeredAt"),
     createdAt: integer("createdAt").notNull(),
     updatedAt: integer("updatedAt").notNull()
@@ -531,14 +544,22 @@ export const alertEmailRecipients = sqliteTable("alertEmailRecipients", {
     recipientId: integer("recipientId").primaryKey({ autoIncrement: true }),
     emailActionId: integer("emailActionId")
         .notNull()
-        .references(() => alertEmailActions.emailActionId, { onDelete: "cascade" }),
+        .references(() => alertEmailActions.emailActionId, {
-    userId: text("userId").references(() => users.userId, { onDelete: "cascade" }),
+            onDelete: "cascade"
-    roleId: integer("roleId").references(() => roles.roleId, { onDelete: "cascade" }),
+        }),
+    userId: text("userId").references(() => users.userId, {
+        onDelete: "cascade"
+    }),
+    roleId: integer("roleId").references(() => roles.roleId, {
+        onDelete: "cascade"
+    }),
     email: text("email")
 });
 
 export const alertWebhookActions = sqliteTable("alertWebhookActions", {
-    webhookActionId: integer("webhookActionId").primaryKey({ autoIncrement: true }),
+    webhookActionId: integer("webhookActionId").primaryKey({
+        autoIncrement: true
+    }),
     alertRuleId: integer("alertRuleId")
         .notNull()
         .references(() => alertRules.alertRuleId, { onDelete: "cascade" }),

server/emails/templates/AlertNotification.tsx

@@ -23,6 +23,7 @@ export type AlertEventType =
     | "health_check_toggle"
     | "resource_healthy"
     | "resource_unhealthy"
+    | "resource_degraded"
     | "resource_toggle";
 
 export type AlertNotificationProps = {
@@ -114,6 +115,15 @@ function getEventMeta(eventType: AlertEventType): {
                 statusLabel: "Unhealthy",
                 statusColor: "#dc2626"
             };
+        case "resource_degraded":
+            return {
+                heading: "Resource Unhealthy",
+                previewText: "A resource in your organization is not healthy.",
+                summary:
+                    "A resource in your organization is currently unhealthy.",
+                statusLabel: "Unhealthy",
+                statusColor: "#dc2626"
+            };
         case "resource_toggle":
             return {
                 heading: "Resource Status Changed",
@@ -135,7 +145,10 @@ function getEventMeta(eventType: AlertEventType): {
     }
 }
 
-function resolveToggleStatus(status: unknown): { label: string; color: string } {
+function resolveToggleStatus(status: unknown): {
+    label: string;
+    color: string;
+} {
     switch (String(status).toLowerCase()) {
         case "online":
             return { label: "Online", color: "#16a34a" };

server/private/lib/alerts/events/healthCheckEvents.ts

@@ -23,6 +23,7 @@ import {
 } from "@server/db";
 import { eq } from "drizzle-orm";
 import {
+    fireResourceDegradedAlert,
     fireResourceHealthyAlert,
     fireResourceUnhealthyAlert
 } from "./resourceEvents";
@@ -217,6 +218,14 @@ async function handleResource(orgId: string, healthCheckTargetId?: number | null
                 undefined,
                 trx
             );
+        } else if (health === "degraded") {
+            await fireResourceDegradedAlert(
+                orgId,
+                resource.resourceId,
+                resource.name,
+                undefined,
+                trx
+            );
         }
     }
 }

server/private/lib/alerts/events/resourceEvents.ts

@@ -130,9 +130,9 @@ export async function fireResourceUnhealthyAlert(
 }
 
 /**
- * Fire a `resource_toggle` alert for the given resource.
+ * Fire a `resource_degraded` alert for the given resource.
  *
- * Call this when a resource's enabled/disabled status is toggled so that any
+ * Call this after a resource has been detected as degraded so that any
  * matching `alertRules` can dispatch their email and webhook actions.
  *
  * @param orgId        - Organisation that owns the resource.
@@ -140,7 +140,7 @@ export async function fireResourceUnhealthyAlert(
  * @param resourceName - Human-readable name shown in notifications (optional).
  * @param extra        - Any additional key/value pairs to include in the payload.
  */
-export async function fireResourceToggleAlert(
+export async function fireResourceDegradedAlert(
     orgId: string,
     resourceId: number,
     resourceName?: string | null,
@@ -148,8 +148,16 @@ export async function fireResourceToggleAlert(
     trx: Transaction | typeof db = db
 ): Promise<void> {
     try {
+        await trx.insert(statusHistory).values({
+            entityType: "resource",
+            entityId: resourceId,
+            orgId: orgId,
+            status: "degraded",
+            timestamp: Math.floor(Date.now() / 1000)
+        });
+
         await processAlerts({
-            eventType: "resource_toggle",
+            eventType: "resource_degraded",
             orgId,
             resourceId,
             data: {
@@ -157,9 +165,20 @@ export async function fireResourceToggleAlert(
                 ...extra
             }
         });
+        await processAlerts({
+            eventType: "resource_toggle",
+            orgId,
+            resourceId,
+            data: {
+                resourceId,
+                status: "degraded",
+                ...(resourceName != null ? { resourceName } : {}),
+                ...extra
+            }
+        });
     } catch (err) {
         logger.error(
-            `fireResourceToggleAlert: unexpected error for resourceId ${resourceId}`,
+            `fireResourceDegradedAlert: unexpected error for resourceId ${resourceId}`,
             err
         );
     }

server/private/lib/alerts/sendAlertEmail.ts

@@ -88,6 +88,8 @@ function buildSubject(context: AlertContext): string {
             return "[Alert] Resource Healthy";
         case "resource_unhealthy":
             return "[Alert] Resource Unhealthy";
+        case "resource_degraded":
+            return "[Alert] Resource Degraded";
         case "resource_toggle":
             return "[Alert] Resource Status Changed";
         default: {

server/private/lib/alerts/sendAlertWebhook.ts

@@ -12,7 +12,10 @@
  */
 
 import logger from "@server/logger";
-import { AlertContext, WebhookAlertConfig } from "@server/routers/alertRule/types";
+import {
+    AlertContext,
+    WebhookAlertConfig
+} from "@server/routers/alertRule/types";
 
 const REQUEST_TIMEOUT_MS = 15_000;
 const MAX_RETRIES = 3;
@@ -56,7 +59,10 @@ export async function sendAlertWebhook(
 
     for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
         const controller = new AbortController();
-        const timeoutHandle = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
+        const timeoutHandle = setTimeout(
+            () => controller.abort(),
+            REQUEST_TIMEOUT_MS
+        );
 
         let response: Response;
         try {
@@ -75,7 +81,9 @@ export async function sendAlertWebhook(
                 );
             } else {
                 const msg = err instanceof Error ? err.message : String(err);
-                lastError = new Error(`Alert webhook: request to "${url}" failed – ${msg}`);
+                lastError = new Error(
+                    `Alert webhook: request to "${url}" failed – ${msg}`
+                );
             }
             if (attempt < MAX_RETRIES) {
                 const delay = RETRY_BASE_DELAY_MS * 2 ** (attempt - 1);
@@ -111,11 +119,18 @@ export async function sendAlertWebhook(
             continue;
         }
 
-        logger.debug(`Alert webhook sent successfully to "${url}" for event "${context.eventType}" (attempt ${attempt}/${MAX_RETRIES})`);
+        logger.debug(
+            `Alert webhook sent successfully to "${url}" for event "${context.eventType}" (attempt ${attempt}/${MAX_RETRIES})`
+        );
         return;
     }
 
-    throw lastError ?? new Error(`Alert webhook: all ${MAX_RETRIES} attempts failed for "${url}"`);
+    throw (
+        lastError ??
+        new Error(
+            `Alert webhook: all ${MAX_RETRIES} attempts failed for "${url}"`
+        )
+    );
 }
 
 // ---------------------------------------------------------------------------
@@ -139,6 +154,8 @@ function deriveStatus(
         case "health_check_unhealthy":
         case "resource_unhealthy":
             return "unhealthy";
+        case "resource_degraded":
+            return "degraded";
         case "health_check_toggle":
         case "resource_toggle":
             return String(data.status ?? "unknown");
@@ -154,7 +171,9 @@ function deriveStatus(
 // Header construction (mirrors HttpLogDestination.buildHeaders)
 // ---------------------------------------------------------------------------
 
-function buildHeaders(webhookConfig: WebhookAlertConfig): Record<string, string> {
+function buildHeaders(
+    webhookConfig: WebhookAlertConfig
+): Record<string, string> {
     const headers: Record<string, string> = {
         "Content-Type": "application/json"
     };

server/private/routers/alertEvents/triggerResourceAlert.ts

@@ -24,7 +24,8 @@ import { eq, and } from "drizzle-orm";
 import {
     fireResourceHealthyAlert,
     fireResourceUnhealthyAlert,
-    fireResourceToggleAlert
+    fireResourceToggleAlert,
+    fireResourceDegradedAlert
 } from "#private/lib/alerts/events/resourceEvents";
 
 const paramsSchema = z.strictObject({
@@ -33,7 +34,12 @@ const paramsSchema = z.strictObject({
 });
 
 const bodySchema = z.strictObject({
-    eventType: z.enum(["resource_healthy", "resource_unhealthy", "resource_toggle"])
+    eventType: z.enum([
+        "resource_healthy",
+        "resource_unhealthy",
+        "resource_degraded",
+        "resource_toggle"
+    ])
 });
 
 export type TriggerResourceAlertResponse = {
@@ -101,8 +107,8 @@ export async function triggerResourceAlert(
                 resourceId,
                 resource.name ?? undefined
             );
-        } else {
+        } else if (eventType === "resource_degraded") {
-            await fireResourceToggleAlert(
+            await fireResourceDegradedAlert(
                 orgId,
                 resourceId,
                 resource.name ?? undefined

server/private/routers/alertRule/createAlertRule.ts

@@ -33,7 +33,11 @@ import { encrypt } from "@server/lib/crypto";
 import config from "@server/lib/config";
 import { CreateAlertRuleResponse } from "@server/routers/alertRule/types";
 
-export const SITE_EVENT_TYPES = ["site_online", "site_offline", "site_toggle"] as const;
+export const SITE_EVENT_TYPES = [
+    "site_online",
+    "site_offline",
+    "site_toggle"
+] as const;
 export const HC_EVENT_TYPES = [
     "health_check_healthy",
     "health_check_unhealthy",
@@ -42,6 +46,7 @@ export const HC_EVENT_TYPES = [
 export const RESOURCE_EVENT_TYPES = [
     "resource_healthy",
     "resource_unhealthy",
+    "resource_degraded",
     "resource_toggle"
 ] as const;
 
@@ -92,19 +97,24 @@ const bodySchema = z
         const isHcEvent = (HC_EVENT_TYPES as readonly string[]).includes(
             val.eventType
         );
-        const isResourceEvent = (RESOURCE_EVENT_TYPES as readonly string[]).includes(
+        const isResourceEvent = (
-            val.eventType
+            RESOURCE_EVENT_TYPES as readonly string[]
-        );
+        ).includes(val.eventType);
 
         if (isSiteEvent && !val.allSites && val.siteIds.length === 0) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
-                message: "At least one siteId is required for site event types when allSites is false",
+                message:
+                    "At least one siteId is required for site event types when allSites is false",
                 path: ["siteIds"]
             });
         }
 
-        if (isHcEvent && !val.allHealthChecks && val.healthCheckIds.length === 0) {
+        if (
+            isHcEvent &&
+            !val.allHealthChecks &&
+            val.healthCheckIds.length === 0
+        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
                 message:
@@ -129,10 +139,15 @@ const bodySchema = z
             });
         }
 
-        if (isResourceEvent && !val.allResources && val.resourceIds.length === 0) {
+        if (
+            isResourceEvent &&
+            !val.allResources &&
+            val.resourceIds.length === 0
+        ) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
-                message: "At least one resourceId is required for resource event types when allResources is false",
+                message:
+                    "At least one resourceId is required for resource event types when allResources is false",
                 path: ["resourceIds"]
             });
         }
@@ -148,7 +163,8 @@ const bodySchema = z
         if (isResourceEvent && val.healthCheckIds.length > 0) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
-                message: "healthCheckIds must not be set for resource event types",
+                message:
+                    "healthCheckIds must not be set for resource event types",
                 path: ["healthCheckIds"]
             });
         }
@@ -164,7 +180,8 @@ const bodySchema = z
         if (isHcEvent && val.resourceIds.length > 0) {
             ctx.addIssue({
                 code: z.ZodIssueCode.custom,
-                message: "resourceIds must not be set for health check event types",
+                message:
+                    "resourceIds must not be set for health check event types",
                 path: ["resourceIds"]
             });
         }
@@ -284,9 +301,7 @@ export async function createAlertRule(
         // Create the email action pivot row and recipients if any recipients
         // were supplied (userIds, roleIds, or raw emails).
         const hasRecipients =
-            userIds.length > 0 ||
+            userIds.length > 0 || roleIds.length > 0 || emails.length > 0;
-            roleIds.length > 0 ||
-            emails.length > 0;
 
         if (hasRecipients) {
             const [emailActionRow] = await db

server/private/routers/alertRule/listAlertRules.ts

@@ -76,6 +76,7 @@ const SITE_ALERT_EVENT_TYPES = [
 const RESOURCE_ALERT_EVENT_TYPES = [
     "resource_healthy",
     "resource_unhealthy",
+    "resource_degraded",
     "resource_toggle"
 ] as const;
 

server/routers/alertRule/types.ts

@@ -37,6 +37,7 @@ export type GetAlertRuleResponse = {
         | "health_check_toggle"
         | "resource_healthy"
         | "resource_unhealthy"
+        | "resource_degraded"
         | "resource_toggle";
     enabled: boolean;
     cooldownSeconds: number;
@@ -94,6 +95,7 @@ export type AlertEventType =
     | "health_check_toggle"
     | "resource_healthy"
     | "resource_unhealthy"
+    | "resource_degraded"
     | "resource_toggle";
 
 // ---------------------------------------------------------------------------

src/components/AlertingRulesTable.tsx

@@ -118,6 +118,8 @@ function triggerLabel(rule: AlertRuleRow, t: (k: string) => string) {
             return t("alertingTriggerResourceHealthy");
         case "resource_unhealthy":
             return t("alertingTriggerResourceUnhealthy");
+        case "resource_degraded":
+            return t("alertingTriggerResourceDegraded");
         case "resource_toggle":
             return t("alertingTriggerResourceToggle");
         default:

src/components/ProxyResourcesTable.tsx

@@ -32,7 +32,6 @@ import { toast } from "@app/hooks/useToast";
 import { createApiClient, formatAxiosError } from "@app/lib/api";
 import { UpdateResourceResponse } from "@server/routers/resource";
 import type { PaginationState } from "@tanstack/react-table";
-import { useQuery } from "@tanstack/react-query";
 import { AxiosResponse } from "axios";
 import {
     ArrowDown01Icon,

src/components/alert-rule-editor/AlertRuleFields.tsx

@@ -1147,6 +1147,7 @@ export function AlertRuleSourceFields({
                                     if (
                                         curTrigger !== "resource_healthy" &&
                                         curTrigger !== "resource_unhealthy" &&
+                                        curTrigger !== "resource_degraded" &&
                                         curTrigger !== "resource_toggle"
                                     ) {
                                         setValue("trigger", "resource_toggle", {
@@ -1367,6 +1368,9 @@ export function AlertRuleTriggerFields({
                                     <SelectItem value="resource_unhealthy">
                                         {t("alertingTriggerResourceUnhealthy")}
                                     </SelectItem>
+                                    <SelectItem value="resource_degraded">
+                                        {t("alertingTriggerResourceDegraded")}
+                                    </SelectItem>
                                 </>
                             ) : (
                                 <>

src/lib/alertRuleForm.ts

@@ -25,6 +25,7 @@ export type AlertTrigger =
     | "health_check_toggle"
     | "resource_healthy"
     | "resource_unhealthy"
+    | "resource_degraded"
     | "resource_toggle";
 
 export type AlertRuleFormAction =
@@ -77,6 +78,7 @@ export type AlertRuleApiPayload = {
         | "health_check_toggle"
         | "resource_healthy"
         | "resource_unhealthy"
+        | "resource_degraded"
         | "resource_toggle";
     enabled: boolean;
     allSites: boolean;
@@ -160,6 +162,7 @@ export function buildFormSchema(t: (k: string) => string) {
                 "health_check_toggle",
                 "resource_healthy",
                 "resource_unhealthy",
+                "resource_degraded",
                 "resource_toggle"
             ]),
             actions: z.array(
@@ -243,6 +246,7 @@ export function buildFormSchema(t: (k: string) => string) {
             const resourceTriggers: AlertTrigger[] = [
                 "resource_healthy",
                 "resource_unhealthy",
+                "resource_degraded",
                 "resource_toggle"
             ];
             if (
@@ -344,7 +348,9 @@ export function alertRuleAllResourcesSelected(
     eventType: string,
     resourceIds: number[] | undefined
 ): boolean {
-    return eventType.startsWith("resource_") && (resourceIds?.length ?? 0) === 0;
+    return (
+        eventType.startsWith("resource_") && (resourceIds?.length ?? 0) === 0
+    );
 }
 
 export function alertRuleAllHealthChecksSelected(