Add resource degraded

server/private/lib/alerts/events/healthCheckEvents.ts

@@ -23,6 +23,7 @@ import {
 } from "@server/db";
 import { eq } from "drizzle-orm";
 import {
+    fireResourceDegradedAlert,
     fireResourceHealthyAlert,
     fireResourceUnhealthyAlert
 } from "./resourceEvents";
@@ -217,6 +218,14 @@ async function handleResource(orgId: string, healthCheckTargetId?: number | null
                 undefined,
                 trx
             );
+        } else if (health === "degraded") {
+            await fireResourceDegradedAlert(
+                orgId,
+                resource.resourceId,
+                resource.name,
+                undefined,
+                trx
+            );
         }
     }
 }

server/private/lib/alerts/events/resourceEvents.ts

@@ -130,9 +130,9 @@ export async function fireResourceUnhealthyAlert(
 }
 
 /**
- * Fire a `resource_toggle` alert for the given resource.
+ * Fire a `resource_degraded` alert for the given resource.
  *
- * Call this when a resource's enabled/disabled status is toggled so that any
+ * Call this after a resource has been detected as degraded so that any
  * matching `alertRules` can dispatch their email and webhook actions.
  *
  * @param orgId        - Organisation that owns the resource.
@@ -140,7 +140,7 @@ export async function fireResourceUnhealthyAlert(
  * @param resourceName - Human-readable name shown in notifications (optional).
  * @param extra        - Any additional key/value pairs to include in the payload.
  */
-export async function fireResourceToggleAlert(
+export async function fireResourceDegradedAlert(
     orgId: string,
     resourceId: number,
     resourceName?: string | null,
@@ -148,8 +148,16 @@ export async function fireResourceToggleAlert(
     trx: Transaction | typeof db = db
 ): Promise<void> {
     try {
+        await trx.insert(statusHistory).values({
+            entityType: "resource",
+            entityId: resourceId,
+            orgId: orgId,
+            status: "degraded",
+            timestamp: Math.floor(Date.now() / 1000)
+        });
+
         await processAlerts({
-            eventType: "resource_toggle",
+            eventType: "resource_degraded",
             orgId,
             resourceId,
             data: {
@@ -157,9 +165,20 @@ export async function fireResourceToggleAlert(
                 ...extra
             }
         });
+        await processAlerts({
+            eventType: "resource_toggle",
+            orgId,
+            resourceId,
+            data: {
+                resourceId,
+                status: "degraded",
+                ...(resourceName != null ? { resourceName } : {}),
+                ...extra
+            }
+        });
     } catch (err) {
         logger.error(
-            `fireResourceToggleAlert: unexpected error for resourceId ${resourceId}`,
+            `fireResourceDegradedAlert: unexpected error for resourceId ${resourceId}`,
             err
         );
     }

server/private/lib/alerts/sendAlertEmail.ts

@@ -88,6 +88,8 @@ function buildSubject(context: AlertContext): string {
             return "[Alert] Resource Healthy";
         case "resource_unhealthy":
             return "[Alert] Resource Unhealthy";
+        case "resource_degraded":
+            return "[Alert] Resource Degraded";
         case "resource_toggle":
             return "[Alert] Resource Status Changed";
         default: {

server/private/lib/alerts/sendAlertWebhook.ts

@@ -12,7 +12,10 @@
  */
 
 import logger from "@server/logger";
-import { AlertContext, WebhookAlertConfig } from "@server/routers/alertRule/types";
+import {
+    AlertContext,
+    WebhookAlertConfig
+} from "@server/routers/alertRule/types";
 
 const REQUEST_TIMEOUT_MS = 15_000;
 const MAX_RETRIES = 3;
@@ -56,7 +59,10 @@ export async function sendAlertWebhook(
 
     for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
         const controller = new AbortController();
-        const timeoutHandle = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
+        const timeoutHandle = setTimeout(
+            () => controller.abort(),
+            REQUEST_TIMEOUT_MS
+        );
 
         let response: Response;
         try {
@@ -75,7 +81,9 @@ export async function sendAlertWebhook(
                 );
             } else {
                 const msg = err instanceof Error ? err.message : String(err);
-                lastError = new Error(`Alert webhook: request to "${url}" failed – ${msg}`);
+                lastError = new Error(
+                    `Alert webhook: request to "${url}" failed – ${msg}`
+                );
             }
             if (attempt < MAX_RETRIES) {
                 const delay = RETRY_BASE_DELAY_MS * 2 ** (attempt - 1);
@@ -111,11 +119,18 @@ export async function sendAlertWebhook(
             continue;
         }
 
-        logger.debug(`Alert webhook sent successfully to "${url}" for event "${context.eventType}" (attempt ${attempt}/${MAX_RETRIES})`);
+        logger.debug(
+            `Alert webhook sent successfully to "${url}" for event "${context.eventType}" (attempt ${attempt}/${MAX_RETRIES})`
+        );
         return;
     }
 
-    throw lastError ?? new Error(`Alert webhook: all ${MAX_RETRIES} attempts failed for "${url}"`);
+    throw (
+        lastError ??
+        new Error(
+            `Alert webhook: all ${MAX_RETRIES} attempts failed for "${url}"`
+        )
+    );
 }
 
 // ---------------------------------------------------------------------------
@@ -139,6 +154,8 @@ function deriveStatus(
         case "health_check_unhealthy":
         case "resource_unhealthy":
             return "unhealthy";
+        case "resource_degraded":
+            return "degraded";
         case "health_check_toggle":
         case "resource_toggle":
             return String(data.status ?? "unknown");
@@ -154,7 +171,9 @@ function deriveStatus(
 // Header construction (mirrors HttpLogDestination.buildHeaders)
 // ---------------------------------------------------------------------------
 
-function buildHeaders(webhookConfig: WebhookAlertConfig): Record<string, string> {
+function buildHeaders(
+    webhookConfig: WebhookAlertConfig
+): Record<string, string> {
     const headers: Record<string, string> = {
         "Content-Type": "application/json"
     };