Handle all of the alerting from the functions

server/private/lib/alerts/events/healthCheckEvents.ts

@@ -13,6 +13,18 @@
 
 import logger from "@server/logger";
 import { processAlerts } from "../processAlerts";
+import {
+    db,
+    statusHistory,
+    targetHealthCheck,
+    targets,
+    resources
+} from "@server/db";
+import { eq } from "drizzle-orm";
+import {
+    fireResourceHealthyAlert,
+    fireResourceUnhealthyAlert
+} from "./resourceEvents";
 
 // ---------------------------------------------------------------------------
 // Public API
@@ -33,9 +45,20 @@ export async function fireHealthCheckHealthyAlert(
     orgId: string,
     healthCheckId: number,
     healthCheckName?: string | null,
+    healthCheckTargetId?: number | null,
     extra?: Record<string, unknown>
 ): Promise<void> {
     try {
+        await db.insert(statusHistory).values({
+            entityType: "health_check",
+            entityId: healthCheckId,
+            orgId: orgId,
+            status: "healthy",
+            timestamp: Math.floor(Date.now() / 1000)
+        });
+
+        await handleResource(orgId, healthCheckTargetId);
+
         await processAlerts({
             eventType: "health_check_healthy",
             orgId,
@@ -78,9 +101,20 @@ export async function fireHealthCheckUnhealthyAlert(
     orgId: string,
     healthCheckId: number,
     healthCheckName?: string | null,
+    healthCheckTargetId?: number | null,
     extra?: Record<string, unknown>
 ): Promise<void> {
     try {
+        await db.insert(statusHistory).values({
+            entityType: "health_check",
+            entityId: healthCheckId,
+            orgId: orgId,
+            status: "unhealthy",
+            timestamp: Math.floor(Date.now() / 1000)
+        });
+
+        await handleResource(orgId, healthCheckTargetId);
+
         await processAlerts({
             eventType: "health_check_unhealthy",
             orgId,
@@ -107,3 +141,63 @@ export async function fireHealthCheckUnhealthyAlert(
         );
     }
 }
+
+async function handleResource(orgId: string, healthCheckTargetId?: number | null) {
+    if (!healthCheckTargetId) {
+        return;
+    }
+    // we have resources lets get them
+    const [target] = await db
+        .select()
+        .from(targets)
+        .where(eq(targets.targetId, healthCheckTargetId))
+        .limit(1);
+
+    if (!target) {
+        return;
+    }
+    const [resource] = await db
+        .select()
+        .from(resources)
+        .where(eq(resources.resourceId, target.resourceId))
+        .limit(1);
+
+    if (!resource) {
+        return;
+    }
+    const otherTargets = await db
+        .select({ hcHealth: targetHealthCheck.hcHealth })
+        .from(targets)
+        .where(eq(targets.resourceId, resource.resourceId));
+
+    let health = "healthy";
+    const allHealthy = otherTargets.every((t) => t.hcHealth === "healthy");
+    if (!allHealthy) {
+        logger.debug(
+            `Not marking resource ${resource.resourceId} as healthy because not all targets are healthy`
+        );
+        health = "unhealthy";
+    }
+
+    if (health != resource.health) {
+        // it changed
+        await db
+            .update(resources)
+            .set({ health })
+            .where(eq(resources.resourceId, resource.resourceId));
+
+        if (health === "unhealthy") {
+            await fireResourceUnhealthyAlert(
+                orgId,
+                resource.resourceId,
+                resource.name
+            );
+        } else if (health === "healthy") {
+            await fireResourceHealthyAlert(
+                orgId,
+                resource.resourceId,
+                resource.name
+            );
+        }
+    }
+}

server/private/lib/alerts/events/resourceEvents.ts

@@ -13,6 +13,7 @@
 
 import logger from "@server/logger";
 import { processAlerts } from "../processAlerts";
+import { db, statusHistory } from "@server/db";
 
 // ---------------------------------------------------------------------------
 // Public API
@@ -36,6 +37,14 @@ export async function fireResourceHealthyAlert(
     extra?: Record<string, unknown>
 ): Promise<void> {
     try {
+        await db.insert(statusHistory).values({
+            entityType: "resource",
+            entityId: resourceId,
+            orgId: orgId,
+            status: "healthy",
+            timestamp: Math.floor(Date.now() / 1000)
+        });
+
         await processAlerts({
             eventType: "resource_healthy",
             orgId,
@@ -81,6 +90,14 @@ export async function fireResourceUnhealthyAlert(
     extra?: Record<string, unknown>
 ): Promise<void> {
     try {
+        await db.insert(statusHistory).values({
+            entityType: "resource",
+            entityId: resourceId,
+            orgId: orgId,
+            status: "unhealthy",
+            timestamp: Math.floor(Date.now() / 1000)
+        });
+
         await processAlerts({
             eventType: "resource_unhealthy",
             orgId,

server/private/lib/alerts/events/siteEvents.ts

@@ -13,6 +13,9 @@
 
 import logger from "@server/logger";
 import { processAlerts } from "../processAlerts";
+import { db, sites, statusHistory, targetHealthCheck } from "@server/db";
+import { and, eq, inArray } from "drizzle-orm";
+import { fireHealthCheckUnhealthyAlert } from "./healthCheckEvents";
 
 // ---------------------------------------------------------------------------
 // Public API
@@ -36,6 +39,14 @@ export async function fireSiteOnlineAlert(
     extra?: Record<string, unknown>
 ): Promise<void> {
     try {
+        await db.insert(statusHistory).values({
+            entityType: "site",
+            entityId: siteId,
+            orgId: orgId,
+            status: "online",
+            timestamp: Math.floor(Date.now() / 1000)
+        });
+
         await processAlerts({
             eventType: "site_online",
             orgId,
@@ -81,6 +92,37 @@ export async function fireSiteOfflineAlert(
     extra?: Record<string, unknown>
 ): Promise<void> {
     try {
+        await db.insert(statusHistory).values({
+            entityType: "site",
+            entityId: siteId,
+            orgId: orgId,
+            status: "offline",
+            timestamp: Math.floor(Date.now() / 1000)
+        });
+
+        const unhealthyHealthChecks = await db
+            .update(targetHealthCheck)
+            .set({ hcHealth: "unhealthy" })
+            .where(
+                and(
+                    eq(targetHealthCheck.orgId, orgId),
+                    eq(targetHealthCheck.siteId, siteId)
+                )
+            )
+            .returning();
+
+        for (const healthCheck of unhealthyHealthChecks) {
+            logger.info(
+                `Marking health check ${healthCheck.targetHealthCheckId} unhealthy due to site ${siteId} being marked offline`
+            );
+
+            await fireHealthCheckUnhealthyAlert(
+                healthCheck.orgId,
+                healthCheck.targetHealthCheckId,
+                healthCheck.name
+            );
+        }
+
         await processAlerts({
             eventType: "site_offline",
             orgId,