calvin.erfmann/pangolin
1
0
Fork 0
mirror of https://github.com/fosrl/pangolin.git synced 2026-05-08 21:37:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix CE not processing alert status

Browse Source 
Fixes #2968
This commit is contained in:
Owen
2026-05-02 13:38:05 -07:00
parent e1afbc226c
commit b8822b4d25
22 changed files with 729 additions and 807 deletions
Download Patch File Download Diff File Expand all files Collapse all files

306
server/private/lib/alerts/events/healthCheckEvents.ts
View File

@@ -1,306 +0,0 @@
/*
* This file is part of a proprietary work.
*
* Copyright (c) 2025-2026 Fossorial, Inc.
* All rights reserved.
*
* This file is licensed under the Fossorial Commercial License.
* You may not use this file except in compliance with the License.
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
*
* This file is not licensed under the AGPLv3.
*/
import logger from "@server/logger";
import { processAlerts } from "../processAlerts";
import {
db,
statusHistory,
targetHealthCheck,
targets,
resources,
Transaction,
logsDb
} from "@server/db";
import { eq } from "drizzle-orm";
import { invalidateStatusHistoryCache } from "@server/lib/statusHistory";
import {
fireResourceDegradedAlert,
fireResourceHealthyAlert,
fireResourceUnhealthyAlert,
fireResourceUnknownAlert
} from "./resourceEvents";
// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------
/**
* Fire a `health_check_healthy` alert for the given health check.
*
* Call this after a previously-failing health check has recovered so that any
* matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the health check.
* @param healthCheckId - Numeric primary key of the health check.
* @param healthCheckName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireHealthCheckHealthyAlert(
orgId: string,
healthCheckId: number,
healthCheckName?: string | null,
healthCheckTargetId?: number | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "health_check",
entityId: healthCheckId,
orgId: orgId,
status: "healthy",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("health_check", healthCheckId);
await handleResource(orgId, healthCheckTargetId, send, trx);
if (!send) {
return;
}
await processAlerts({
eventType: "health_check_healthy",
orgId,
healthCheckId,
data: {
...(healthCheckName != null ? { healthCheckName } : {}),
...extra
}
});
await processAlerts({
eventType: "health_check_toggle",
orgId,
healthCheckId,
data: {
healthCheckId,
status: "healthy",
...(healthCheckName != null ? { healthCheckName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireHealthCheckHealthyAlert: unexpected error for healthCheckId ${healthCheckId}`,
err
);
}
}
/**
* Fire a `health_check_unhealthy` alert for the given health check.
*
* Call this after a health check has been detected as failing so that any
* matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the health check.
* @param healthCheckId - Numeric primary key of the health check.
* @param healthCheckName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireHealthCheckUnhealthyAlert(
orgId: string,
healthCheckId: number,
healthCheckName?: string | null,
healthCheckTargetId?: number | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "health_check",
entityId: healthCheckId,
orgId: orgId,
status: "unhealthy",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("health_check", healthCheckId);
await handleResource(orgId, healthCheckTargetId, send, trx);
if (!send) {
return;
}
await processAlerts({
eventType: "health_check_unhealthy",
orgId,
healthCheckId,
data: {
...(healthCheckName != null ? { healthCheckName } : {}),
...extra
}
});
await processAlerts({
eventType: "health_check_toggle",
orgId,
healthCheckId,
data: {
healthCheckId,
status: "unhealthy",
...(healthCheckName != null ? { healthCheckName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireHealthCheckUnhealthyAlert: unexpected error for healthCheckId ${healthCheckId}`,
err
);
}
}
export async function fireHealthCheckUnknownAlert(
orgId: string,
healthCheckId: number,
healthCheckName?: string | null,
healthCheckTargetId?: number | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "health_check",
entityId: healthCheckId,
orgId: orgId,
status: "unknown",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("health_check", healthCheckId);
await handleResource(orgId, healthCheckTargetId, send, trx);
if (!send) {
return;
}
} catch (err) {
logger.error(
`fireHealthCheckUnknownAlert: unexpected error for healthCheckId ${healthCheckId}`,
err
);
}
}
async function handleResource(
orgId: string,
healthCheckTargetId?: number | null,
send: boolean = true,
trx: Transaction | typeof db = db
) {
if (!healthCheckTargetId) {
return;
}
// we have targets lets get them
const [target] = await trx
.select()
.from(targets)
.where(eq(targets.targetId, healthCheckTargetId))
.limit(1);
if (!target) {
return;
}
const [resource] = await trx
.select()
.from(resources)
.where(eq(resources.resourceId, target.resourceId))
.limit(1);
if (!resource) {
return;
}
const otherTargets = await trx
.select({ hcHealth: targetHealthCheck.hcHealth })
.from(targets)
.innerJoin(
targetHealthCheck,
eq(targetHealthCheck.targetId, targets.targetId)
)
.where(eq(targets.resourceId, resource.resourceId));
let health = "healthy";
const allUnknown = otherTargets.every((t) => t.hcHealth === "unknown");
const allHealthy = otherTargets.every((t) => t.hcHealth === "healthy");
const allUnhealthy = otherTargets.every((t) => t.hcHealth === "unhealthy");
if (allUnknown) {
logger.debug(
`Marking resource ${resource.resourceId} as unknown because all health checks are disabled`
);
health = "unknown";
} else if (allHealthy) {
health = "healthy";
} else if (allUnhealthy) {
logger.debug(
`Marking resource ${resource.resourceId} as unhealthy because all targets are unhealthy`
);
health = "unhealthy";
} else {
logger.debug(
`Marking resource ${resource.resourceId} as degraded because some targets are unhealthy`
);
health = "degraded";
}
if (health != resource.health) {
// it changed
await trx
.update(resources)
.set({ health })
.where(eq(resources.resourceId, resource.resourceId));
if (health === "unknown") {
await fireResourceUnknownAlert(
orgId,
resource.resourceId,
resource.name,
undefined,
send,
trx
);
} else if (health === "unhealthy") {
await fireResourceUnhealthyAlert(
orgId,
resource.resourceId,
resource.name,
undefined,
send,
trx
);
} else if (health === "healthy") {
await fireResourceHealthyAlert(
orgId,
resource.resourceId,
resource.name,
undefined,
send,
trx
);
} else if (health === "degraded") {
await fireResourceDegradedAlert(
orgId,
resource.resourceId,
resource.name,
undefined,
send,
trx
);
}
}
}

256
server/private/lib/alerts/events/resourceEvents.ts
View File

@@ -1,256 +0,0 @@
/*
* This file is part of a proprietary work.
*
* Copyright (c) 2025-2026 Fossorial, Inc.
* All rights reserved.
*
* This file is licensed under the Fossorial Commercial License.
* You may not use this file except in compliance with the License.
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
*
* This file is not licensed under the AGPLv3.
*/
import logger from "@server/logger";
import { processAlerts } from "../processAlerts";
import { db, logsDb, statusHistory, Transaction } from "@server/db";
import { invalidateStatusHistoryCache } from "@server/lib/statusHistory";
// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------
/**
* Fire a `resource_healthy` alert for the given resource.
*
* Call this after a previously-unhealthy resource has recovered so that any
* matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the resource.
* @param resourceId - Numeric primary key of the resource.
* @param resourceName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireResourceHealthyAlert(
orgId: string,
resourceId: number,
resourceName?: string | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "resource",
entityId: resourceId,
orgId: orgId,
status: "healthy",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("resource", resourceId);
if (!send) {
return;
}
await processAlerts({
eventType: "resource_healthy",
orgId,
resourceId,
data: {
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
await processAlerts({
eventType: "resource_toggle",
orgId,
resourceId,
data: {
resourceId,
status: "healthy",
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireResourceHealthyAlert: unexpected error for resourceId ${resourceId}`,
err
);
}
}
/**
* Fire a `resource_unhealthy` alert for the given resource.
*
* Call this after a resource has been detected as unhealthy so that any
* matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the resource.
* @param resourceId - Numeric primary key of the resource.
* @param resourceName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireResourceUnhealthyAlert(
orgId: string,
resourceId: number,
resourceName?: string | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "resource",
entityId: resourceId,
orgId: orgId,
status: "unhealthy",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("resource", resourceId);
if (!send) {
return;
}
await processAlerts({
eventType: "resource_unhealthy",
orgId,
resourceId,
data: {
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
await processAlerts({
eventType: "resource_toggle",
orgId,
resourceId,
data: {
resourceId,
status: "unhealthy",
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireResourceUnhealthyAlert: unexpected error for resourceId ${resourceId}`,
err
);
}
}
/**
* Fire a `resource_degraded` alert for the given resource.
*
* Call this after a resource has been detected as degraded so that any
* matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the resource.
* @param resourceId - Numeric primary key of the resource.
* @param resourceName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireResourceDegradedAlert(
orgId: string,
resourceId: number,
resourceName?: string | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "resource",
entityId: resourceId,
orgId: orgId,
status: "degraded",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("resource", resourceId);
if (!send) {
return;
}
await processAlerts({
eventType: "resource_degraded",
orgId,
resourceId,
data: {
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
await processAlerts({
eventType: "resource_toggle",
orgId,
resourceId,
data: {
resourceId,
status: "degraded",
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireResourceDegradedAlert: unexpected error for resourceId ${resourceId}`,
err
);
}
}
/**
* Fire a `resource_unknown` alert for the given resource.
*
* Call this when all health checks on a resource are disabled so that the
* resource status transitions to unknown.
*
* @param orgId - Organisation that owns the resource.
* @param resourceId - Numeric primary key of the resource.
* @param resourceName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireResourceUnknownAlert(
orgId: string,
resourceId: number,
resourceName?: string | null,
extra?: Record<string, unknown>,
send: boolean = true,
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "resource",
entityId: resourceId,
orgId: orgId,
status: "unknown",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("resource", resourceId);
if (!send) {
return;
}
await processAlerts({
eventType: "resource_toggle",
orgId,
resourceId,
data: {
resourceId,
status: "unknown",
...(resourceName != null ? { resourceName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireResourceUnknownAlert: unexpected error for resourceId ${resourceId}`,
err
);
}
}

169
server/private/lib/alerts/events/siteEvents.ts
View File

@@ -1,169 +0,0 @@
/*
* This file is part of a proprietary work.
*
* Copyright (c) 2025-2026 Fossorial, Inc.
* All rights reserved.
*
* This file is licensed under the Fossorial Commercial License.
* You may not use this file except in compliance with the License.
* Unauthorized use, copying, modification, or distribution is strictly prohibited.
*
* This file is not licensed under the AGPLv3.
*/
import logger from "@server/logger";
import { processAlerts } from "../processAlerts";
import {
db,
logsDb,
statusHistory,
targetHealthCheck,
Transaction
} from "@server/db";
import { invalidateStatusHistoryCache } from "@server/lib/statusHistory";
import { and, eq, inArray } from "drizzle-orm";
import { fireHealthCheckUnhealthyAlert } from "./healthCheckEvents";
// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------
/**
* Fire a `site_online` alert for the given site.
*
* Call this after the site has been confirmed reachable / connected so that
* any matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the site.
* @param siteId - Numeric primary key of the site.
* @param siteName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireSiteOnlineAlert(
orgId: string,
siteId: number,
siteName?: string,
extra?: Record<string, unknown>,
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "site",
entityId: siteId,
orgId: orgId,
status: "online",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("site", siteId);
await processAlerts({
eventType: "site_online",
orgId,
siteId,
data: {
...(siteName != null ? { siteName } : {}),
...extra
}
});
await processAlerts({
eventType: "site_toggle",
orgId,
siteId,
data: {
siteId,
status: "online",
...(siteName != null ? { siteName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireSiteOnlineAlert: unexpected error for siteId ${siteId}`,
err
);
}
}
/**
* Fire a `site_offline` alert for the given site.
*
* Call this after the site has been detected as unreachable / disconnected so
* that any matching `alertRules` can dispatch their email and webhook actions.
*
* @param orgId - Organisation that owns the site.
* @param siteId - Numeric primary key of the site.
* @param siteName - Human-readable name shown in notifications (optional).
* @param extra - Any additional key/value pairs to include in the payload.
*/
export async function fireSiteOfflineAlert(
orgId: string,
siteId: number,
siteName?: string,
extra?: Record<string, unknown>,
trx: Transaction | typeof db = db
): Promise<void> {
try {
await logsDb.insert(statusHistory).values({
entityType: "site",
entityId: siteId,
orgId: orgId,
status: "offline",
timestamp: Math.floor(Date.now() / 1000)
});
await invalidateStatusHistoryCache("site", siteId);
const unhealthyHealthChecks = await trx
.update(targetHealthCheck)
.set({ hcHealth: "unhealthy" })
.where(
and(
eq(targetHealthCheck.orgId, orgId),
eq(targetHealthCheck.siteId, siteId),
eq(targetHealthCheck.hcEnabled, true) // only effect the ones that are enabled
)
)
.returning();
for (const healthCheck of unhealthyHealthChecks) {
logger.info(
`Marking health check ${healthCheck.targetHealthCheckId} unhealthy due to site ${siteId} being marked offline`
);
await fireHealthCheckUnhealthyAlert(
healthCheck.orgId,
healthCheck.targetHealthCheckId,
healthCheck.name,
healthCheck.targetId, // for the resource if we have one
undefined,
true,
trx
);
}
await processAlerts({
eventType: "site_offline",
orgId,
siteId,
data: {
...(siteName != null ? { siteName } : {}),
...extra
}
});
await processAlerts({
eventType: "site_toggle",
orgId,
siteId,
data: {
siteId,
status: "offline",
...(siteName != null ? { siteName } : {}),
...extra
}
});
} catch (err) {
logger.error(
`fireSiteOfflineAlert: unexpected error for siteId ${siteId}`,
err
);
}
}

3
server/private/lib/alerts/index.ts
View File

@@ -14,6 +14,3 @@
export * from "./processAlerts";
export * from "./sendAlertWebhook";
export * from "./sendAlertEmail";
export * from "./events/siteEvents";
export * from "./events/healthCheckEvents";
export * from "./events/resourceEvents";

7
server/private/routers/alertEvents/triggerHealthCheckAlert.ts
View File

@@ -24,7 +24,7 @@ import { eq, and } from "drizzle-orm";
import {
fireHealthCheckHealthyAlert,
fireHealthCheckUnhealthyAlert
} from "#private/lib/alerts/events/healthCheckEvents";
} from "@server/lib/alerts";
const paramsSchema = z.strictObject({
orgId: z.string().nonempty(),
@@ -73,10 +73,7 @@ export async function triggerHealthCheckAlert(
.from(targetHealthCheck)
.where(
and(
eq(
targetHealthCheck.targetHealthCheckId,
healthCheckId
),
eq(targetHealthCheck.targetHealthCheckId, healthCheckId),
eq(targetHealthCheck.orgId, orgId)
)
)

2
server/private/routers/alertEvents/triggerResourceAlert.ts
View File

@@ -25,7 +25,7 @@ import {
fireResourceHealthyAlert,
fireResourceUnhealthyAlert,
fireResourceDegradedAlert
} from "#private/lib/alerts/events/resourceEvents";
} from "@server/lib/alerts";
const paramsSchema = z.strictObject({
orgId: z.string().nonempty(),

5
server/private/routers/alertEvents/triggerSiteAlert.ts
View File

@@ -21,10 +21,7 @@ import createHttpError from "http-errors";
import logger from "@server/logger";
import { fromError } from "zod-validation-error";
import { eq, and } from "drizzle-orm";
import {
fireSiteOnlineAlert,
fireSiteOfflineAlert
} from "#private/lib/alerts/events/siteEvents";
import { fireSiteOnlineAlert, fireSiteOfflineAlert } from "@server/lib/alerts";
const paramsSchema = z.strictObject({
orgId: z.string().nonempty(),

2
server/private/routers/healthChecks/createHealthCheck.ts
View File

@@ -22,7 +22,7 @@ import logger from "@server/logger";
import { fromError } from "zod-validation-error";
import { OpenAPITags, registry } from "@server/openApi";
import { addStandaloneHealthCheck } from "@server/routers/newt/targets";
import { fireHealthCheckUnhealthyAlert } from "#private/lib/alerts";
import { fireHealthCheckUnhealthyAlert } from "@server/lib/alerts";
const paramsSchema = z.strictObject({
orgId: z.string().nonempty()

22
server/private/routers/healthChecks/updateHealthCheck.ts
View File

@@ -22,7 +22,11 @@ import { fromError } from "zod-validation-error";
import { OpenAPITags, registry } from "@server/openApi";
import { and, eq, isNull } from "drizzle-orm";
import { addStandaloneHealthCheck } from "@server/routers/newt/targets";
import { fireHealthCheckUnhealthyAlert, fireHealthCheckUnknownAlert, fireHealthCheckHealthyAlert } from "#private/lib/alerts";
import {
fireHealthCheckUnhealthyAlert,
fireHealthCheckUnknownAlert,
fireHealthCheckHealthyAlert
} from "@server/lib/alerts";
const paramsSchema = z
.object({
@@ -234,7 +238,10 @@ export async function updateHealthCheck(
)
.returning();
if (updated.hcHealth === "unhealthy" && existingHealthCheck.hcHealth !== "unhealthy") {
if (
updated.hcHealth === "unhealthy" &&
existingHealthCheck.hcHealth !== "unhealthy"
) {
await fireHealthCheckUnhealthyAlert(
updated.orgId,
updated.targetHealthCheckId,
@@ -243,7 +250,10 @@ export async function updateHealthCheck(
undefined,
false // dont send the alert because we just want to create the alert, not notify users yet
);
} else if (updated.hcHealth === "unknown" && existingHealthCheck.hcHealth !== "unknown") {
} else if (
updated.hcHealth === "unknown" &&
existingHealthCheck.hcHealth !== "unknown"
) {
// if the health is unknown, we want to fire an alert to notify users to enable health checks
await fireHealthCheckUnknownAlert(
updated.orgId,
@@ -253,7 +263,10 @@ export async function updateHealthCheck(
undefined,
false // dont send the alert because we just want to create the alert, not notify users yet
);
} else if (updated.hcHealth === "healthy" && existingHealthCheck.hcHealth !== "healthy") {
} else if (
updated.hcHealth === "healthy" &&
existingHealthCheck.hcHealth !== "healthy"
) {
await fireHealthCheckHealthyAlert(
updated.orgId,
updated.targetHealthCheckId,
@@ -264,7 +277,6 @@ export async function updateHealthCheck(
);
}
// Push updated health check to newt if the site is a newt site
const [newt] = await db
.select()